{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1872,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005344378381989445,
      "grad_norm": 63.52920150756836,
      "learning_rate": 0.0,
      "logits/chosen": -0.144775390625,
      "logits/rejected": -0.1036376953125,
      "logps/chosen": -110.828125,
      "logps/rejected": -116.828125,
      "logps/weighted_chosen": -0.2086181640625,
      "logps/weighted_rejected": -1.271392822265625,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/weighted_accuracies": 0.0,
      "rewards/weighted_chosen": 0.0,
      "rewards/weighted_margins": 0.0,
      "rewards/weighted_rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005344378381989445,
      "grad_norm": 102.77810668945312,
      "learning_rate": 4.787234042553191e-08,
      "logits/chosen": -0.2549913227558136,
      "logits/rejected": -0.24070316553115845,
      "logps/chosen": -135.32899475097656,
      "logps/rejected": -138.99305725097656,
      "logps/weighted_chosen": -0.8547499179840088,
      "logps/weighted_rejected": -0.9027608036994934,
      "loss": 0.6928,
      "rewards/accuracies": 0.2951388955116272,
      "rewards/chosen": -0.0164930559694767,
      "rewards/margins": -0.0349392369389534,
      "rewards/rejected": 0.0184461809694767,
      "rewards/weighted_accuracies": 0.4027777910232544,
      "rewards/weighted_chosen": -0.002239651046693325,
      "rewards/weighted_margins": 0.0028160943184047937,
      "rewards/weighted_rejected": -0.0050794812850654125,
      "step": 10
    },
    {
      "epoch": 0.01068875676397889,
      "grad_norm": 9.201544761657715,
      "learning_rate": 1.0106382978723404e-07,
      "logits/chosen": -0.19488906860351562,
      "logits/rejected": -0.183399960398674,
      "logps/chosen": -116.0374984741211,
      "logps/rejected": -115.5054702758789,
      "logps/weighted_chosen": -0.6929916143417358,
      "logps/weighted_rejected": -0.6838119626045227,
      "loss": 0.6917,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.01796874962747097,
      "rewards/margins": 0.03007812425494194,
      "rewards/rejected": -0.01210937462747097,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": 0.0022293091751635075,
      "rewards/weighted_margins": 0.00339679722674191,
      "rewards/weighted_rejected": -0.0011629104847088456,
      "step": 20
    },
    {
      "epoch": 0.016033135145968335,
      "grad_norm": 7.109442234039307,
      "learning_rate": 1.5425531914893615e-07,
      "logits/chosen": -0.1776607483625412,
      "logits/rejected": -0.17884215712547302,
      "logps/chosen": -123.2867202758789,
      "logps/rejected": -129.75,
      "logps/weighted_chosen": -0.7567901611328125,
      "logps/weighted_rejected": -1.2207733392715454,
      "loss": 0.6913,
      "rewards/accuracies": 0.3968749940395355,
      "rewards/chosen": -0.0074218749068677425,
      "rewards/margins": 0.04843749850988388,
      "rewards/rejected": -0.05585937574505806,
      "rewards/weighted_accuracies": 0.5,
      "rewards/weighted_chosen": 0.0007101058727130294,
      "rewards/weighted_margins": 0.004384422209113836,
      "rewards/weighted_rejected": -0.0036745071411132812,
      "step": 30
    },
    {
      "epoch": 0.02137751352795778,
      "grad_norm": 6.0048956871032715,
      "learning_rate": 2.074468085106383e-07,
      "logits/chosen": -0.23842163383960724,
      "logits/rejected": -0.2298881560564041,
      "logps/chosen": -127.34062194824219,
      "logps/rejected": -128.1710968017578,
      "logps/weighted_chosen": -0.8051223754882812,
      "logps/weighted_rejected": -0.5736129879951477,
      "loss": 0.6934,
      "rewards/accuracies": 0.34062498807907104,
      "rewards/chosen": -0.01992187462747097,
      "rewards/margins": -0.05351562425494194,
      "rewards/rejected": 0.03359375149011612,
      "rewards/weighted_accuracies": 0.4593749940395355,
      "rewards/weighted_chosen": -7.83920258982107e-05,
      "rewards/weighted_margins": -0.0009832382202148438,
      "rewards/weighted_rejected": 0.000904083251953125,
      "step": 40
    },
    {
      "epoch": 0.026721891909947223,
      "grad_norm": 32.7233772277832,
      "learning_rate": 2.6063829787234044e-07,
      "logits/chosen": -0.18835143744945526,
      "logits/rejected": -0.170085147023201,
      "logps/chosen": -120.88594055175781,
      "logps/rejected": -121.3515625,
      "logps/weighted_chosen": -0.6867736577987671,
      "logps/weighted_rejected": -1.138249158859253,
      "loss": 0.6903,
      "rewards/accuracies": 0.3656249940395355,
      "rewards/chosen": 0.03554687649011612,
      "rewards/margins": -0.03164062649011612,
      "rewards/rejected": 0.06718750298023224,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.006287765689194202,
      "rewards/weighted_margins": 0.0064636231400072575,
      "rewards/weighted_rejected": -0.01274185162037611,
      "step": 50
    },
    {
      "epoch": 0.03206627029193667,
      "grad_norm": 55.560707092285156,
      "learning_rate": 3.1382978723404253e-07,
      "logits/chosen": -0.1308601349592209,
      "logits/rejected": -0.1308906525373459,
      "logps/chosen": -116.5875015258789,
      "logps/rejected": -119.5406265258789,
      "logps/weighted_chosen": -0.8019469976425171,
      "logps/weighted_rejected": -0.8834930658340454,
      "loss": 0.7002,
      "rewards/accuracies": 0.3343749940395355,
      "rewards/chosen": 0.03476562350988388,
      "rewards/margins": -0.06171875074505806,
      "rewards/rejected": 0.09648437798023224,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.01903533935546875,
      "rewards/weighted_margins": -2.880096508306451e-05,
      "rewards/weighted_rejected": -0.019007300958037376,
      "step": 60
    },
    {
      "epoch": 0.037410648673926114,
      "grad_norm": 20.22484588623047,
      "learning_rate": 3.6702127659574467e-07,
      "logits/chosen": -0.20572929084300995,
      "logits/rejected": -0.19457244873046875,
      "logps/chosen": -114.4507827758789,
      "logps/rejected": -118.63594055175781,
      "logps/weighted_chosen": -0.6070801019668579,
      "logps/weighted_rejected": -0.6768280267715454,
      "loss": 0.6911,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.16484375298023224,
      "rewards/margins": 0.07148437201976776,
      "rewards/rejected": 0.09335937350988388,
      "rewards/weighted_accuracies": 0.4749999940395355,
      "rewards/weighted_chosen": -0.010281562805175781,
      "rewards/weighted_margins": 0.007259177975356579,
      "rewards/weighted_rejected": -0.017521286383271217,
      "step": 70
    },
    {
      "epoch": 0.04275502705591556,
      "grad_norm": 15.305061340332031,
      "learning_rate": 4.202127659574468e-07,
      "logits/chosen": -0.19508513808250427,
      "logits/rejected": -0.12093047797679901,
      "logps/chosen": -112.57734680175781,
      "logps/rejected": -112.77734375,
      "logps/weighted_chosen": -0.4597320556640625,
      "logps/weighted_rejected": -0.9711883664131165,
      "loss": 0.6952,
      "rewards/accuracies": 0.37812501192092896,
      "rewards/chosen": 0.04570312425494194,
      "rewards/margins": 0.01914062537252903,
      "rewards/rejected": 0.02656250074505806,
      "rewards/weighted_accuracies": 0.4625000059604645,
      "rewards/weighted_chosen": -0.02759857103228569,
      "rewards/weighted_margins": 0.021048355847597122,
      "rewards/weighted_rejected": -0.04872245714068413,
      "step": 80
    },
    {
      "epoch": 0.048099405437905,
      "grad_norm": 11.991273880004883,
      "learning_rate": 4.734042553191489e-07,
      "logits/chosen": -0.17391815781593323,
      "logits/rejected": -0.11656417697668076,
      "logps/chosen": -119.3070297241211,
      "logps/rejected": -118.51249694824219,
      "logps/weighted_chosen": -1.0416854619979858,
      "logps/weighted_rejected": -1.5859466791152954,
      "loss": 0.6736,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08164062350988388,
      "rewards/margins": 0.341796875,
      "rewards/rejected": -0.4234375059604645,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.04084587097167969,
      "rewards/weighted_margins": 0.10567589104175568,
      "rewards/weighted_rejected": -0.14661255478858948,
      "step": 90
    },
    {
      "epoch": 0.053443783819894446,
      "grad_norm": 17.195886611938477,
      "learning_rate": 5.26595744680851e-07,
      "logits/chosen": -0.204203799366951,
      "logits/rejected": -0.14691162109375,
      "logps/chosen": -125.05155944824219,
      "logps/rejected": -125.3671875,
      "logps/weighted_chosen": -0.7374237179756165,
      "logps/weighted_rejected": -0.7348953485488892,
      "loss": 0.7064,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.3832031190395355,
      "rewards/margins": 0.15585938096046448,
      "rewards/rejected": -0.5390625,
      "rewards/weighted_accuracies": 0.46875,
      "rewards/weighted_chosen": -0.060018159449100494,
      "rewards/weighted_margins": 0.0024120330344885588,
      "rewards/weighted_rejected": -0.06242714077234268,
      "step": 100
    },
    {
      "epoch": 0.058788162201883896,
      "grad_norm": 9.103920936584473,
      "learning_rate": 5.797872340425531e-07,
      "logits/chosen": -0.2138931304216385,
      "logits/rejected": -0.19011077284812927,
      "logps/chosen": -118.6820297241211,
      "logps/rejected": -121.28593444824219,
      "logps/weighted_chosen": -0.4821228086948395,
      "logps/weighted_rejected": -0.7759445309638977,
      "loss": 0.6939,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.896484375,
      "rewards/margins": 0.09531249850988388,
      "rewards/rejected": -0.9917968511581421,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.05413341522216797,
      "rewards/weighted_margins": 0.04503212124109268,
      "rewards/weighted_rejected": -0.09918193519115448,
      "step": 110
    },
    {
      "epoch": 0.06413254058387334,
      "grad_norm": 8.539896011352539,
      "learning_rate": 6.329787234042553e-07,
      "logits/chosen": -0.17510680854320526,
      "logits/rejected": -0.14726562798023224,
      "logps/chosen": -122.58906555175781,
      "logps/rejected": -128.4656219482422,
      "logps/weighted_chosen": -0.9249496459960938,
      "logps/weighted_rejected": -0.762683093547821,
      "loss": 0.7325,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.068359375,
      "rewards/margins": 0.28398436307907104,
      "rewards/rejected": -1.3523437976837158,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.098027803003788,
      "rewards/weighted_margins": -0.029563140124082565,
      "rewards/weighted_rejected": -0.06836700439453125,
      "step": 120
    },
    {
      "epoch": 0.06947691896586278,
      "grad_norm": 15.564044952392578,
      "learning_rate": 6.861702127659574e-07,
      "logits/chosen": -0.20233002305030823,
      "logits/rejected": -0.18396148085594177,
      "logps/chosen": -122.2671890258789,
      "logps/rejected": -123.8382797241211,
      "logps/weighted_chosen": -0.7501281499862671,
      "logps/weighted_rejected": -0.821154773235321,
      "loss": 0.7078,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -1.45703125,
      "rewards/margins": 0.29179686307907104,
      "rewards/rejected": -1.7488281726837158,
      "rewards/weighted_accuracies": 0.5062500238418579,
      "rewards/weighted_chosen": -0.007884788326919079,
      "rewards/weighted_margins": 0.006314086727797985,
      "rewards/weighted_rejected": -0.014200759120285511,
      "step": 130
    },
    {
      "epoch": 0.07482129734785223,
      "grad_norm": 9.176558494567871,
      "learning_rate": 7.393617021276596e-07,
      "logits/chosen": -0.16455383598804474,
      "logits/rejected": -0.14660033583641052,
      "logps/chosen": -116.328125,
      "logps/rejected": -119.9664077758789,
      "logps/weighted_chosen": -0.9873703122138977,
      "logps/weighted_rejected": -0.6730011105537415,
      "loss": 0.7174,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.6007812023162842,
      "rewards/margins": 0.533203125,
      "rewards/rejected": -2.133984327316284,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": 0.02084655687212944,
      "rewards/weighted_margins": 0.05102996900677681,
      "rewards/weighted_rejected": -0.030106354504823685,
      "step": 140
    },
    {
      "epoch": 0.08016567572984168,
      "grad_norm": 10.172534942626953,
      "learning_rate": 7.925531914893616e-07,
      "logits/chosen": -0.1509498655796051,
      "logits/rejected": -0.13156357407569885,
      "logps/chosen": -120.0562515258789,
      "logps/rejected": -117.85546875,
      "logps/weighted_chosen": -0.8198806643486023,
      "logps/weighted_rejected": -0.7782562375068665,
      "loss": 0.7005,
      "rewards/accuracies": 0.46562498807907104,
      "rewards/chosen": -2.0234375,
      "rewards/margins": 0.607421875,
      "rewards/rejected": -2.630859375,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.0781441181898117,
      "rewards/weighted_margins": 0.034273337572813034,
      "rewards/weighted_rejected": -0.11248989403247833,
      "step": 150
    },
    {
      "epoch": 0.08551005411183112,
      "grad_norm": 25.879701614379883,
      "learning_rate": 8.457446808510637e-07,
      "logits/chosen": -0.17474937438964844,
      "logits/rejected": -0.17009583115577698,
      "logps/chosen": -121.2367172241211,
      "logps/rejected": -121.62109375,
      "logps/weighted_chosen": -0.8304061889648438,
      "logps/weighted_rejected": -0.866925060749054,
      "loss": 0.7072,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -2.219531297683716,
      "rewards/margins": 0.716796875,
      "rewards/rejected": -2.936328172683716,
      "rewards/weighted_accuracies": 0.47187501192092896,
      "rewards/weighted_chosen": -0.07932891696691513,
      "rewards/weighted_margins": 0.017035294324159622,
      "rewards/weighted_rejected": -0.09644661098718643,
      "step": 160
    },
    {
      "epoch": 0.09085443249382057,
      "grad_norm": 9.220558166503906,
      "learning_rate": 8.989361702127659e-07,
      "logits/chosen": -0.22574539482593536,
      "logits/rejected": -0.19882050156593323,
      "logps/chosen": -120.62968444824219,
      "logps/rejected": -124.78437805175781,
      "logps/weighted_chosen": -0.9401000738143921,
      "logps/weighted_rejected": -0.8424423336982727,
      "loss": 0.7174,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -2.4033203125,
      "rewards/margins": 0.613085925579071,
      "rewards/rejected": -3.016406297683716,
      "rewards/weighted_accuracies": 0.48750001192092896,
      "rewards/weighted_chosen": -0.04837837070226669,
      "rewards/weighted_margins": 0.04595222324132919,
      "rewards/weighted_rejected": -0.09437103569507599,
      "step": 170
    },
    {
      "epoch": 0.09619881087581,
      "grad_norm": 55.62498092651367,
      "learning_rate": 9.52127659574468e-07,
      "logits/chosen": -0.2216789275407791,
      "logits/rejected": -0.20373383164405823,
      "logps/chosen": -120.3851547241211,
      "logps/rejected": -121.2328109741211,
      "logps/weighted_chosen": -0.435232549905777,
      "logps/weighted_rejected": -0.5373809933662415,
      "loss": 0.7302,
      "rewards/accuracies": 0.47187501192092896,
      "rewards/chosen": -1.338281273841858,
      "rewards/margins": 0.3658203184604645,
      "rewards/rejected": -1.7041015625,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.08163909614086151,
      "rewards/weighted_margins": -0.008893203921616077,
      "rewards/weighted_rejected": -0.07281456142663956,
      "step": 180
    },
    {
      "epoch": 0.10154318925779945,
      "grad_norm": 39.38788986206055,
      "learning_rate": 9.99999129927477e-07,
      "logits/chosen": -0.19243164360523224,
      "logits/rejected": -0.15758666396141052,
      "logps/chosen": -121.8960952758789,
      "logps/rejected": -123.01094055175781,
      "logps/weighted_chosen": -0.953173816204071,
      "logps/weighted_rejected": -1.0853087902069092,
      "loss": 0.6702,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.100390672683716,
      "rewards/margins": 0.662890613079071,
      "rewards/rejected": -2.7632813453674316,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.09999503940343857,
      "rewards/weighted_margins": 0.12822990119457245,
      "rewards/weighted_rejected": -0.22818031907081604,
      "step": 190
    },
    {
      "epoch": 0.10688756763978889,
      "grad_norm": 12.883384704589844,
      "learning_rate": 9.99894724888679e-07,
      "logits/chosen": -0.17392578721046448,
      "logits/rejected": -0.147491455078125,
      "logps/chosen": -129.53750610351562,
      "logps/rejected": -128.95703125,
      "logps/weighted_chosen": -0.7322753667831421,
      "logps/weighted_rejected": -1.0728164911270142,
      "loss": 0.6777,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.215625047683716,
      "rewards/margins": 0.9507812261581421,
      "rewards/rejected": -4.166406154632568,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.1104557067155838,
      "rewards/weighted_margins": 0.13128623366355896,
      "rewards/weighted_rejected": -0.2415626496076584,
      "step": 200
    },
    {
      "epoch": 0.11223194602177834,
      "grad_norm": 17.96660041809082,
      "learning_rate": 9.996163469793475e-07,
      "logits/chosen": -0.18083801865577698,
      "logits/rejected": -0.12766647338867188,
      "logps/chosen": -128.1953125,
      "logps/rejected": -123.03047180175781,
      "logps/weighted_chosen": -0.8699920773506165,
      "logps/weighted_rejected": -0.8845611810684204,
      "loss": 0.7572,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.5113282203674316,
      "rewards/margins": 1.3683593273162842,
      "rewards/rejected": -4.879296779632568,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.07919616997241974,
      "rewards/weighted_margins": 0.04895629733800888,
      "rewards/weighted_rejected": -0.12815475463867188,
      "step": 210
    },
    {
      "epoch": 0.11757632440376779,
      "grad_norm": 63.790077209472656,
      "learning_rate": 9.991640930802883e-07,
      "logits/chosen": -0.14619751274585724,
      "logits/rejected": -0.13860473036766052,
      "logps/chosen": -126.61015319824219,
      "logps/rejected": -129.9773406982422,
      "logps/weighted_chosen": -1.0776779651641846,
      "logps/weighted_rejected": -0.9309722781181335,
      "loss": 0.7107,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -4.296484470367432,
      "rewards/margins": 1.10546875,
      "rewards/rejected": -5.40234375,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.013729858212172985,
      "rewards/weighted_margins": 0.14467468857765198,
      "rewards/weighted_rejected": -0.1582542359828949,
      "step": 220
    },
    {
      "epoch": 0.12292070278575723,
      "grad_norm": 9.634794235229492,
      "learning_rate": 9.98538120584459e-07,
      "logits/chosen": -0.1792152374982834,
      "logits/rejected": -0.1362098753452301,
      "logps/chosen": -133.7140655517578,
      "logps/rejected": -137.7546844482422,
      "logps/weighted_chosen": -0.6676223874092102,
      "logps/weighted_rejected": -0.9518234133720398,
      "loss": 0.7614,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -4.493554592132568,
      "rewards/margins": 1.0623047351837158,
      "rewards/rejected": -5.555859565734863,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.1120578795671463,
      "rewards/weighted_margins": 0.031700897961854935,
      "rewards/weighted_rejected": -0.14376182854175568,
      "step": 230
    },
    {
      "epoch": 0.12826508116774668,
      "grad_norm": 8.717682838439941,
      "learning_rate": 9.977386473421917e-07,
      "logits/chosen": -0.09295501559972763,
      "logits/rejected": -0.07836151123046875,
      "logps/chosen": -121.78202819824219,
      "logps/rejected": -126.39453125,
      "logps/weighted_chosen": -0.8116515874862671,
      "logps/weighted_rejected": -1.2165496349334717,
      "loss": 0.6731,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -7.553515434265137,
      "rewards/margins": 1.669921875,
      "rewards/rejected": -9.224218368530273,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.1755084991455078,
      "rewards/weighted_margins": 0.15414047241210938,
      "rewards/weighted_rejected": -0.32989272475242615,
      "step": 240
    },
    {
      "epoch": 0.13360945954973613,
      "grad_norm": 35.197269439697266,
      "learning_rate": 9.96765951585378e-07,
      "logits/chosen": -0.021549224853515625,
      "logits/rejected": 0.00345611572265625,
      "logps/chosen": -123.0296859741211,
      "logps/rejected": -127.9671859741211,
      "logps/weighted_chosen": -1.186151146888733,
      "logps/weighted_rejected": -1.386474609375,
      "loss": 0.7026,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -9.133593559265137,
      "rewards/margins": 1.150781273841858,
      "rewards/rejected": -10.283984184265137,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.3282794952392578,
      "rewards/weighted_margins": 0.1091407760977745,
      "rewards/weighted_rejected": -0.4371795654296875,
      "step": 250
    },
    {
      "epoch": 0.13895383793172555,
      "grad_norm": 84.2197494506836,
      "learning_rate": 9.956203718306388e-07,
      "logits/chosen": 0.025267411023378372,
      "logits/rejected": 0.06335143744945526,
      "logps/chosen": -128.8429718017578,
      "logps/rejected": -133.50625610351562,
      "logps/weighted_chosen": -0.9628280401229858,
      "logps/weighted_rejected": -1.2008453607559204,
      "loss": 0.7443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -10.861719131469727,
      "rewards/margins": 0.871874988079071,
      "rewards/rejected": -11.733593940734863,
      "rewards/weighted_accuracies": 0.49687498807907104,
      "rewards/weighted_chosen": -0.26259079575538635,
      "rewards/weighted_margins": 0.06770706176757812,
      "rewards/weighted_rejected": -0.3305191099643707,
      "step": 260
    },
    {
      "epoch": 0.144298216313715,
      "grad_norm": 57.09628677368164,
      "learning_rate": 9.943023067615136e-07,
      "logits/chosen": -0.017973709851503372,
      "logits/rejected": -0.0057998658157885075,
      "logps/chosen": -137.60311889648438,
      "logps/rejected": -140.91561889648438,
      "logps/weighted_chosen": -0.8694183230400085,
      "logps/weighted_rejected": -0.8910156488418579,
      "loss": 0.7462,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -10.946093559265137,
      "rewards/margins": 1.662109375,
      "rewards/rejected": -12.608202934265137,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.06702041625976562,
      "rewards/weighted_margins": 0.13949890434741974,
      "rewards/weighted_rejected": -0.20627327263355255,
      "step": 270
    },
    {
      "epoch": 0.14964259469570446,
      "grad_norm": 45.0294303894043,
      "learning_rate": 9.928122150897112e-07,
      "logits/chosen": -0.045014191418886185,
      "logits/rejected": 0.02437591552734375,
      "logps/chosen": -130.1125030517578,
      "logps/rejected": -131.6171875,
      "logps/weighted_chosen": -0.7659690976142883,
      "logps/weighted_rejected": -0.8499618768692017,
      "loss": 0.7079,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -12.664453506469727,
      "rewards/margins": 1.584375023841858,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.22265930473804474,
      "rewards/weighted_margins": 0.07556991279125214,
      "rewards/weighted_rejected": -0.298013299703598,
      "step": 280
    },
    {
      "epoch": 0.1549869730776939,
      "grad_norm": 9.370684623718262,
      "learning_rate": 9.9115061539547e-07,
      "logits/chosen": -0.02069244347512722,
      "logits/rejected": 0.01569976843893528,
      "logps/chosen": -139.4453125,
      "logps/rejected": -141.2578125,
      "logps/weighted_chosen": -0.8775177001953125,
      "logps/weighted_rejected": -1.2375701665878296,
      "loss": 0.6949,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -13.07421875,
      "rewards/margins": 1.177734375,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.18484345078468323,
      "rewards/weighted_margins": 0.12048187106847763,
      "rewards/weighted_rejected": -0.30528029799461365,
      "step": 290
    },
    {
      "epoch": 0.16033135145968336,
      "grad_norm": 30.68979263305664,
      "learning_rate": 9.893180859470818e-07,
      "logits/chosen": -0.0137786865234375,
      "logits/rejected": 0.041573334485292435,
      "logps/chosen": -130.22265625,
      "logps/rejected": -132.06094360351562,
      "logps/weighted_chosen": -1.1769134998321533,
      "logps/weighted_rejected": -1.2438628673553467,
      "loss": 0.7261,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -13.093358993530273,
      "rewards/margins": 2.108203172683716,
      "rewards/rejected": -15.200780868530273,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.2848831117153168,
      "rewards/weighted_margins": 0.07841148227453232,
      "rewards/weighted_rejected": -0.36317139863967896,
      "step": 300
    },
    {
      "epoch": 0.16567572984167278,
      "grad_norm": 71.56196594238281,
      "learning_rate": 9.873152644996424e-07,
      "logits/chosen": -0.11114044487476349,
      "logits/rejected": -0.11599807441234589,
      "logps/chosen": -131.01718139648438,
      "logps/rejected": -132.28125,
      "logps/weighted_chosen": -1.1032531261444092,
      "logps/weighted_rejected": -0.8420807123184204,
      "loss": 0.7872,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -12.979687690734863,
      "rewards/margins": 2.407421827316284,
      "rewards/rejected": -15.388280868530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": 0.05532989650964737,
      "rewards/weighted_margins": 0.2024696320295334,
      "rewards/weighted_rejected": -0.1469375640153885,
      "step": 310
    },
    {
      "epoch": 0.17102010822366223,
      "grad_norm": 12.055798530578613,
      "learning_rate": 9.85142848073103e-07,
      "logits/chosen": -0.10557708889245987,
      "logits/rejected": -0.06234588474035263,
      "logps/chosen": -133.39218139648438,
      "logps/rejected": -134.2109375,
      "logps/weighted_chosen": -0.700244128704071,
      "logps/weighted_rejected": -1.4902985095977783,
      "loss": 0.7558,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -14.2265625,
      "rewards/margins": 0.6207031011581421,
      "rewards/rejected": -14.846094131469727,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.2170669585466385,
      "rewards/weighted_margins": 0.03446922451257706,
      "rewards/weighted_rejected": -0.25187987089157104,
      "step": 320
    },
    {
      "epoch": 0.17636448660565168,
      "grad_norm": 41.381507873535156,
      "learning_rate": 9.828015927096914e-07,
      "logits/chosen": -0.085169218480587,
      "logits/rejected": -0.05568351596593857,
      "logps/chosen": -143.8234405517578,
      "logps/rejected": -142.39608764648438,
      "logps/weighted_chosen": -0.977459728717804,
      "logps/weighted_rejected": -0.7958831787109375,
      "loss": 0.6935,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -14.643750190734863,
      "rewards/margins": 1.578125,
      "rewards/rejected": -16.222265243530273,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.16206398606300354,
      "rewards/weighted_margins": 0.07153511047363281,
      "rewards/weighted_rejected": -0.23361511528491974,
      "step": 330
    },
    {
      "epoch": 0.18170886498764113,
      "grad_norm": 7.313599586486816,
      "learning_rate": 9.802923132107968e-07,
      "logits/chosen": -0.10784301906824112,
      "logits/rejected": -0.052645109593868256,
      "logps/chosen": -143.91561889648438,
      "logps/rejected": -147.7234344482422,
      "logps/weighted_chosen": -0.7267853021621704,
      "logps/weighted_rejected": -1.24786376953125,
      "loss": 0.7102,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -16.782812118530273,
      "rewards/margins": 1.23046875,
      "rewards/rejected": -18.012500762939453,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.1999809294939041,
      "rewards/weighted_margins": 0.056002043187618256,
      "rewards/weighted_rejected": -0.2561977505683899,
      "step": 340
    },
    {
      "epoch": 0.18705324336963056,
      "grad_norm": 66.15936279296875,
      "learning_rate": 9.776158828534024e-07,
      "logits/chosen": -0.12608642876148224,
      "logits/rejected": -0.09711913764476776,
      "logps/chosen": -143.48281860351562,
      "logps/rejected": -188.25936889648438,
      "logps/weighted_chosen": -1.260833740234375,
      "logps/weighted_rejected": -0.9638397097587585,
      "loss": 0.7338,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -16.727733612060547,
      "rewards/margins": 3.159374952316284,
      "rewards/rejected": -19.883594512939453,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.26622849702835083,
      "rewards/weighted_margins": 0.05975799635052681,
      "rewards/weighted_rejected": -0.3257743716239929,
      "step": 350
    },
    {
      "epoch": 0.19239762175162,
      "grad_norm": 10.4098539352417,
      "learning_rate": 9.747732330861695e-07,
      "logits/chosen": -0.05247039720416069,
      "logits/rejected": -0.024548912420868874,
      "logps/chosen": -135.78750610351562,
      "logps/rejected": -137.03671264648438,
      "logps/weighted_chosen": -0.8440704345703125,
      "logps/weighted_rejected": -0.8307411074638367,
      "loss": 0.7415,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -16.864452362060547,
      "rewards/margins": 2.5316405296325684,
      "rewards/rejected": -19.396093368530273,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": 0.005719375796616077,
      "rewards/weighted_margins": 0.17883948981761932,
      "rewards/weighted_rejected": -0.17323684692382812,
      "step": 360
    },
    {
      "epoch": 0.19774200013360946,
      "grad_norm": 8.8024263381958,
      "learning_rate": 9.717653532052742e-07,
      "logits/chosen": -0.0410335548222065,
      "logits/rejected": -0.049485016614198685,
      "logps/chosen": -130.2781219482422,
      "logps/rejected": -137.9343719482422,
      "logps/weighted_chosen": -1.1705901622772217,
      "logps/weighted_rejected": -0.7656036615371704,
      "loss": 0.7402,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -17.155858993530273,
      "rewards/margins": 2.863085985183716,
      "rewards/rejected": -20.016016006469727,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.07308082282543182,
      "rewards/weighted_margins": 0.04105224460363388,
      "rewards/weighted_rejected": -0.114354707300663,
      "step": 370
    },
    {
      "epoch": 0.2030863785155989,
      "grad_norm": 26.81167221069336,
      "learning_rate": 9.685932900101146e-07,
      "logits/chosen": -0.07604064792394638,
      "logits/rejected": -0.06302948296070099,
      "logps/chosen": -134.7859344482422,
      "logps/rejected": -137.11093139648438,
      "logps/weighted_chosen": -0.98638916015625,
      "logps/weighted_rejected": -0.6243560910224915,
      "loss": 0.7116,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.729297637939453,
      "rewards/margins": 1.8025391101837158,
      "rewards/rejected": -19.528514862060547,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.11945419013500214,
      "rewards/weighted_margins": 0.07185058295726776,
      "rewards/weighted_rejected": -0.1911766082048416,
      "step": 380
    },
    {
      "epoch": 0.20843075689758836,
      "grad_norm": 40.89613342285156,
      "learning_rate": 9.652581474390043e-07,
      "logits/chosen": -0.09274902194738388,
      "logits/rejected": -0.05452308803796768,
      "logps/chosen": -139.3249969482422,
      "logps/rejected": -141.53750610351562,
      "logps/weighted_chosen": -0.9199386835098267,
      "logps/weighted_rejected": -0.9217513799667358,
      "loss": 0.7323,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -19.379688262939453,
      "rewards/margins": 2.321093797683716,
      "rewards/rejected": -21.702733993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.05511016771197319,
      "rewards/weighted_margins": 0.04057302325963974,
      "rewards/weighted_rejected": -0.09564266353845596,
      "step": 390
    },
    {
      "epoch": 0.21377513527957778,
      "grad_norm": 51.860347747802734,
      "learning_rate": 9.61761086184981e-07,
      "logits/chosen": -0.10268402099609375,
      "logits/rejected": -0.05923309177160263,
      "logps/chosen": -139.50936889648438,
      "logps/rejected": -141.4015655517578,
      "logps/weighted_chosen": -0.9449096918106079,
      "logps/weighted_rejected": -1.177301049232483,
      "loss": 0.7387,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -20.024608612060547,
      "rewards/margins": 2.9205079078674316,
      "rewards/rejected": -22.942577362060547,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.26060789823532104,
      "rewards/weighted_margins": 0.017241667956113815,
      "rewards/weighted_rejected": -0.2779647707939148,
      "step": 400
    },
    {
      "epoch": 0.21911951366156723,
      "grad_norm": 7.539150238037109,
      "learning_rate": 9.581033232918629e-07,
      "logits/chosen": -0.015906523913145065,
      "logits/rejected": 0.02000865899026394,
      "logps/chosen": -138.39999389648438,
      "logps/rejected": -141.68905639648438,
      "logps/weighted_chosen": -1.0313812494277954,
      "logps/weighted_rejected": -1.1196472644805908,
      "loss": 0.7283,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -21.702342987060547,
      "rewards/margins": 2.826953172683716,
      "rewards/rejected": -24.53125,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": -0.2783859372138977,
      "rewards/weighted_margins": 0.06903991848230362,
      "rewards/weighted_rejected": -0.3477470278739929,
      "step": 410
    },
    {
      "epoch": 0.22446389204355668,
      "grad_norm": 6.638446807861328,
      "learning_rate": 9.542861317306952e-07,
      "logits/chosen": -0.05767669528722763,
      "logits/rejected": -0.03759918361902237,
      "logps/chosen": -140.64688110351562,
      "logps/rejected": -141.9718780517578,
      "logps/weighted_chosen": -1.5940430164337158,
      "logps/weighted_rejected": -1.458154320716858,
      "loss": 0.7371,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -22.33203125,
      "rewards/margins": 1.1359374523162842,
      "rewards/rejected": -23.46484375,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.299835205078125,
      "rewards/weighted_margins": 0.032170869410037994,
      "rewards/weighted_rejected": -0.3319748044013977,
      "step": 420
    },
    {
      "epoch": 0.22980827042554614,
      "grad_norm": 24.621816635131836,
      "learning_rate": 9.503108399567308e-07,
      "logits/chosen": -0.15150222182273865,
      "logits/rejected": -0.09339599311351776,
      "logps/chosen": -149.8671875,
      "logps/rejected": -153.6296844482422,
      "logps/weighted_chosen": -0.7455719113349915,
      "logps/weighted_rejected": -0.826556384563446,
      "loss": 0.7125,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -22.685546875,
      "rewards/margins": 1.7355468273162842,
      "rewards/rejected": -24.420312881469727,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.19403991103172302,
      "rewards/weighted_margins": 0.02692871168255806,
      "rewards/weighted_rejected": -0.2209148406982422,
      "step": 430
    },
    {
      "epoch": 0.23515264880753559,
      "grad_norm": 10.726137161254883,
      "learning_rate": 9.461788314471034e-07,
      "logits/chosen": -0.08421440422534943,
      "logits/rejected": -0.031351469457149506,
      "logps/chosen": -143.1171875,
      "logps/rejected": -146.890625,
      "logps/weighted_chosen": -0.7524811029434204,
      "logps/weighted_rejected": -0.8818389773368835,
      "loss": 0.7062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -24.516796112060547,
      "rewards/margins": 2.0640625953674316,
      "rewards/rejected": -26.577733993530273,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.22655639052391052,
      "rewards/weighted_margins": 0.07337188720703125,
      "rewards/weighted_rejected": -0.30005186796188354,
      "step": 440
    },
    {
      "epoch": 0.240497027189525,
      "grad_norm": 8.586684226989746,
      "learning_rate": 9.418915442193509e-07,
      "logits/chosen": -0.03446502611041069,
      "logits/rejected": 0.007968139834702015,
      "logps/chosen": -144.7843780517578,
      "logps/rejected": -155.96249389648438,
      "logps/weighted_chosen": -1.1978485584259033,
      "logps/weighted_rejected": -1.2447845935821533,
      "loss": 0.6841,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -26.516016006469727,
      "rewards/margins": 4.714453220367432,
      "rewards/rejected": -31.236328125,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.25520628690719604,
      "rewards/weighted_margins": 0.09325332939624786,
      "rewards/weighted_rejected": -0.3483436703681946,
      "step": 450
    },
    {
      "epoch": 0.24584140557151446,
      "grad_norm": 10.014894485473633,
      "learning_rate": 9.374504703309579e-07,
      "logits/chosen": -0.1379852294921875,
      "logits/rejected": -0.0861412063241005,
      "logps/chosen": -141.83828735351562,
      "logps/rejected": -143.34530639648438,
      "logps/weighted_chosen": -1.1420501470565796,
      "logps/weighted_rejected": -1.055877685546875,
      "loss": 0.6804,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -25.53515625,
      "rewards/margins": 2.038281202316284,
      "rewards/rejected": -27.571874618530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.21983489394187927,
      "rewards/weighted_margins": 0.17051848769187927,
      "rewards/weighted_rejected": -0.3904357850551605,
      "step": 460
    },
    {
      "epoch": 0.2511857839535039,
      "grad_norm": 10.024358749389648,
      "learning_rate": 9.328571553600915e-07,
      "logits/chosen": -0.1318313628435135,
      "logits/rejected": -0.0828731507062912,
      "logps/chosen": -144.39688110351562,
      "logps/rejected": -147.41796875,
      "logps/weighted_chosen": -0.9446808099746704,
      "logps/weighted_rejected": -1.138859510421753,
      "loss": 0.7137,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -25.697265625,
      "rewards/margins": 1.9656250476837158,
      "rewards/rejected": -27.665233612060547,
      "rewards/weighted_accuracies": 0.4906249940395355,
      "rewards/weighted_chosen": -0.2078697234392166,
      "rewards/weighted_margins": 0.052256010472774506,
      "rewards/weighted_rejected": -0.260000616312027,
      "step": 470
    },
    {
      "epoch": 0.25653016233549336,
      "grad_norm": 13.618422508239746,
      "learning_rate": 9.281131978677106e-07,
      "logits/chosen": -0.16686096787452698,
      "logits/rejected": -0.1246364563703537,
      "logps/chosen": -149.2546844482422,
      "logps/rejected": -152.47811889648438,
      "logps/weighted_chosen": -0.686419665813446,
      "logps/weighted_rejected": -1.0338714122772217,
      "loss": 0.6805,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -25.559375762939453,
      "rewards/margins": 3.1332030296325684,
      "rewards/rejected": -28.689844131469727,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.22538375854492188,
      "rewards/weighted_margins": 0.13199158012866974,
      "rewards/weighted_rejected": -0.35721856355667114,
      "step": 480
    },
    {
      "epoch": 0.2618745407174828,
      "grad_norm": 30.852863311767578,
      "learning_rate": 9.232202488412361e-07,
      "logits/chosen": -0.10437698662281036,
      "logits/rejected": -0.05600614473223686,
      "logps/chosen": -143.5671844482422,
      "logps/rejected": -149.11874389648438,
      "logps/weighted_chosen": -1.236029028892517,
      "logps/weighted_rejected": -1.6542327404022217,
      "loss": 0.6937,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -28.171483993530273,
      "rewards/margins": 2.311328172683716,
      "rewards/rejected": -30.477733612060547,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.27268296480178833,
      "rewards/weighted_margins": 0.08454742282629013,
      "rewards/weighted_rejected": -0.35735321044921875,
      "step": 490
    },
    {
      "epoch": 0.26721891909947226,
      "grad_norm": 9.151970863342285,
      "learning_rate": 9.181800111199766e-07,
      "logits/chosen": -0.1230342835187912,
      "logits/rejected": -0.0801507979631424,
      "logps/chosen": -147.54843139648438,
      "logps/rejected": -147.5437469482422,
      "logps/weighted_chosen": -1.5909912586212158,
      "logps/weighted_rejected": -1.286199927330017,
      "loss": 0.7005,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -29.640233993530273,
      "rewards/margins": 3.1507811546325684,
      "rewards/rejected": -32.789451599121094,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.34203338623046875,
      "rewards/weighted_margins": 0.12214355170726776,
      "rewards/weighted_rejected": -0.4641265869140625,
      "step": 500
    },
    {
      "epoch": 0.26721891909947226,
      "eval_logits/chosen": -0.29476672410964966,
      "eval_logits/rejected": -0.26891082525253296,
      "eval_logps/chosen": -149.5992889404297,
      "eval_logps/rejected": -153.5249481201172,
      "eval_logps/weighted_chosen": -0.8761399984359741,
      "eval_logps/weighted_rejected": -1.120854377746582,
      "eval_loss": 0.6999610066413879,
      "eval_rewards/accuracies": 0.5784114003181458,
      "eval_rewards/chosen": -28.267057418823242,
      "eval_rewards/margins": 2.7842414379119873,
      "eval_rewards/rejected": -31.05422592163086,
      "eval_rewards/weighted_accuracies": 0.5264765620231628,
      "eval_rewards/weighted_chosen": -0.23699289560317993,
      "eval_rewards/weighted_margins": 0.08454981446266174,
      "eval_rewards/weighted_rejected": -0.3215427100658417,
      "eval_runtime": 1248.3443,
      "eval_samples_per_second": 1.571,
      "eval_steps_per_second": 0.393,
      "step": 500
    },
    {
      "epoch": 0.2725632974814617,
      "grad_norm": 7.777375221252441,
      "learning_rate": 9.129942388025066e-07,
      "logits/chosen": -0.1614234894514084,
      "logits/rejected": -0.12539061903953552,
      "logps/chosen": -138.70468139648438,
      "logps/rejected": -144.50936889648438,
      "logps/weighted_chosen": -1.0583159923553467,
      "logps/weighted_rejected": -1.0353851318359375,
      "loss": 0.7783,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -27.291797637939453,
      "rewards/margins": 2.485156297683716,
      "rewards/rejected": -29.778514862060547,
      "rewards/weighted_accuracies": 0.5093749761581421,
      "rewards/weighted_chosen": -0.21576309204101562,
      "rewards/weighted_margins": 0.003093719482421875,
      "rewards/weighted_rejected": -0.219013974070549,
      "step": 510
    },
    {
      "epoch": 0.2779076758634511,
      "grad_norm": 42.880157470703125,
      "learning_rate": 9.076647366362082e-07,
      "logits/chosen": -0.0863800048828125,
      "logits/rejected": -0.04528322070837021,
      "logps/chosen": -148.21875,
      "logps/rejected": -155.3046875,
      "logps/weighted_chosen": -1.141107201576233,
      "logps/weighted_rejected": -1.2995758056640625,
      "loss": 0.7748,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -33.478515625,
      "rewards/margins": 4.057812690734863,
      "rewards/rejected": -37.53515625,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.2628006041049957,
      "rewards/weighted_margins": -0.00189971923828125,
      "rewards/weighted_rejected": -0.26109999418258667,
      "step": 520
    },
    {
      "epoch": 0.2832520542454406,
      "grad_norm": 24.728120803833008,
      "learning_rate": 9.021933593891841e-07,
      "logits/chosen": -0.020507430657744408,
      "logits/rejected": 0.01460342388600111,
      "logps/chosen": -154.15780639648438,
      "logps/rejected": -156.328125,
      "logps/weighted_chosen": -1.0978972911834717,
      "logps/weighted_rejected": -0.9736785888671875,
      "loss": 0.6998,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -40.849220275878906,
      "rewards/margins": 2.678515672683716,
      "rewards/rejected": -43.53046798706055,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.2724204957485199,
      "rewards/weighted_margins": 0.09940185397863388,
      "rewards/weighted_rejected": -0.3718253970146179,
      "step": 530
    },
    {
      "epoch": 0.28859643262743,
      "grad_norm": 11.090974807739258,
      "learning_rate": 8.965820112047629e-07,
      "logits/chosen": -0.07119598239660263,
      "logits/rejected": -0.04578246921300888,
      "logps/chosen": -163.8781280517578,
      "logps/rejected": -172.9187469482422,
      "logps/weighted_chosen": -0.9050750732421875,
      "logps/weighted_rejected": -1.05401611328125,
      "loss": 0.6621,
      "rewards/accuracies": 0.559374988079071,
      "rewards/chosen": -42.501564025878906,
      "rewards/margins": 4.053515434265137,
      "rewards/rejected": -46.54999923706055,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.1999252289533615,
      "rewards/weighted_margins": 0.1526111662387848,
      "rewards/weighted_rejected": -0.3521057069301605,
      "step": 540
    },
    {
      "epoch": 0.2939408110094195,
      "grad_norm": 13.709502220153809,
      "learning_rate": 8.908326449388213e-07,
      "logits/chosen": -0.07627105712890625,
      "logits/rejected": -0.03014526329934597,
      "logps/chosen": -156.328125,
      "logps/rejected": -161.06405639648438,
      "logps/weighted_chosen": -1.228173851966858,
      "logps/weighted_rejected": -1.3901275396347046,
      "loss": 0.7368,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -41.07109451293945,
      "rewards/margins": 4.337500095367432,
      "rewards/rejected": -45.392967224121094,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.172210693359375,
      "rewards/weighted_margins": 0.07322921603918076,
      "rewards/weighted_rejected": -0.2451316863298416,
      "step": 550
    },
    {
      "epoch": 0.2992851893914089,
      "grad_norm": 10.929818153381348,
      "learning_rate": 8.849472614801527e-07,
      "logits/chosen": -0.07251434028148651,
      "logits/rejected": -0.02443542517721653,
      "logps/chosen": -161.86874389648438,
      "logps/rejected": -165.0749969482422,
      "logps/weighted_chosen": -0.752777099609375,
      "logps/weighted_rejected": -1.0892517566680908,
      "loss": 0.6642,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.9609375,
      "rewards/margins": 3.3421874046325684,
      "rewards/rejected": -48.30546951293945,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.1927940398454666,
      "rewards/weighted_margins": 0.19287796318531036,
      "rewards/weighted_rejected": -0.38528138399124146,
      "step": 560
    },
    {
      "epoch": 0.30462956777339834,
      "grad_norm": 11.730274200439453,
      "learning_rate": 8.789279090541208e-07,
      "logits/chosen": -0.05120544508099556,
      "logits/rejected": 0.01478500384837389,
      "logps/chosen": -161.8796844482422,
      "logps/rejected": -172.51406860351562,
      "logps/weighted_chosen": -0.8590911626815796,
      "logps/weighted_rejected": -1.1839568614959717,
      "loss": 0.6821,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.99609375,
      "rewards/margins": 4.257031440734863,
      "rewards/rejected": -48.24609375,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.29502803087234497,
      "rewards/weighted_margins": 0.16868743300437927,
      "rewards/weighted_rejected": -0.4639228880405426,
      "step": 570
    },
    {
      "epoch": 0.3099739461553878,
      "grad_norm": 16.967859268188477,
      "learning_rate": 8.72776682509837e-07,
      "logits/chosen": 0.011944961734116077,
      "logits/rejected": 0.08212585747241974,
      "logps/chosen": -157.47500610351562,
      "logps/rejected": -163.43594360351562,
      "logps/weighted_chosen": -0.8815460205078125,
      "logps/weighted_rejected": -1.220306396484375,
      "loss": 0.6705,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.369529724121094,
      "rewards/margins": 4.614453315734863,
      "rewards/rejected": -47.984375,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.3169898986816406,
      "rewards/weighted_margins": 0.19199447333812714,
      "rewards/weighted_rejected": -0.5094040036201477,
      "step": 580
    },
    {
      "epoch": 0.31531832453737724,
      "grad_norm": 9.289381980895996,
      "learning_rate": 8.664957225911138e-07,
      "logits/chosen": -0.04825134202837944,
      "logits/rejected": -0.01284942589700222,
      "logps/chosen": -163.6437530517578,
      "logps/rejected": -184.9796905517578,
      "logps/weighted_chosen": -0.775604248046875,
      "logps/weighted_rejected": -1.1336090564727783,
      "loss": 0.6889,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -42.566795349121094,
      "rewards/margins": 3.8046875,
      "rewards/rejected": -46.36328125,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.30945053696632385,
      "rewards/weighted_margins": 0.116083525121212,
      "rewards/weighted_rejected": -0.4253120422363281,
      "step": 590
    },
    {
      "epoch": 0.3206627029193667,
      "grad_norm": 7.966893672943115,
      "learning_rate": 8.600872151914451e-07,
      "logits/chosen": 0.06866455078125,
      "logits/rejected": 0.09470061957836151,
      "logps/chosen": -158.2546844482422,
      "logps/rejected": -167.48281860351562,
      "logps/weighted_chosen": -1.3719971179962158,
      "logps/weighted_rejected": -1.7499603033065796,
      "loss": 0.7417,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -44.11249923706055,
      "rewards/margins": 3.3988280296325684,
      "rewards/rejected": -47.50468826293945,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.21634292602539062,
      "rewards/weighted_margins": 0.19443893432617188,
      "rewards/weighted_rejected": -0.41086500883102417,
      "step": 600
    },
    {
      "epoch": 0.32600708130135614,
      "grad_norm": 49.298519134521484,
      "learning_rate": 8.535533905932737e-07,
      "logits/chosen": 0.06249542161822319,
      "logits/rejected": 0.12158355861902237,
      "logps/chosen": -173.4250030517578,
      "logps/rejected": -183.015625,
      "logps/weighted_chosen": -0.9283081293106079,
      "logps/weighted_rejected": -1.2264496088027954,
      "loss": 0.6778,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -48.342185974121094,
      "rewards/margins": 5.698437690734863,
      "rewards/rejected": -54.049217224121094,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.3028602600097656,
      "rewards/weighted_margins": 0.1688835173845291,
      "rewards/weighted_rejected": -0.4717727601528168,
      "step": 610
    },
    {
      "epoch": 0.33135145968334556,
      "grad_norm": 19.377832412719727,
      "learning_rate": 8.468965226918105e-07,
      "logits/chosen": 0.037828825414180756,
      "logits/rejected": 0.0580875389277935,
      "logps/chosen": -170.1281280517578,
      "logps/rejected": -178.9093780517578,
      "logps/weighted_chosen": -1.4841537475585938,
      "logps/weighted_rejected": -1.118981957435608,
      "loss": 0.7542,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -46.46210861206055,
      "rewards/margins": 3.794921875,
      "rewards/rejected": -50.26640701293945,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.40849533677101135,
      "rewards/weighted_margins": 0.0321044921875,
      "rewards/weighted_rejected": -0.4405921995639801,
      "step": 620
    },
    {
      "epoch": 0.33669583806533504,
      "grad_norm": 10.768741607666016,
      "learning_rate": 8.40118928203676e-07,
      "logits/chosen": 0.05730438232421875,
      "logits/rejected": 0.07560119777917862,
      "logps/chosen": -152.3718719482422,
      "logps/rejected": -164.1218719482422,
      "logps/weighted_chosen": -1.4566253423690796,
      "logps/weighted_rejected": -1.3677704334259033,
      "loss": 0.7025,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.721092224121094,
      "rewards/margins": 4.366015434265137,
      "rewards/rejected": -48.07500076293945,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.30116957426071167,
      "rewards/weighted_margins": 0.09331168979406357,
      "rewards/weighted_rejected": -0.39441680908203125,
      "step": 630
    },
    {
      "epoch": 0.34204021644732446,
      "grad_norm": 8.60303020477295,
      "learning_rate": 8.332229658606382e-07,
      "logits/chosen": -0.055472563952207565,
      "logits/rejected": -0.03988037258386612,
      "logps/chosen": -175.6671905517578,
      "logps/rejected": -180.06875610351562,
      "logps/weighted_chosen": -0.8608856201171875,
      "logps/weighted_rejected": -0.986541748046875,
      "loss": 0.7297,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -44.84062576293945,
      "rewards/margins": 4.931250095367432,
      "rewards/rejected": -49.77031326293945,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.23048630356788635,
      "rewards/weighted_margins": 0.10724487155675888,
      "rewards/weighted_rejected": -0.33786240220069885,
      "step": 640
    },
    {
      "epoch": 0.34738459482931394,
      "grad_norm": 26.148893356323242,
      "learning_rate": 8.262110355887302e-07,
      "logits/chosen": -0.055730439722537994,
      "logits/rejected": -0.00980529747903347,
      "logps/chosen": -166.38125610351562,
      "logps/rejected": -169.40469360351562,
      "logps/weighted_chosen": -0.7286010980606079,
      "logps/weighted_rejected": -0.8990936279296875,
      "loss": 0.7103,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -46.994529724121094,
      "rewards/margins": 3.8031249046325684,
      "rewards/rejected": -50.78515625,
      "rewards/weighted_accuracies": 0.518750011920929,
      "rewards/weighted_chosen": -0.2977249026298523,
      "rewards/weighted_margins": 0.08674392849206924,
      "rewards/weighted_rejected": -0.3846077024936676,
      "step": 650
    },
    {
      "epoch": 0.35272897321130336,
      "grad_norm": 11.823346138000488,
      "learning_rate": 8.190855776730293e-07,
      "logits/chosen": -0.07448883354663849,
      "logits/rejected": -0.014659118838608265,
      "logps/chosen": -171.80313110351562,
      "logps/rejected": -182.2687530517578,
      "logps/weighted_chosen": -0.9873107671737671,
      "logps/weighted_rejected": -1.4951263666152954,
      "loss": 0.7347,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -50.564064025878906,
      "rewards/margins": 4.532812595367432,
      "rewards/rejected": -55.109375,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.28445130586624146,
      "rewards/weighted_margins": 0.06389693915843964,
      "rewards/weighted_rejected": -0.34831467270851135,
      "step": 660
    },
    {
      "epoch": 0.3580733515932928,
      "grad_norm": 13.33145523071289,
      "learning_rate": 8.118490719083917e-07,
      "logits/chosen": -0.10270843654870987,
      "logits/rejected": -0.062209319323301315,
      "logps/chosen": -191.3000030517578,
      "logps/rejected": -193.4109344482422,
      "logps/weighted_chosen": -1.0586059093475342,
      "logps/weighted_rejected": -1.3695876598358154,
      "loss": 0.6575,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -58.06640625,
      "rewards/margins": 4.108984470367432,
      "rewards/rejected": -62.18359375,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.2945011258125305,
      "rewards/weighted_margins": 0.18829631805419922,
      "rewards/weighted_rejected": -0.4826883375644684,
      "step": 670
    },
    {
      "epoch": 0.36341772997528227,
      "grad_norm": 23.127918243408203,
      "learning_rate": 8.045040367364368e-07,
      "logits/chosen": 0.03829193115234375,
      "logits/rejected": 0.055706024169921875,
      "logps/chosen": -166.234375,
      "logps/rejected": -169.96249389648438,
      "logps/weighted_chosen": -1.025201439857483,
      "logps/weighted_rejected": -1.302404761314392,
      "loss": 0.7166,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -56.759376525878906,
      "rewards/margins": 4.711718559265137,
      "rewards/rejected": -61.46796798706055,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.4052978456020355,
      "rewards/weighted_margins": 0.09704723209142685,
      "rewards/weighted_rejected": -0.5025444030761719,
      "step": 680
    },
    {
      "epoch": 0.3687621083572717,
      "grad_norm": 7.137462615966797,
      "learning_rate": 7.970530283690819e-07,
      "logits/chosen": 0.04682769626379013,
      "logits/rejected": 0.066925048828125,
      "logps/chosen": -172.50155639648438,
      "logps/rejected": -180.54061889648438,
      "logps/weighted_chosen": -1.301055908203125,
      "logps/weighted_rejected": -1.128961205482483,
      "loss": 0.6864,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -58.544532775878906,
      "rewards/margins": 5.397265434265137,
      "rewards/rejected": -63.9453125,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.3540634214878082,
      "rewards/weighted_margins": 0.1536247283220291,
      "rewards/weighted_rejected": -0.5076873898506165,
      "step": 690
    },
    {
      "epoch": 0.3741064867392611,
      "grad_norm": 7.699697971343994,
      "learning_rate": 7.89498639898931e-07,
      "logits/chosen": 0.05636443942785263,
      "logits/rejected": 0.09223861992359161,
      "logps/chosen": -169.8562469482422,
      "logps/rejected": -173.65625,
      "logps/weighted_chosen": -0.9024292230606079,
      "logps/weighted_rejected": -1.1884033679962158,
      "loss": 0.6642,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -59.00078201293945,
      "rewards/margins": 4.887109279632568,
      "rewards/rejected": -63.889060974121094,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.3017433285713196,
      "rewards/weighted_margins": 0.20005035400390625,
      "rewards/weighted_rejected": -0.5016593933105469,
      "step": 700
    },
    {
      "epoch": 0.3794508651212506,
      "grad_norm": 18.41610336303711,
      "learning_rate": 7.818435003968305e-07,
      "logits/chosen": -0.03687286376953125,
      "logits/rejected": -0.00025310515775345266,
      "logps/chosen": -181.0812530517578,
      "logps/rejected": -189.1828155517578,
      "logps/weighted_chosen": -1.0574157238006592,
      "logps/weighted_rejected": -1.2661011219024658,
      "loss": 0.6918,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -58.548439025878906,
      "rewards/margins": 5.687890529632568,
      "rewards/rejected": -64.234375,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.33240431547164917,
      "rewards/weighted_margins": 0.08642272651195526,
      "rewards/weighted_rejected": -0.4188430905342102,
      "step": 710
    },
    {
      "epoch": 0.38479524350324,
      "grad_norm": 26.664974212646484,
      "learning_rate": 7.740902739969008e-07,
      "logits/chosen": -0.08480529487133026,
      "logits/rejected": -0.07062835991382599,
      "logps/chosen": -171.16561889648438,
      "logps/rejected": -175.68905639648438,
      "logps/weighted_chosen": -0.828997790813446,
      "logps/weighted_rejected": -0.9821502566337585,
      "loss": 0.6819,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -49.982810974121094,
      "rewards/margins": 4.688672065734863,
      "rewards/rejected": -54.672264099121094,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.18975982069969177,
      "rewards/weighted_margins": 0.1927967071533203,
      "rewards/weighted_rejected": -0.38214111328125,
      "step": 720
    },
    {
      "epoch": 0.3901396218852295,
      "grad_norm": 8.73801040649414,
      "learning_rate": 7.662416589693695e-07,
      "logits/chosen": -0.00596694927662611,
      "logits/rejected": 0.019218826666474342,
      "logps/chosen": -161.8249969482422,
      "logps/rejected": -166.8312530517578,
      "logps/weighted_chosen": -1.3982207775115967,
      "logps/weighted_rejected": -1.7395598888397217,
      "loss": 0.7772,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -46.8515625,
      "rewards/margins": 5.096484184265137,
      "rewards/rejected": -51.95000076293945,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.23512879014015198,
      "rewards/weighted_margins": 0.10187530517578125,
      "rewards/weighted_rejected": -0.3369201719760895,
      "step": 730
    },
    {
      "epoch": 0.3954840002672189,
      "grad_norm": 19.624563217163086,
      "learning_rate": 7.583003867815192e-07,
      "logits/chosen": -0.0639801025390625,
      "logits/rejected": -0.05755653232336044,
      "logps/chosen": -189.7468719482422,
      "logps/rejected": -196.7062530517578,
      "logps/weighted_chosen": -1.1809478998184204,
      "logps/weighted_rejected": -1.3041900396347046,
      "loss": 0.698,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -58.209373474121094,
      "rewards/margins": 4.971875190734863,
      "rewards/rejected": -63.1640625,
      "rewards/weighted_accuracies": 0.518750011920929,
      "rewards/weighted_chosen": -0.39410895109176636,
      "rewards/weighted_margins": 0.08784560859203339,
      "rewards/weighted_rejected": -0.4818374514579773,
      "step": 740
    },
    {
      "epoch": 0.40082837864920834,
      "grad_norm": 19.04839324951172,
      "learning_rate": 7.502692211470869e-07,
      "logits/chosen": -0.05941009521484375,
      "logits/rejected": -0.02816619910299778,
      "logps/chosen": -182.8984375,
      "logps/rejected": -190.1203155517578,
      "logps/weighted_chosen": -0.853833019733429,
      "logps/weighted_rejected": -1.1572723388671875,
      "loss": 0.7079,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -60.310157775878906,
      "rewards/margins": 6.424609184265137,
      "rewards/rejected": -66.73905944824219,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.4228218197822571,
      "rewards/weighted_margins": 0.10426559299230576,
      "rewards/weighted_rejected": -0.5270500183105469,
      "step": 750
    },
    {
      "epoch": 0.4061727570311978,
      "grad_norm": 12.947752952575684,
      "learning_rate": 7.421509570644387e-07,
      "logits/chosen": -0.05351562425494194,
      "logits/rejected": -0.0015235900646075606,
      "logps/chosen": -180.66250610351562,
      "logps/rejected": -185.6828155517578,
      "logps/weighted_chosen": -1.2698882818222046,
      "logps/weighted_rejected": -1.5226471424102783,
      "loss": 0.6831,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -62.2109375,
      "rewards/margins": 5.124218940734863,
      "rewards/rejected": -67.33280944824219,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.32732313871383667,
      "rewards/weighted_margins": 0.18027648329734802,
      "rewards/weighted_rejected": -0.5074516534805298,
      "step": 760
    },
    {
      "epoch": 0.41151713541318724,
      "grad_norm": 10.666881561279297,
      "learning_rate": 7.339484198438566e-07,
      "logits/chosen": 0.00444793701171875,
      "logits/rejected": 0.03400421142578125,
      "logps/chosen": -165.93124389648438,
      "logps/rejected": -172.2937469482422,
      "logps/weighted_chosen": -1.7782714366912842,
      "logps/weighted_rejected": -1.320520043373108,
      "loss": 0.6969,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -57.14374923706055,
      "rewards/margins": 7.307421684265137,
      "rewards/rejected": -64.4468765258789,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.3784065246582031,
      "rewards/weighted_margins": 0.17845840752124786,
      "rewards/weighted_rejected": -0.557085394859314,
      "step": 770
    },
    {
      "epoch": 0.4168615137951767,
      "grad_norm": 11.188801765441895,
      "learning_rate": 7.256644641242773e-07,
      "logits/chosen": -0.03662109375,
      "logits/rejected": 0.02340393140912056,
      "logps/chosen": -170.4765625,
      "logps/rejected": -175.390625,
      "logps/weighted_chosen": -1.057519555091858,
      "logps/weighted_rejected": -1.3154876232147217,
      "loss": 0.7128,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -56.12578201293945,
      "rewards/margins": 3.3226561546325684,
      "rewards/rejected": -59.43828201293945,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.37098997831344604,
      "rewards/weighted_margins": 0.07502517849206924,
      "rewards/weighted_rejected": -0.4461258053779602,
      "step": 780
    },
    {
      "epoch": 0.42220589217716614,
      "grad_norm": 9.07581615447998,
      "learning_rate": 7.173019728798234e-07,
      "logits/chosen": -0.03949751704931259,
      "logits/rejected": -0.02464141882956028,
      "logps/chosen": -169.55313110351562,
      "logps/rejected": -177.18124389648438,
      "logps/weighted_chosen": -0.8541290163993835,
      "logps/weighted_rejected": -1.1333099603652954,
      "loss": 0.6788,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -55.47734451293945,
      "rewards/margins": 6.255859375,
      "rewards/rejected": -61.71406173706055,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.307028204202652,
      "rewards/weighted_margins": 0.12036208808422089,
      "rewards/weighted_rejected": -0.4274864196777344,
      "step": 790
    },
    {
      "epoch": 0.42755027055915557,
      "grad_norm": 19.591575622558594,
      "learning_rate": 7.088638564164738e-07,
      "logits/chosen": -0.08669586479663849,
      "logits/rejected": -0.05663032457232475,
      "logps/chosen": -185.1593780517578,
      "logps/rejected": -195.95938110351562,
      "logps/weighted_chosen": -1.067468285560608,
      "logps/weighted_rejected": -1.238824486732483,
      "loss": 0.693,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -63.07421875,
      "rewards/margins": 6.429296970367432,
      "rewards/rejected": -69.49296569824219,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.3178604245185852,
      "rewards/weighted_margins": 0.1114906296133995,
      "rewards/weighted_rejected": -0.4294891357421875,
      "step": 800
    },
    {
      "epoch": 0.43289464894114504,
      "grad_norm": 13.074841499328613,
      "learning_rate": 7.003530513592215e-07,
      "logits/chosen": -0.05083770677447319,
      "logits/rejected": -0.0205841064453125,
      "logps/chosen": -177.95156860351562,
      "logps/rejected": -189.63436889648438,
      "logps/weighted_chosen": -1.188201904296875,
      "logps/weighted_rejected": -1.1756408214569092,
      "loss": 0.6926,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -62.192970275878906,
      "rewards/margins": 7.530859470367432,
      "rewards/rejected": -69.73905944824219,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.3717803955078125,
      "rewards/weighted_margins": 0.12430572509765625,
      "rewards/weighted_rejected": -0.49610137939453125,
      "step": 810
    },
    {
      "epoch": 0.43823902732313447,
      "grad_norm": 51.0267448425293,
      "learning_rate": 6.917725196300726e-07,
      "logits/chosen": -0.1438644379377365,
      "logits/rejected": -0.15563049912452698,
      "logps/chosen": -191.125,
      "logps/rejected": -203.52188110351562,
      "logps/weighted_chosen": -1.097863793373108,
      "logps/weighted_rejected": -0.971697986125946,
      "loss": 0.6974,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -63.764060974121094,
      "rewards/margins": 7.415234565734863,
      "rewards/rejected": -71.171875,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.3276786804199219,
      "rewards/weighted_margins": 0.08215713500976562,
      "rewards/weighted_rejected": -0.4100082516670227,
      "step": 820
    },
    {
      "epoch": 0.44358340570512395,
      "grad_norm": 20.933467864990234,
      "learning_rate": 6.831252474172411e-07,
      "logits/chosen": -0.05628051608800888,
      "logits/rejected": -0.03361053392291069,
      "logps/chosen": -176.0031280517578,
      "logps/rejected": -186.4343719482422,
      "logps/weighted_chosen": -1.6239410638809204,
      "logps/weighted_rejected": -1.3564453125,
      "loss": 0.6817,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -60.45234298706055,
      "rewards/margins": 6.214453220367432,
      "rewards/rejected": -66.66015625,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.2794052064418793,
      "rewards/weighted_margins": 0.22185592353343964,
      "rewards/weighted_rejected": -0.500841498374939,
      "step": 830
    },
    {
      "epoch": 0.44892778408711337,
      "grad_norm": 11.393433570861816,
      "learning_rate": 6.74414244135898e-07,
      "logits/chosen": -0.15757599472999573,
      "logits/rejected": -0.116943359375,
      "logps/chosen": -177.0984344482422,
      "logps/rejected": -185.13436889648438,
      "logps/weighted_chosen": -0.666271984577179,
      "logps/weighted_rejected": -0.7827666997909546,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -60.349998474121094,
      "rewards/margins": 6.200390815734863,
      "rewards/rejected": -66.5289077758789,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.33733367919921875,
      "rewards/weighted_margins": 0.08320007473230362,
      "rewards/weighted_rejected": -0.42046278715133667,
      "step": 840
    },
    {
      "epoch": 0.4542721624691028,
      "grad_norm": 9.733312606811523,
      "learning_rate": 6.656425413808388e-07,
      "logits/chosen": -0.12353897094726562,
      "logits/rejected": -0.08713607490062714,
      "logps/chosen": -179.66561889648438,
      "logps/rejected": -183.109375,
      "logps/weighted_chosen": -1.0826873779296875,
      "logps/weighted_rejected": -1.106787085533142,
      "loss": 0.6875,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -60.806251525878906,
      "rewards/margins": 2.733203172683716,
      "rewards/rejected": -63.51250076293945,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.22345809638500214,
      "rewards/weighted_margins": 0.1886741667985916,
      "rewards/weighted_rejected": -0.4119575619697571,
      "step": 850
    },
    {
      "epoch": 0.45961654085109227,
      "grad_norm": 9.04237174987793,
      "learning_rate": 6.568131918714294e-07,
      "logits/chosen": -0.09643097221851349,
      "logits/rejected": -0.08455581963062286,
      "logps/chosen": -178.4453125,
      "logps/rejected": -187.99063110351562,
      "logps/weighted_chosen": -0.840087890625,
      "logps/weighted_rejected": -1.158935546875,
      "loss": 0.7138,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -61.162498474121094,
      "rewards/margins": 5.386328220367432,
      "rewards/rejected": -66.54765319824219,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.2982444763183594,
      "rewards/weighted_margins": 0.08280716091394424,
      "rewards/weighted_rejected": -0.38090819120407104,
      "step": 860
    },
    {
      "epoch": 0.4649609192330817,
      "grad_norm": 7.623125076293945,
      "learning_rate": 6.47929268389203e-07,
      "logits/chosen": -0.06428680568933487,
      "logits/rejected": -0.04297828674316406,
      "logps/chosen": -190.86093139648438,
      "logps/rejected": -205.9656219482422,
      "logps/weighted_chosen": -0.9338012933731079,
      "logps/weighted_rejected": -0.741748034954071,
      "loss": 0.6808,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -65.76640319824219,
      "rewards/margins": 7.994921684265137,
      "rewards/rejected": -73.75703430175781,
      "rewards/weighted_accuracies": 0.5,
      "rewards/weighted_chosen": -0.28187790513038635,
      "rewards/weighted_margins": 0.11656570434570312,
      "rewards/weighted_rejected": -0.39813536405563354,
      "step": 870
    },
    {
      "epoch": 0.47030529761507117,
      "grad_norm": 36.31256103515625,
      "learning_rate": 6.389938627084732e-07,
      "logits/chosen": -0.04478149488568306,
      "logits/rejected": -0.054534912109375,
      "logps/chosen": -181.5046844482422,
      "logps/rejected": -192.84375,
      "logps/weighted_chosen": -1.712792992591858,
      "logps/weighted_rejected": -1.4381287097930908,
      "loss": 0.7264,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -64.04609680175781,
      "rewards/margins": 6.507421970367432,
      "rewards/rejected": -70.56953430175781,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.3980148434638977,
      "rewards/weighted_margins": 0.10754700005054474,
      "rewards/weighted_rejected": -0.5054702758789062,
      "step": 880
    },
    {
      "epoch": 0.4756496759970606,
      "grad_norm": 6.0423126220703125,
      "learning_rate": 6.300100845203373e-07,
      "logits/chosen": -0.07687301933765411,
      "logits/rejected": -0.04276275634765625,
      "logps/chosen": -194.1609344482422,
      "logps/rejected": -200.828125,
      "logps/weighted_chosen": -0.6031554937362671,
      "logps/weighted_rejected": -0.86151123046875,
      "loss": 0.6793,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -63.5546875,
      "rewards/margins": 5.604296684265137,
      "rewards/rejected": -69.13984680175781,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.27074965834617615,
      "rewards/weighted_margins": 0.11608581244945526,
      "rewards/weighted_rejected": -0.38665771484375,
      "step": 890
    },
    {
      "epoch": 0.48099405437905,
      "grad_norm": 21.55549430847168,
      "learning_rate": 6.20981060350445e-07,
      "logits/chosen": -0.12602996826171875,
      "logits/rejected": -0.08534087985754013,
      "logps/chosen": -186.3312530517578,
      "logps/rejected": -190.0625,
      "logps/weighted_chosen": -0.8782455325126648,
      "logps/weighted_rejected": -1.0832672119140625,
      "loss": 0.6442,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -65.2984390258789,
      "rewards/margins": 6.321875095367432,
      "rewards/rejected": -71.61796569824219,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.3018638491630554,
      "rewards/weighted_margins": 0.19889220595359802,
      "rewards/weighted_rejected": -0.5003150701522827,
      "step": 900
    },
    {
      "epoch": 0.4863384327610395,
      "grad_norm": 15.255428314208984,
      "learning_rate": 6.119099324709087e-07,
      "logits/chosen": -0.08440399169921875,
      "logits/rejected": -0.03687591478228569,
      "logps/chosen": -183.1640625,
      "logps/rejected": -192.125,
      "logps/weighted_chosen": -0.7069732546806335,
      "logps/weighted_rejected": -1.348388671875,
      "loss": 0.7239,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -65.2796859741211,
      "rewards/margins": 7.408593654632568,
      "rewards/rejected": -72.6890640258789,
      "rewards/weighted_accuracies": 0.5093749761581421,
      "rewards/weighted_chosen": -0.3756294250488281,
      "rewards/weighted_margins": 0.05287017673254013,
      "rewards/weighted_rejected": -0.4286758303642273,
      "step": 910
    },
    {
      "epoch": 0.4916828111430289,
      "grad_norm": 10.355165481567383,
      "learning_rate": 6.027998578067316e-07,
      "logits/chosen": -0.06318511813879013,
      "logits/rejected": -0.02981414832174778,
      "logps/chosen": -199.36563110351562,
      "logps/rejected": -208.70938110351562,
      "logps/weighted_chosen": -0.6963897943496704,
      "logps/weighted_rejected": -0.969342052936554,
      "loss": 0.676,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -71.73905944824219,
      "rewards/margins": 6.481640815734863,
      "rewards/rejected": -78.2171859741211,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.2880393862724304,
      "rewards/weighted_margins": 0.1389778107404709,
      "rewards/weighted_rejected": -0.426931768655777,
      "step": 920
    },
    {
      "epoch": 0.4970271895250184,
      "grad_norm": 5.530470371246338,
      "learning_rate": 5.936540068371394e-07,
      "logits/chosen": 0.006648254580795765,
      "logits/rejected": 0.005427551455795765,
      "logps/chosen": -193.99374389648438,
      "logps/rejected": -202.4406280517578,
      "logps/weighted_chosen": -1.107049584388733,
      "logps/weighted_rejected": -1.3529266119003296,
      "loss": 0.6926,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -74.79765319824219,
      "rewards/margins": 5.872656345367432,
      "rewards/rejected": -80.6624984741211,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.43673935532569885,
      "rewards/weighted_margins": 0.12621307373046875,
      "rewards/weighted_rejected": -0.5626624822616577,
      "step": 930
    },
    {
      "epoch": 0.5023715679070078,
      "grad_norm": 5.006593704223633,
      "learning_rate": 5.844755624921918e-07,
      "logits/chosen": -0.005947875790297985,
      "logits/rejected": 0.02864990197122097,
      "logps/chosen": -187.453125,
      "logps/rejected": -192.7375030517578,
      "logps/weighted_chosen": -1.132598876953125,
      "logps/weighted_rejected": -1.0424163341522217,
      "loss": 0.7378,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -70.77812194824219,
      "rewards/margins": 5.782812595367432,
      "rewards/rejected": -76.56562805175781,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.44052886962890625,
      "rewards/weighted_margins": 0.07312469184398651,
      "rewards/weighted_rejected": -0.5137832760810852,
      "step": 940
    },
    {
      "epoch": 0.5077159462889973,
      "grad_norm": 16.502933502197266,
      "learning_rate": 5.752677190450634e-07,
      "logits/chosen": -0.021286392584443092,
      "logits/rejected": -0.00705032330006361,
      "logps/chosen": -186.49374389648438,
      "logps/rejected": -196.24844360351562,
      "logps/weighted_chosen": -1.068359375,
      "logps/weighted_rejected": -1.3232848644256592,
      "loss": 0.7251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -67.70234680175781,
      "rewards/margins": 7.561327934265137,
      "rewards/rejected": -75.28047180175781,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.390481561422348,
      "rewards/weighted_margins": 0.058994293212890625,
      "rewards/weighted_rejected": -0.44965285062789917,
      "step": 950
    },
    {
      "epoch": 0.5130603246709867,
      "grad_norm": 12.000792503356934,
      "learning_rate": 5.66033681000375e-07,
      "logits/chosen": -0.03893890231847763,
      "logits/rejected": 0.01745758019387722,
      "logps/chosen": -189.5234375,
      "logps/rejected": -201.2624969482422,
      "logps/weighted_chosen": -1.0573517084121704,
      "logps/weighted_rejected": -1.4250671863555908,
      "loss": 0.6656,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -69.40625,
      "rewards/margins": 8.408203125,
      "rewards/rejected": -77.78593444824219,
      "rewards/weighted_accuracies": 0.5718749761581421,
      "rewards/weighted_chosen": -0.2941024899482727,
      "rewards/weighted_margins": 0.1710365265607834,
      "rewards/weighted_rejected": -0.4653762876987457,
      "step": 960
    },
    {
      "epoch": 0.5184047030529761,
      "grad_norm": 8.829608917236328,
      "learning_rate": 5.567766619789658e-07,
      "logits/chosen": -0.03634033352136612,
      "logits/rejected": 0.0003803253057412803,
      "logps/chosen": -190.27969360351562,
      "logps/rejected": -199.0500030517578,
      "logps/weighted_chosen": -1.1300811767578125,
      "logps/weighted_rejected": -1.3106689453125,
      "loss": 0.6931,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -67.55078125,
      "rewards/margins": 7.454297065734863,
      "rewards/rejected": -74.9703140258789,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": -0.3190757632255554,
      "rewards/weighted_margins": 0.11787338554859161,
      "rewards/weighted_rejected": -0.4370986819267273,
      "step": 970
    },
    {
      "epoch": 0.5237490814349656,
      "grad_norm": 9.326337814331055,
      "learning_rate": 5.474998835994916e-07,
      "logits/chosen": 0.009844208136200905,
      "logits/rejected": 0.051239013671875,
      "logps/chosen": -195.78125,
      "logps/rejected": -208.5343780517578,
      "logps/weighted_chosen": -1.179107666015625,
      "logps/weighted_rejected": -1.546759009361267,
      "loss": 0.6677,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -70.6781234741211,
      "rewards/margins": 6.912109375,
      "rewards/rejected": -77.5609359741211,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.28845518827438354,
      "rewards/weighted_margins": 0.1378173828125,
      "rewards/weighted_rejected": -0.42629241943359375,
      "step": 980
    },
    {
      "epoch": 0.529093459816955,
      "grad_norm": 10.308549880981445,
      "learning_rate": 5.382065743572411e-07,
      "logits/chosen": -0.025909423828125,
      "logits/rejected": 0.01560287456959486,
      "logps/chosen": -192.8093719482422,
      "logps/rejected": -205.2843780517578,
      "logps/weighted_chosen": -0.8877533078193665,
      "logps/weighted_rejected": -1.2557891607284546,
      "loss": 0.6811,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -70.57890319824219,
      "rewards/margins": 8.194921493530273,
      "rewards/rejected": -78.80000305175781,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.3638969361782074,
      "rewards/weighted_margins": 0.13902893662452698,
      "rewards/weighted_rejected": -0.5028846859931946,
      "step": 990
    },
    {
      "epoch": 0.5344378381989445,
      "grad_norm": 6.612018585205078,
      "learning_rate": 5.288999685005571e-07,
      "logits/chosen": -0.03016967698931694,
      "logits/rejected": -0.0050643919967114925,
      "logps/chosen": -192.640625,
      "logps/rejected": -201.4656219482422,
      "logps/weighted_chosen": -0.626147449016571,
      "logps/weighted_rejected": -0.8550964593887329,
      "loss": 0.672,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -72.01640319824219,
      "rewards/margins": 6.489062309265137,
      "rewards/rejected": -78.5062484741211,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.2895965576171875,
      "rewards/weighted_margins": 0.1136070266366005,
      "rewards/weighted_rejected": -0.4031356871128082,
      "step": 1000
    },
    {
      "epoch": 0.5344378381989445,
      "eval_logits/chosen": -0.1623995155096054,
      "eval_logits/rejected": -0.13676175475120544,
      "eval_logps/chosen": -193.06211853027344,
      "eval_logps/rejected": -200.78919982910156,
      "eval_logps/weighted_chosen": -1.0030580759048462,
      "eval_logps/weighted_rejected": -1.3398514986038208,
      "eval_loss": 0.6716485619544983,
      "eval_rewards/accuracies": 0.5982688665390015,
      "eval_rewards/chosen": -71.73777770996094,
      "eval_rewards/margins": 6.556771755218506,
      "eval_rewards/rejected": -78.29378509521484,
      "eval_rewards/weighted_accuracies": 0.5570265054702759,
      "eval_rewards/weighted_chosen": -0.36391088366508484,
      "eval_rewards/weighted_margins": 0.17662912607192993,
      "eval_rewards/weighted_rejected": -0.5405399203300476,
      "eval_runtime": 1144.2369,
      "eval_samples_per_second": 1.714,
      "eval_steps_per_second": 0.429,
      "step": 1000
    },
    {
      "epoch": 0.539782216580934,
      "grad_norm": 6.273863315582275,
      "learning_rate": 5.19583304905257e-07,
      "logits/chosen": -0.04050292819738388,
      "logits/rejected": -0.010775757022202015,
      "logps/chosen": -187.6999969482422,
      "logps/rejected": -195.6999969482422,
      "logps/weighted_chosen": -0.898364245891571,
      "logps/weighted_rejected": -1.051977515220642,
      "loss": 0.699,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -72.2406234741211,
      "rewards/margins": 6.910546779632568,
      "rewards/rejected": -79.1578140258789,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.3900245726108551,
      "rewards/weighted_margins": 0.1439811736345291,
      "rewards/weighted_rejected": -0.5338424444198608,
      "step": 1010
    },
    {
      "epoch": 0.5451265949629234,
      "grad_norm": 8.971517562866211,
      "learning_rate": 5.102598259474434e-07,
      "logits/chosen": -0.005509376525878906,
      "logits/rejected": 0.02565765380859375,
      "logps/chosen": -184.39999389648438,
      "logps/rejected": -187.3781280517578,
      "logps/weighted_chosen": -0.8080657720565796,
      "logps/weighted_rejected": -1.043609619140625,
      "loss": 0.7008,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -72.9703140258789,
      "rewards/margins": 5.461718559265137,
      "rewards/rejected": -78.44844055175781,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.3826606869697571,
      "rewards/weighted_margins": 0.062392424792051315,
      "rewards/weighted_rejected": -0.44501036405563354,
      "step": 1020
    },
    {
      "epoch": 0.5504709733449128,
      "grad_norm": 12.744133949279785,
      "learning_rate": 5.009327763750932e-07,
      "logits/chosen": -0.049317169934511185,
      "logits/rejected": -0.0027618408203125,
      "logps/chosen": -192.65625,
      "logps/rejected": -202.55624389648438,
      "logps/weighted_chosen": -0.8524872064590454,
      "logps/weighted_rejected": -1.3228943347930908,
      "loss": 0.6779,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -74.0992202758789,
      "rewards/margins": 7.507031440734863,
      "rewards/rejected": -81.62187194824219,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.3904098570346832,
      "rewards/weighted_margins": 0.14227446913719177,
      "rewards/weighted_rejected": -0.5326492190361023,
      "step": 1030
    },
    {
      "epoch": 0.5558153517269022,
      "grad_norm": 6.275015354156494,
      "learning_rate": 4.916054021788265e-07,
      "logits/chosen": -0.04349822923541069,
      "logits/rejected": -0.005797195248305798,
      "logps/chosen": -192.04061889648438,
      "logps/rejected": -196.96875,
      "logps/weighted_chosen": -1.060455322265625,
      "logps/weighted_rejected": -1.1041381359100342,
      "loss": 0.676,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -73.9984359741211,
      "rewards/margins": 6.270703315734863,
      "rewards/rejected": -80.27656555175781,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.3287033140659332,
      "rewards/weighted_margins": 0.17382964491844177,
      "rewards/weighted_rejected": -0.5024040341377258,
      "step": 1040
    },
    {
      "epoch": 0.5611597301088918,
      "grad_norm": 7.003974437713623,
      "learning_rate": 4.822809494622379e-07,
      "logits/chosen": -0.07877655327320099,
      "logits/rejected": -0.06013183668255806,
      "logps/chosen": -189.9453125,
      "logps/rejected": -201.99374389648438,
      "logps/weighted_chosen": -1.28973388671875,
      "logps/weighted_rejected": -1.1621277332305908,
      "loss": 0.7014,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -69.9781265258789,
      "rewards/margins": 8.531641006469727,
      "rewards/rejected": -78.4828109741211,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.38208693265914917,
      "rewards/weighted_margins": 0.09144439548254013,
      "rewards/weighted_rejected": -0.4734359681606293,
      "step": 1050
    },
    {
      "epoch": 0.5665041084908812,
      "grad_norm": 7.428668975830078,
      "learning_rate": 4.729626633121914e-07,
      "logits/chosen": -0.065679170191288,
      "logits/rejected": -0.02408447302877903,
      "logps/chosen": -191.80313110351562,
      "logps/rejected": -202.32186889648438,
      "logps/weighted_chosen": -1.075415015220642,
      "logps/weighted_rejected": -1.3513977527618408,
      "loss": 0.6657,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -73.8218765258789,
      "rewards/margins": 7.512499809265137,
      "rewards/rejected": -81.32499694824219,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.32668304443359375,
      "rewards/weighted_margins": 0.21370239555835724,
      "rewards/weighted_rejected": -0.5402389764785767,
      "step": 1060
    },
    {
      "epoch": 0.5718484868728706,
      "grad_norm": 10.470946311950684,
      "learning_rate": 4.636537866694692e-07,
      "logits/chosen": -0.023516083136200905,
      "logits/rejected": 0.0013282776344567537,
      "logps/chosen": -193.6531219482422,
      "logps/rejected": -200.671875,
      "logps/weighted_chosen": -1.336614966392517,
      "logps/weighted_rejected": -1.549737572669983,
      "loss": 0.6482,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -75.12890625,
      "rewards/margins": 6.398828029632568,
      "rewards/rejected": -81.51094055175781,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.3157196044921875,
      "rewards/weighted_margins": 0.2204277068376541,
      "rewards/weighted_rejected": -0.5359970331192017,
      "step": 1070
    },
    {
      "epoch": 0.57719286525486,
      "grad_norm": 5.486422061920166,
      "learning_rate": 4.5435755920016557e-07,
      "logits/chosen": -0.15928420424461365,
      "logits/rejected": -0.16242370009422302,
      "logps/chosen": -213.3874969482422,
      "logps/rejected": -219.72500610351562,
      "logps/weighted_chosen": -0.9821227788925171,
      "logps/weighted_rejected": -0.865551769733429,
      "loss": 0.6778,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -76.74687194824219,
      "rewards/margins": 8.090624809265137,
      "rewards/rejected": -84.828125,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.32687264680862427,
      "rewards/weighted_margins": 0.08376693725585938,
      "rewards/weighted_rejected": -0.4107864499092102,
      "step": 1080
    },
    {
      "epoch": 0.5825372436368494,
      "grad_norm": 8.151789665222168,
      "learning_rate": 4.450772161682221e-07,
      "logits/chosen": -0.068756103515625,
      "logits/rejected": -0.015399170108139515,
      "logps/chosen": -199.9187469482422,
      "logps/rejected": -204.18124389648438,
      "logps/weighted_chosen": -1.588891625404358,
      "logps/weighted_rejected": -1.757287621498108,
      "loss": 0.6788,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -74.18281555175781,
      "rewards/margins": 8.891406059265137,
      "rewards/rejected": -83.09687805175781,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.30588531494140625,
      "rewards/weighted_margins": 0.1938468962907791,
      "rewards/weighted_rejected": -0.49979859590530396,
      "step": 1090
    },
    {
      "epoch": 0.587881622018839,
      "grad_norm": 6.956281661987305,
      "learning_rate": 4.3581598730949376e-07,
      "logits/chosen": -0.064672090113163,
      "logits/rejected": -0.038168907165527344,
      "logps/chosen": -203.828125,
      "logps/rejected": -218.46249389648438,
      "logps/weighted_chosen": -1.4421265125274658,
      "logps/weighted_rejected": -1.3359496593475342,
      "loss": 0.6969,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -78.1781234741211,
      "rewards/margins": 10.614843368530273,
      "rewards/rejected": -88.7796859741211,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.38915711641311646,
      "rewards/weighted_margins": 0.0950775146484375,
      "rewards/weighted_rejected": -0.4842666685581207,
      "step": 1100
    },
    {
      "epoch": 0.5932260004008284,
      "grad_norm": 9.076947212219238,
      "learning_rate": 4.2657709570774007e-07,
      "logits/chosen": -0.06801605224609375,
      "logits/rejected": -0.05637931823730469,
      "logps/chosen": -195.67813110351562,
      "logps/rejected": -204.66250610351562,
      "logps/weighted_chosen": -1.285638451576233,
      "logps/weighted_rejected": -1.3720200061798096,
      "loss": 0.6571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -76.2890625,
      "rewards/margins": 8.073827743530273,
      "rewards/rejected": -84.35624694824219,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.4137374758720398,
      "rewards/weighted_margins": 0.19571837782859802,
      "rewards/weighted_rejected": -0.6095176935195923,
      "step": 1110
    },
    {
      "epoch": 0.5985703787828178,
      "grad_norm": 7.4237751960754395,
      "learning_rate": 4.173637566729304e-07,
      "logits/chosen": -0.09175796806812286,
      "logits/rejected": -0.07670821994543076,
      "logps/chosen": -196.52188110351562,
      "logps/rejected": -210.00625610351562,
      "logps/weighted_chosen": -1.169805884361267,
      "logps/weighted_rejected": -1.150994896888733,
      "loss": 0.6803,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -76.2906265258789,
      "rewards/margins": 10.744921684265137,
      "rewards/rejected": -87.03125,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.39191436767578125,
      "rewards/weighted_margins": 0.15792007744312286,
      "rewards/weighted_rejected": -0.549652099609375,
      "step": 1120
    },
    {
      "epoch": 0.6039147571648072,
      "grad_norm": 6.803237438201904,
      "learning_rate": 4.0817917662225467e-07,
      "logits/chosen": -0.14934463798999786,
      "logits/rejected": -0.13103675842285156,
      "logps/chosen": -204.88436889648438,
      "logps/rejected": -215.25155639648438,
      "logps/weighted_chosen": -1.14501953125,
      "logps/weighted_rejected": -1.147515892982483,
      "loss": 0.6706,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -80.02812194824219,
      "rewards/margins": 9.226953506469727,
      "rewards/rejected": -89.2734375,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.3231521546840668,
      "rewards/weighted_margins": 0.16764679551124573,
      "rewards/weighted_rejected": -0.49060362577438354,
      "step": 1130
    },
    {
      "epoch": 0.6092591355467967,
      "grad_norm": 7.308501720428467,
      "learning_rate": 3.9902655196422957e-07,
      "logits/chosen": -0.17415618896484375,
      "logits/rejected": -0.13456878066062927,
      "logps/chosen": -204.93594360351562,
      "logps/rejected": -214.28750610351562,
      "logps/weighted_chosen": -0.799731433391571,
      "logps/weighted_rejected": -1.1109497547149658,
      "loss": 0.691,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -82.1039047241211,
      "rewards/margins": 7.87109375,
      "rewards/rejected": -89.9781265258789,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.4164085388183594,
      "rewards/weighted_margins": 0.125599667429924,
      "rewards/weighted_rejected": -0.5417296886444092,
      "step": 1140
    },
    {
      "epoch": 0.6146035139287862,
      "grad_norm": 13.811458587646484,
      "learning_rate": 3.8990906798628763e-07,
      "logits/chosen": -0.08937682956457138,
      "logits/rejected": -0.06486892700195312,
      "logps/chosen": -210.97500610351562,
      "logps/rejected": -214.6125030517578,
      "logps/weighted_chosen": -1.324133276939392,
      "logps/weighted_rejected": -1.3257598876953125,
      "loss": 0.7048,
      "rewards/accuracies": 0.528124988079071,
      "rewards/chosen": -80.96875,
      "rewards/margins": 6.654687404632568,
      "rewards/rejected": -87.6578140258789,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.3775787353515625,
      "rewards/weighted_margins": 0.11986999213695526,
      "rewards/weighted_rejected": -0.4972671568393707,
      "step": 1150
    },
    {
      "epoch": 0.6199478923107756,
      "grad_norm": 10.00770378112793,
      "learning_rate": 3.8082989774623587e-07,
      "logits/chosen": -0.01934986189007759,
      "logits/rejected": -0.015110015869140625,
      "logps/chosen": -188.74063110351562,
      "logps/rejected": -204.4187469482422,
      "logps/weighted_chosen": -1.20220947265625,
      "logps/weighted_rejected": -1.109490990638733,
      "loss": 0.6717,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -77.26094055175781,
      "rewards/margins": 9.564844131469727,
      "rewards/rejected": -86.8343734741211,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.3926025331020355,
      "rewards/weighted_margins": 0.13916321098804474,
      "rewards/weighted_rejected": -0.5318969488143921,
      "step": 1160
    },
    {
      "epoch": 0.625292270692765,
      "grad_norm": 28.483694076538086,
      "learning_rate": 3.717922009679725e-07,
      "logits/chosen": -0.053440093994140625,
      "logits/rejected": -0.0072647095657885075,
      "logps/chosen": -201.890625,
      "logps/rejected": -210.5187530517578,
      "logps/weighted_chosen": -1.0479736328125,
      "logps/weighted_rejected": -1.5367584228515625,
      "loss": 0.6686,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -78.7554702758789,
      "rewards/margins": 8.385546684265137,
      "rewards/rejected": -87.1624984741211,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.352639764547348,
      "rewards/weighted_margins": 0.159870907664299,
      "rewards/weighted_rejected": -0.5122512578964233,
      "step": 1170
    },
    {
      "epoch": 0.6306366490747545,
      "grad_norm": 7.887178897857666,
      "learning_rate": 3.6279912294184207e-07,
      "logits/chosen": -0.06793518364429474,
      "logits/rejected": -0.03975982591509819,
      "logps/chosen": -208.45938110351562,
      "logps/rejected": -216.97500610351562,
      "logps/weighted_chosen": -1.1531188488006592,
      "logps/weighted_rejected": -1.3199646472930908,
      "loss": 0.664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -83.76875305175781,
      "rewards/margins": 8.121874809265137,
      "rewards/rejected": -91.8890609741211,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.3757583498954773,
      "rewards/weighted_margins": 0.18969421088695526,
      "rewards/weighted_rejected": -0.5654312372207642,
      "step": 1180
    },
    {
      "epoch": 0.6359810274567439,
      "grad_norm": 21.66887664794922,
      "learning_rate": 3.5385379343001507e-07,
      "logits/chosen": -0.041289519518613815,
      "logits/rejected": -0.01831817626953125,
      "logps/chosen": -198.00936889648438,
      "logps/rejected": -204.0437469482422,
      "logps/weighted_chosen": -1.02520751953125,
      "logps/weighted_rejected": -1.4494140148162842,
      "loss": 0.6816,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -79.15156555175781,
      "rewards/margins": 7.596484184265137,
      "rewards/rejected": -86.73515319824219,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": -0.3985244631767273,
      "rewards/weighted_margins": 0.12162628024816513,
      "rewards/weighted_rejected": -0.5204399228096008,
      "step": 1190
    },
    {
      "epoch": 0.6413254058387334,
      "grad_norm": 11.715490341186523,
      "learning_rate": 3.4495932557727114e-07,
      "logits/chosen": -0.0670623779296875,
      "logits/rejected": -0.04256897047162056,
      "logps/chosen": -200.75936889648438,
      "logps/rejected": -214.21875,
      "logps/weighted_chosen": -1.1181519031524658,
      "logps/weighted_rejected": -1.4523468017578125,
      "loss": 0.6876,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -81.3304672241211,
      "rewards/margins": 10.844531059265137,
      "rewards/rejected": -92.1703109741211,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.4529586732387543,
      "rewards/weighted_margins": 0.178059384226799,
      "rewards/weighted_rejected": -0.6310364007949829,
      "step": 1200
    },
    {
      "epoch": 0.6466697842207229,
      "grad_norm": 5.799643516540527,
      "learning_rate": 3.3611881482756464e-07,
      "logits/chosen": -0.02250823937356472,
      "logits/rejected": -0.02404480054974556,
      "logps/chosen": -209.1593780517578,
      "logps/rejected": -213.9921875,
      "logps/weighted_chosen": -1.55322265625,
      "logps/weighted_rejected": -1.4918396472930908,
      "loss": 0.6919,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -84.61796569824219,
      "rewards/margins": 6.630078315734863,
      "rewards/rejected": -91.2593765258789,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.4175216555595398,
      "rewards/weighted_margins": 0.15537413954734802,
      "rewards/weighted_rejected": -0.5732803344726562,
      "step": 1210
    },
    {
      "epoch": 0.6520141626027123,
      "grad_norm": 10.090937614440918,
      "learning_rate": 3.2733533784675273e-07,
      "logits/chosen": 0.013554382137954235,
      "logits/rejected": 0.04766502231359482,
      "logps/chosen": -197.359375,
      "logps/rejected": -209.89688110351562,
      "logps/weighted_chosen": -1.3024734258651733,
      "logps/weighted_rejected": -1.784857153892517,
      "loss": 0.69,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -79.140625,
      "rewards/margins": 10.651171684265137,
      "rewards/rejected": -89.7906265258789,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.4760070741176605,
      "rewards/weighted_margins": 0.15030670166015625,
      "rewards/weighted_rejected": -0.6262580752372742,
      "step": 1220
    },
    {
      "epoch": 0.6573585409847017,
      "grad_norm": 7.221394062042236,
      "learning_rate": 3.1861195145185603e-07,
      "logits/chosen": -0.03308219835162163,
      "logits/rejected": 0.00255584716796875,
      "logps/chosen": -206.4031219482422,
      "logps/rejected": -215.953125,
      "logps/weighted_chosen": -1.181341528892517,
      "logps/weighted_rejected": -1.657598853111267,
      "loss": 0.6826,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -83.6734390258789,
      "rewards/margins": 7.490234375,
      "rewards/rejected": -91.1578140258789,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.3481903076171875,
      "rewards/weighted_margins": 0.16850128769874573,
      "rewards/weighted_rejected": -0.5166076421737671,
      "step": 1230
    },
    {
      "epoch": 0.6627029193666911,
      "grad_norm": 50.77132797241211,
      "learning_rate": 3.099516915472289e-07,
      "logits/chosen": 0.006022644229233265,
      "logits/rejected": 0.08475188910961151,
      "logps/chosen": -200.8328094482422,
      "logps/rejected": -205.4609375,
      "logps/weighted_chosen": -0.924468994140625,
      "logps/weighted_rejected": -1.4624541997909546,
      "loss": 0.6753,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -80.05937194824219,
      "rewards/margins": 5.724609375,
      "rewards/rejected": -85.7828140258789,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.3424621522426605,
      "rewards/weighted_margins": 0.1885635405778885,
      "rewards/weighted_rejected": -0.5309646725654602,
      "step": 1240
    },
    {
      "epoch": 0.6680472977486807,
      "grad_norm": 7.760387897491455,
      "learning_rate": 3.013575720680062e-07,
      "logits/chosen": 0.020127106457948685,
      "logits/rejected": 0.042375944554805756,
      "logps/chosen": -202.19375610351562,
      "logps/rejected": -208.9968719482422,
      "logps/weighted_chosen": -1.1934722661972046,
      "logps/weighted_rejected": -1.6453979015350342,
      "loss": 0.665,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -81.3656234741211,
      "rewards/margins": 8.227343559265137,
      "rewards/rejected": -89.56718444824219,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.3932647705078125,
      "rewards/weighted_margins": 0.21837767958641052,
      "rewards/weighted_rejected": -0.6111129522323608,
      "step": 1250
    },
    {
      "epoch": 0.6733916761306701,
      "grad_norm": 16.958906173706055,
      "learning_rate": 2.928325839311974e-07,
      "logits/chosen": -0.02686614915728569,
      "logits/rejected": -0.016692351549863815,
      "logps/chosen": -194.46719360351562,
      "logps/rejected": -203.3093719482422,
      "logps/weighted_chosen": -1.3333160877227783,
      "logps/weighted_rejected": -1.2732360363006592,
      "loss": 0.7271,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -79.31562805175781,
      "rewards/margins": 8.610547065734863,
      "rewards/rejected": -87.9281234741211,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.4805351197719574,
      "rewards/weighted_margins": 0.173136904835701,
      "rewards/weighted_rejected": -0.6535751223564148,
      "step": 1260
    },
    {
      "epoch": 0.6787360545126595,
      "grad_norm": 19.331541061401367,
      "learning_rate": 2.843796939947896e-07,
      "logits/chosen": -0.015142058953642845,
      "logits/rejected": 0.009578704833984375,
      "logps/chosen": -199.8125,
      "logps/rejected": -213.27499389648438,
      "logps/weighted_chosen": -1.1416137218475342,
      "logps/weighted_rejected": -1.472558617591858,
      "loss": 0.6635,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -82.5484390258789,
      "rewards/margins": 10.644140243530273,
      "rewards/rejected": -93.1968765258789,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.4067550599575043,
      "rewards/weighted_margins": 0.1748603880405426,
      "rewards/weighted_rejected": -0.5817291140556335,
      "step": 1270
    },
    {
      "epoch": 0.6840804328946489,
      "grad_norm": 15.563817977905273,
      "learning_rate": 2.7600184402522575e-07,
      "logits/chosen": 0.02512359619140625,
      "logits/rejected": 0.06499175727367401,
      "logps/chosen": -196.04061889648438,
      "logps/rejected": -209.2218780517578,
      "logps/weighted_chosen": -1.2925841808319092,
      "logps/weighted_rejected": -1.1955718994140625,
      "loss": 0.6787,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -79.7046890258789,
      "rewards/margins": 11.271875381469727,
      "rewards/rejected": -90.94921875,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.4067886471748352,
      "rewards/weighted_margins": 0.17460021376609802,
      "rewards/weighted_rejected": -0.5814231634140015,
      "step": 1280
    },
    {
      "epoch": 0.6894248112766383,
      "grad_norm": 13.546245574951172,
      "learning_rate": 2.6770194967361303e-07,
      "logits/chosen": -0.0179290771484375,
      "logits/rejected": 0.0178680419921875,
      "logps/chosen": -213.72500610351562,
      "logps/rejected": -227.05313110351562,
      "logps/weighted_chosen": -1.001013159751892,
      "logps/weighted_rejected": -1.686248779296875,
      "loss": 0.6712,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -86.60624694824219,
      "rewards/margins": 8.127344131469727,
      "rewards/rejected": -94.73124694824219,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.3357299864292145,
      "rewards/weighted_margins": 0.20237883925437927,
      "rewards/weighted_rejected": -0.5380722284317017,
      "step": 1290
    },
    {
      "epoch": 0.6947691896586279,
      "grad_norm": 14.806809425354004,
      "learning_rate": 2.594828994610211e-07,
      "logits/chosen": 0.027711868286132812,
      "logits/rejected": 0.05919818952679634,
      "logps/chosen": -193.265625,
      "logps/rejected": -206.9031219482422,
      "logps/weighted_chosen": -1.0292694568634033,
      "logps/weighted_rejected": -1.295262098312378,
      "loss": 0.6787,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -80.3687515258789,
      "rewards/margins": 8.969141006469727,
      "rewards/rejected": -89.3671875,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.3460655212402344,
      "rewards/weighted_margins": 0.17020568251609802,
      "rewards/weighted_rejected": -0.5160125494003296,
      "step": 1300
    },
    {
      "epoch": 0.7001135680406173,
      "grad_norm": 12.198771476745605,
      "learning_rate": 2.513475537732219e-07,
      "logits/chosen": 0.0022941590286791325,
      "logits/rejected": 0.025787353515625,
      "logps/chosen": -204.703125,
      "logps/rejected": -217.66250610351562,
      "logps/weighted_chosen": -1.215307593345642,
      "logps/weighted_rejected": -1.44464111328125,
      "loss": 0.665,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -84.4281234741211,
      "rewards/margins": 8.751562118530273,
      "rewards/rejected": -93.18281555175781,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.3498634397983551,
      "rewards/weighted_margins": 0.19211502373218536,
      "rewards/weighted_rejected": -0.541577160358429,
      "step": 1310
    },
    {
      "epoch": 0.7054579464226067,
      "grad_norm": 18.498014450073242,
      "learning_rate": 2.4329874386522113e-07,
      "logits/chosen": 0.0073295594193041325,
      "logits/rejected": 0.055358123034238815,
      "logps/chosen": -199.6437530517578,
      "logps/rejected": -208.70156860351562,
      "logps/weighted_chosen": -1.0841186046600342,
      "logps/weighted_rejected": -1.4295532703399658,
      "loss": 0.6551,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -83.765625,
      "rewards/margins": 8.8515625,
      "rewards/rejected": -92.6343765258789,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.4359336793422699,
      "rewards/weighted_margins": 0.21386031806468964,
      "rewards/weighted_rejected": -0.6501365900039673,
      "step": 1320
    },
    {
      "epoch": 0.7108023248045962,
      "grad_norm": 15.962686538696289,
      "learning_rate": 2.3533927087592713e-07,
      "logits/chosen": -0.000156402587890625,
      "logits/rejected": 0.01469268836081028,
      "logps/chosen": -206.29061889648438,
      "logps/rejected": -216.75,
      "logps/weighted_chosen": -1.1605316400527954,
      "logps/weighted_rejected": -1.276606798171997,
      "loss": 0.7058,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -83.1468734741211,
      "rewards/margins": 7.533203125,
      "rewards/rejected": -90.6796875,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.422750860452652,
      "rewards/weighted_margins": 0.102025605738163,
      "rewards/weighted_rejected": -0.5250183343887329,
      "step": 1330
    },
    {
      "epoch": 0.7161467031865856,
      "grad_norm": 10.239470481872559,
      "learning_rate": 2.2747190485330193e-07,
      "logits/chosen": 0.02880706824362278,
      "logits/rejected": 0.07036437839269638,
      "logps/chosen": -199.8859405517578,
      "logps/rejected": -209.57186889648438,
      "logps/weighted_chosen": -1.2549285888671875,
      "logps/weighted_rejected": -1.352288842201233,
      "loss": 0.6718,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -80.2109375,
      "rewards/margins": 10.519922256469727,
      "rewards/rejected": -90.71875,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.3369941711425781,
      "rewards/weighted_margins": 0.2133018523454666,
      "rewards/weighted_rejected": -0.5501846075057983,
      "step": 1340
    },
    {
      "epoch": 0.7214910815685751,
      "grad_norm": 12.942761421203613,
      "learning_rate": 2.1969938379033166e-07,
      "logits/chosen": -0.0024871826171875,
      "logits/rejected": -0.01717834547162056,
      "logps/chosen": -206.4250030517578,
      "logps/rejected": -220.47030639648438,
      "logps/weighted_chosen": -1.2035309076309204,
      "logps/weighted_rejected": -1.3001312017440796,
      "loss": 0.7016,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -82.8765640258789,
      "rewards/margins": 8.98828125,
      "rewards/rejected": -91.8671875,
      "rewards/weighted_accuracies": 0.5718749761581421,
      "rewards/weighted_chosen": -0.37746506929397583,
      "rewards/weighted_margins": 0.11672210693359375,
      "rewards/weighted_rejected": -0.4940811097621918,
      "step": 1350
    },
    {
      "epoch": 0.7268354599505645,
      "grad_norm": 16.300827026367188,
      "learning_rate": 2.1202441267215265e-07,
      "logits/chosen": 0.04172973707318306,
      "logits/rejected": 0.06365661323070526,
      "logps/chosen": -183.5437469482422,
      "logps/rejected": -198.49063110351562,
      "logps/weighted_chosen": -1.542272925376892,
      "logps/weighted_rejected": -1.678741455078125,
      "loss": 0.7155,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -78.47343444824219,
      "rewards/margins": 10.018359184265137,
      "rewards/rejected": -88.47343444824219,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.4637313783168793,
      "rewards/weighted_margins": 0.1779930144548416,
      "rewards/weighted_rejected": -0.6415374875068665,
      "step": 1360
    },
    {
      "epoch": 0.732179838332554,
      "grad_norm": 17.129104614257812,
      "learning_rate": 2.0444966253466572e-07,
      "logits/chosen": -0.01758270338177681,
      "logits/rejected": 0.049257658421993256,
      "logps/chosen": -198.3562469482422,
      "logps/rejected": -212.31875610351562,
      "logps/weighted_chosen": -1.0854461193084717,
      "logps/weighted_rejected": -1.6400238275527954,
      "loss": 0.6718,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -81.81718444824219,
      "rewards/margins": 8.70703125,
      "rewards/rejected": -90.52812194824219,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.3319801390171051,
      "rewards/weighted_margins": 0.17666855454444885,
      "rewards/weighted_rejected": -0.5086410641670227,
      "step": 1370
    },
    {
      "epoch": 0.7375242167145434,
      "grad_norm": 15.24010181427002,
      "learning_rate": 1.9697776953496397e-07,
      "logits/chosen": -0.0036972046364098787,
      "logits/rejected": 0.04438323900103569,
      "logps/chosen": -204.8406219482422,
      "logps/rejected": -215.5343780517578,
      "logps/weighted_chosen": -1.2466247081756592,
      "logps/weighted_rejected": -1.588470458984375,
      "loss": 0.6659,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -82.30078125,
      "rewards/margins": 10.021875381469727,
      "rewards/rejected": -92.33515930175781,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.3886352479457855,
      "rewards/weighted_margins": 0.18134918808937073,
      "rewards/weighted_rejected": -0.5699905157089233,
      "step": 1380
    },
    {
      "epoch": 0.7428685950965328,
      "grad_norm": 13.349143028259277,
      "learning_rate": 1.8961133403390113e-07,
      "logits/chosen": -0.03137054294347763,
      "logits/rejected": 0.01921997033059597,
      "logps/chosen": -205.91250610351562,
      "logps/rejected": -215.74374389648438,
      "logps/weighted_chosen": -0.754425048828125,
      "logps/weighted_rejected": -1.830108642578125,
      "loss": 0.6648,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -82.6187515258789,
      "rewards/margins": 9.654296875,
      "rewards/rejected": -92.2750015258789,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.38862380385398865,
      "rewards/weighted_margins": 0.23016509413719177,
      "rewards/weighted_rejected": -0.618792712688446,
      "step": 1390
    },
    {
      "epoch": 0.7482129734785222,
      "grad_norm": 12.553123474121094,
      "learning_rate": 1.823529196911156e-07,
      "logits/chosen": -0.07255706936120987,
      "logits/rejected": -0.03428497165441513,
      "logps/chosen": -207.0109405517578,
      "logps/rejected": -213.7937469482422,
      "logps/weighted_chosen": -1.1017853021621704,
      "logps/weighted_rejected": -1.335351586341858,
      "loss": 0.6721,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -83.64375305175781,
      "rewards/margins": 7.791796684265137,
      "rewards/rejected": -91.4296875,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.35135191679000854,
      "rewards/weighted_margins": 0.21256867051124573,
      "rewards/weighted_rejected": -0.5637038946151733,
      "step": 1400
    },
    {
      "epoch": 0.7535573518605118,
      "grad_norm": 10.907644271850586,
      "learning_rate": 1.7520505257282886e-07,
      "logits/chosen": 0.02479248121380806,
      "logits/rejected": 0.05216522142291069,
      "logps/chosen": -198.2531280517578,
      "logps/rejected": -207.4499969482422,
      "logps/weighted_chosen": -1.016326904296875,
      "logps/weighted_rejected": -1.4452117681503296,
      "loss": 0.6724,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -80.75312805175781,
      "rewards/margins": 9.733593940734863,
      "rewards/rejected": -90.4671859741211,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.3796447813510895,
      "rewards/weighted_margins": 0.1787368804216385,
      "rewards/weighted_rejected": -0.5583381652832031,
      "step": 1410
    },
    {
      "epoch": 0.7589017302425012,
      "grad_norm": 9.989439964294434,
      "learning_rate": 1.6817022027272572e-07,
      "logits/chosen": 0.03133545070886612,
      "logits/rejected": 0.03775329515337944,
      "logps/chosen": -207.06405639648438,
      "logps/rejected": -224.08749389648438,
      "logps/weighted_chosen": -1.253021240234375,
      "logps/weighted_rejected": -1.788061499595642,
      "loss": 0.703,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -82.796875,
      "rewards/margins": 11.674609184265137,
      "rewards/rejected": -94.50468444824219,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.3869224488735199,
      "rewards/weighted_margins": 0.14797210693359375,
      "rewards/weighted_rejected": -0.5350815057754517,
      "step": 1420
    },
    {
      "epoch": 0.7642461086244906,
      "grad_norm": 12.953351974487305,
      "learning_rate": 1.6125087104622502e-07,
      "logits/chosen": -0.03018951416015625,
      "logits/rejected": 0.010332489386200905,
      "logps/chosen": -204.82186889648438,
      "logps/rejected": -211.2375030517578,
      "logps/weighted_chosen": -1.2213348150253296,
      "logps/weighted_rejected": -1.4504730701446533,
      "loss": 0.679,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -81.8109359741211,
      "rewards/margins": 8.600781440734863,
      "rewards/rejected": -90.39375305175781,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.43824997544288635,
      "rewards/weighted_margins": 0.12977370619773865,
      "rewards/weighted_rejected": -0.56793212890625,
      "step": 1430
    },
    {
      "epoch": 0.76959048700648,
      "grad_norm": 51.06248474121094,
      "learning_rate": 1.5444941295843945e-07,
      "logits/chosen": -0.0010368346702307463,
      "logits/rejected": -0.0032970428001135588,
      "logps/chosen": -206.72500610351562,
      "logps/rejected": -225.6125030517578,
      "logps/weighted_chosen": -0.7966338992118835,
      "logps/weighted_rejected": -0.8868865966796875,
      "loss": 0.6564,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -83.40937805175781,
      "rewards/margins": 12.608983993530273,
      "rewards/rejected": -96.03125,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.2895980775356293,
      "rewards/weighted_margins": 0.20346298813819885,
      "rewards/weighted_rejected": -0.49294739961624146,
      "step": 1440
    },
    {
      "epoch": 0.7749348653884695,
      "grad_norm": 7.5457658767700195,
      "learning_rate": 1.4776821304612392e-07,
      "logits/chosen": -0.01905670203268528,
      "logits/rejected": 0.01137466449290514,
      "logps/chosen": -196.77969360351562,
      "logps/rejected": -210.0859375,
      "logps/weighted_chosen": -0.9277313351631165,
      "logps/weighted_rejected": -1.508001685142517,
      "loss": 0.6648,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -79.80155944824219,
      "rewards/margins": 9.823827743530273,
      "rewards/rejected": -89.63984680175781,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.3523406982421875,
      "rewards/weighted_margins": 0.22584763169288635,
      "rewards/weighted_rejected": -0.5784717798233032,
      "step": 1450
    },
    {
      "epoch": 0.780279243770459,
      "grad_norm": 22.205232620239258,
      "learning_rate": 1.4120959649390136e-07,
      "logits/chosen": -0.06232147291302681,
      "logits/rejected": -0.025969695299863815,
      "logps/chosen": -205.7578125,
      "logps/rejected": -216.8156280517578,
      "logps/weighted_chosen": -1.046392798423767,
      "logps/weighted_rejected": -1.1339843273162842,
      "loss": 0.6717,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -82.03125,
      "rewards/margins": 10.355077743530273,
      "rewards/rejected": -92.41093444824219,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.39169007539749146,
      "rewards/weighted_margins": 0.185791015625,
      "rewards/weighted_rejected": -0.5777603387832642,
      "step": 1460
    },
    {
      "epoch": 0.7856236221524484,
      "grad_norm": 15.942556381225586,
      "learning_rate": 1.347758458250541e-07,
      "logits/chosen": 0.012713623233139515,
      "logits/rejected": 0.04683647304773331,
      "logps/chosen": -203.40469360351562,
      "logps/rejected": -219.640625,
      "logps/weighted_chosen": -1.0474426746368408,
      "logps/weighted_rejected": -1.3139832019805908,
      "loss": 0.6875,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -82.4453125,
      "rewards/margins": 7.944531440734863,
      "rewards/rejected": -90.3812484741211,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.4108734130859375,
      "rewards/weighted_margins": 0.16155776381492615,
      "rewards/weighted_rejected": -0.5725204348564148,
      "step": 1470
    },
    {
      "epoch": 0.7909680005344378,
      "grad_norm": 26.816959381103516,
      "learning_rate": 1.2846920010716266e-07,
      "logits/chosen": 0.014247131533920765,
      "logits/rejected": 0.017230987548828125,
      "logps/chosen": -199.45938110351562,
      "logps/rejected": -212.2609405517578,
      "logps/weighted_chosen": -1.156378149986267,
      "logps/weighted_rejected": -1.0633026361465454,
      "loss": 0.6873,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -82.2640609741211,
      "rewards/margins": 11.641016006469727,
      "rewards/rejected": -93.9296875,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.43985289335250854,
      "rewards/weighted_margins": 0.15669021010398865,
      "rewards/weighted_rejected": -0.5962905883789062,
      "step": 1480
    },
    {
      "epoch": 0.7963123789164273,
      "grad_norm": 9.73423957824707,
      "learning_rate": 1.2229185417286764e-07,
      "logits/chosen": -0.011121368035674095,
      "logits/rejected": 0.0305023193359375,
      "logps/chosen": -208.5437469482422,
      "logps/rejected": -219.5906219482422,
      "logps/weighted_chosen": -1.2618286609649658,
      "logps/weighted_rejected": -1.607025146484375,
      "loss": 0.6811,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -81.4375,
      "rewards/margins": 10.083984375,
      "rewards/rejected": -91.5062484741211,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.3379165530204773,
      "rewards/weighted_margins": 0.1427352875471115,
      "rewards/weighted_rejected": -0.480560302734375,
      "step": 1490
    },
    {
      "epoch": 0.8016567572984167,
      "grad_norm": 25.978179931640625,
      "learning_rate": 1.1624595785602576e-07,
      "logits/chosen": -0.02779388427734375,
      "logits/rejected": 0.00923004187643528,
      "logps/chosen": -207.6750030517578,
      "logps/rejected": -214.234375,
      "logps/weighted_chosen": -1.3065948486328125,
      "logps/weighted_rejected": -1.779363989830017,
      "loss": 0.7042,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -82.6429672241211,
      "rewards/margins": 7.233593940734863,
      "rewards/rejected": -89.8929672241211,
      "rewards/weighted_accuracies": 0.5718749761581421,
      "rewards/weighted_chosen": -0.4565597474575043,
      "rewards/weighted_margins": 0.20427818596363068,
      "rewards/weighted_rejected": -0.660656750202179,
      "step": 1500
    },
    {
      "epoch": 0.8016567572984167,
      "eval_logits/chosen": -0.1534786820411682,
      "eval_logits/rejected": -0.12998200953006744,
      "eval_logps/chosen": -201.64256286621094,
      "eval_logps/rejected": -211.74440002441406,
      "eval_logps/weighted_chosen": -1.0274064540863037,
      "eval_logps/weighted_rejected": -1.3898550271987915,
      "eval_loss": 0.66490238904953,
      "eval_rewards/accuracies": 0.6135438084602356,
      "eval_rewards/chosen": -80.33655548095703,
      "eval_rewards/margins": 8.974350929260254,
      "eval_rewards/rejected": -89.31975555419922,
      "eval_rewards/weighted_accuracies": 0.5651730895042419,
      "eval_rewards/weighted_chosen": -0.38825926184654236,
      "eval_rewards/weighted_margins": 0.20228415727615356,
      "eval_rewards/weighted_rejected": -0.5905433893203735,
      "eval_runtime": 1233.4168,
      "eval_samples_per_second": 1.59,
      "eval_steps_per_second": 0.398,
      "step": 1500
    },
    {
      "epoch": 0.8070011356804062,
      "grad_norm": 8.884445190429688,
      "learning_rate": 1.103336152435274e-07,
      "logits/chosen": 0.0062271119095385075,
      "logits/rejected": 0.03198547288775444,
      "logps/chosen": -200.34375,
      "logps/rejected": -211.6671905517578,
      "logps/weighted_chosen": -0.8016601800918579,
      "logps/weighted_rejected": -1.186651587486267,
      "loss": 0.6617,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -83.3515625,
      "rewards/margins": 8.618749618530273,
      "rewards/rejected": -91.9468765258789,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.372201532125473,
      "rewards/weighted_margins": 0.16056060791015625,
      "rewards/weighted_rejected": -0.5327514410018921,
      "step": 1510
    },
    {
      "epoch": 0.8123455140623956,
      "grad_norm": 7.671918869018555,
      "learning_rate": 1.0455688394303419e-07,
      "logits/chosen": -0.004675674252212048,
      "logits/rejected": 0.05005035549402237,
      "logps/chosen": -204.24063110351562,
      "logps/rejected": -209.0500030517578,
      "logps/weighted_chosen": -1.006256103515625,
      "logps/weighted_rejected": -1.2681457996368408,
      "loss": 0.6866,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -84.80937194824219,
      "rewards/margins": 5.722265720367432,
      "rewards/rejected": -90.5546875,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.41632080078125,
      "rewards/weighted_margins": 0.1294807493686676,
      "rewards/weighted_rejected": -0.5454360842704773,
      "step": 1520
    },
    {
      "epoch": 0.8176898924443851,
      "grad_norm": 7.2523908615112305,
      "learning_rate": 9.891777436689208e-08,
      "logits/chosen": -0.008026123046875,
      "logits/rejected": 0.03105773963034153,
      "logps/chosen": -201.49844360351562,
      "logps/rejected": -214.48593139648438,
      "logps/weighted_chosen": -0.7988342046737671,
      "logps/weighted_rejected": -1.346063256263733,
      "loss": 0.6675,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -81.26875305175781,
      "rewards/margins": 7.553906440734863,
      "rewards/rejected": -88.828125,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": -0.3720565736293793,
      "rewards/weighted_margins": 0.18174858391284943,
      "rewards/weighted_rejected": -0.5538619756698608,
      "step": 1530
    },
    {
      "epoch": 0.8230342708263745,
      "grad_norm": 7.528415679931641,
      "learning_rate": 9.341824903247003e-08,
      "logits/chosen": 0.0003150939883198589,
      "logits/rejected": 0.0193023681640625,
      "logps/chosen": -200.9968719482422,
      "logps/rejected": -206.00936889648438,
      "logps/weighted_chosen": -1.262664794921875,
      "logps/weighted_rejected": -1.222955346107483,
      "loss": 0.7044,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -81.1015625,
      "rewards/margins": 7.956250190734863,
      "rewards/rejected": -89.03437805175781,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.3925842344760895,
      "rewards/weighted_margins": 0.1385902464389801,
      "rewards/weighted_rejected": -0.5313628911972046,
      "step": 1540
    },
    {
      "epoch": 0.8283786492083639,
      "grad_norm": 12.314373970031738,
      "learning_rate": 8.806022187916634e-08,
      "logits/chosen": -0.01698608323931694,
      "logits/rejected": -0.0005397796630859375,
      "logps/chosen": -204.90469360351562,
      "logps/rejected": -220.0968780517578,
      "logps/weighted_chosen": -0.826275646686554,
      "logps/weighted_rejected": -0.7519165277481079,
      "loss": 0.6882,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -81.35234069824219,
      "rewards/margins": 9.787890434265137,
      "rewards/rejected": -91.16328430175781,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.3715835511684418,
      "rewards/weighted_margins": 0.13009873032569885,
      "rewards/weighted_rejected": -0.5020004510879517,
      "step": 1550
    },
    {
      "epoch": 0.8337230275903534,
      "grad_norm": 7.998411178588867,
      "learning_rate": 8.284555760232176e-08,
      "logits/chosen": -0.03365020826458931,
      "logits/rejected": -0.011144256219267845,
      "logps/chosen": -187.03125,
      "logps/rejected": -198.0343780517578,
      "logps/weighted_chosen": -0.8406280279159546,
      "logps/weighted_rejected": -0.948577880859375,
      "loss": 0.6634,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -76.2421875,
      "rewards/margins": 10.604687690734863,
      "rewards/rejected": -86.85624694824219,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.3922592103481293,
      "rewards/weighted_margins": 0.20080260932445526,
      "rewards/weighted_rejected": -0.5934112668037415,
      "step": 1560
    },
    {
      "epoch": 0.8390674059723429,
      "grad_norm": 8.670356750488281,
      "learning_rate": 7.777607100426925e-08,
      "logits/chosen": -0.054158784449100494,
      "logits/rejected": -0.03361816331744194,
      "logps/chosen": -207.2312469482422,
      "logps/rejected": -216.125,
      "logps/weighted_chosen": -1.2455352544784546,
      "logps/weighted_rejected": -1.077966332435608,
      "loss": 0.6691,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -82.80937194824219,
      "rewards/margins": 9.444531440734863,
      "rewards/rejected": -92.26094055175781,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": -0.35657501220703125,
      "rewards/weighted_margins": 0.17081375420093536,
      "rewards/weighted_rejected": -0.5276878476142883,
      "step": 1570
    },
    {
      "epoch": 0.8444117843543323,
      "grad_norm": 12.28963565826416,
      "learning_rate": 7.285352636274922e-08,
      "logits/chosen": 0.01006469689309597,
      "logits/rejected": 0.03425750881433487,
      "logps/chosen": -195.58749389648438,
      "logps/rejected": -205.8781280517578,
      "logps/weighted_chosen": -1.3043944835662842,
      "logps/weighted_rejected": -1.3767731189727783,
      "loss": 0.7349,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -77.5609359741211,
      "rewards/margins": 10.790624618530273,
      "rewards/rejected": -88.34062194824219,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.4487808346748352,
      "rewards/weighted_margins": 0.135456845164299,
      "rewards/weighted_rejected": -0.5850158929824829,
      "step": 1580
    },
    {
      "epoch": 0.8497561627363217,
      "grad_norm": 12.43053913116455,
      "learning_rate": 6.80796368169067e-08,
      "logits/chosen": -0.03640174865722656,
      "logits/rejected": 0.024863053113222122,
      "logps/chosen": -209.8828125,
      "logps/rejected": -222.59219360351562,
      "logps/weighted_chosen": -0.916961669921875,
      "logps/weighted_rejected": -1.4554595947265625,
      "loss": 0.6726,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -80.7679672241211,
      "rewards/margins": 12.613672256469727,
      "rewards/rejected": -93.3812484741211,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": -0.3256774842739105,
      "rewards/weighted_margins": 0.18812866508960724,
      "rewards/weighted_rejected": -0.5140304565429688,
      "step": 1590
    },
    {
      "epoch": 0.8551005411183111,
      "grad_norm": 7.3841352462768555,
      "learning_rate": 6.345606377108658e-08,
      "logits/chosen": 0.04971466213464737,
      "logits/rejected": 0.09067229926586151,
      "logps/chosen": -199.7687530517578,
      "logps/rejected": -215.95938110351562,
      "logps/weighted_chosen": -0.9194701910018921,
      "logps/weighted_rejected": -1.5334380865097046,
      "loss": 0.6658,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -81.9398422241211,
      "rewards/margins": 11.660547256469727,
      "rewards/rejected": -93.58906555175781,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.3771987855434418,
      "rewards/weighted_margins": 0.18507537245750427,
      "rewards/weighted_rejected": -0.5623397827148438,
      "step": 1600
    },
    {
      "epoch": 0.8604449195003007,
      "grad_norm": 6.630573749542236,
      "learning_rate": 5.89844163166332e-08,
      "logits/chosen": -0.0077033997513353825,
      "logits/rejected": 0.0152740478515625,
      "logps/chosen": -206.56094360351562,
      "logps/rejected": -213.8093719482422,
      "logps/weighted_chosen": -1.148492455482483,
      "logps/weighted_rejected": -1.014794945716858,
      "loss": 0.6821,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -82.2054672241211,
      "rewards/margins": 8.006640434265137,
      "rewards/rejected": -90.23515319824219,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": -0.3819526731967926,
      "rewards/weighted_margins": 0.1473899781703949,
      "rewards/weighted_rejected": -0.529217541217804,
      "step": 1610
    },
    {
      "epoch": 0.8657892978822901,
      "grad_norm": 9.12763500213623,
      "learning_rate": 5.466625067189601e-08,
      "logits/chosen": 0.0029403686057776213,
      "logits/rejected": 0.038446806371212006,
      "logps/chosen": -195.69686889648438,
      "logps/rejected": -205.2781219482422,
      "logps/weighted_chosen": -0.8219360113143921,
      "logps/weighted_rejected": -1.164337158203125,
      "loss": 0.6719,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -79.7906265258789,
      "rewards/margins": 9.067968368530273,
      "rewards/rejected": -88.85469055175781,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.3714141845703125,
      "rewards/weighted_margins": 0.172444149851799,
      "rewards/weighted_rejected": -0.5441833734512329,
      "step": 1620
    },
    {
      "epoch": 0.8711336762642795,
      "grad_norm": 31.594406127929688,
      "learning_rate": 5.0503069640635944e-08,
      "logits/chosen": -0.013871383853256702,
      "logits/rejected": 0.03087768517434597,
      "logps/chosen": -203.55313110351562,
      "logps/rejected": -214.140625,
      "logps/weighted_chosen": -1.257110595703125,
      "logps/weighted_rejected": -1.71246337890625,
      "loss": 0.6744,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -82.45625305175781,
      "rewards/margins": 9.573046684265137,
      "rewards/rejected": -92.05781555175781,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.33105164766311646,
      "rewards/weighted_margins": 0.23786774277687073,
      "rewards/weighted_rejected": -0.5689346194267273,
      "step": 1630
    },
    {
      "epoch": 0.8764780546462689,
      "grad_norm": 6.86949348449707,
      "learning_rate": 4.649632208902127e-08,
      "logits/chosen": -0.06173248216509819,
      "logits/rejected": -0.0015998840099200606,
      "logps/chosen": -193.95938110351562,
      "logps/rejected": -206.36563110351562,
      "logps/weighted_chosen": -0.985150158405304,
      "logps/weighted_rejected": -1.1464660167694092,
      "loss": 0.6605,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -76.65937805175781,
      "rewards/margins": 9.346484184265137,
      "rewards/rejected": -86.00468444824219,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": -0.3394966125488281,
      "rewards/weighted_margins": 0.15953750908374786,
      "rewards/weighted_rejected": -0.4988296627998352,
      "step": 1640
    },
    {
      "epoch": 0.8818224330282584,
      "grad_norm": 8.658909797668457,
      "learning_rate": 4.2647402441394534e-08,
      "logits/chosen": -0.00011138916306663305,
      "logits/rejected": 0.032086946070194244,
      "logps/chosen": -195.4421844482422,
      "logps/rejected": -208.1671905517578,
      "logps/weighted_chosen": -1.0746338367462158,
      "logps/weighted_rejected": -1.328485131263733,
      "loss": 0.6769,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -81.19999694824219,
      "rewards/margins": 7.428124904632568,
      "rewards/rejected": -88.65937805175781,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.42961883544921875,
      "rewards/weighted_margins": 0.11412658542394638,
      "rewards/weighted_rejected": -0.5438331365585327,
      "step": 1650
    },
    {
      "epoch": 0.8871668114102479,
      "grad_norm": 8.569114685058594,
      "learning_rate": 3.895765019498609e-08,
      "logits/chosen": 0.002704620361328125,
      "logits/rejected": 0.06543274223804474,
      "logps/chosen": -200.77499389648438,
      "logps/rejected": -209.3781280517578,
      "logps/weighted_chosen": -1.202539086341858,
      "logps/weighted_rejected": -1.7360382080078125,
      "loss": 0.6903,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -81.5859375,
      "rewards/margins": 7.512890815734863,
      "rewards/rejected": -89.10624694824219,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.38261109590530396,
      "rewards/weighted_margins": 0.17353057861328125,
      "rewards/weighted_rejected": -0.5557746887207031,
      "step": 1660
    },
    {
      "epoch": 0.8925111897922373,
      "grad_norm": 8.67790412902832,
      "learning_rate": 3.542834945374412e-08,
      "logits/chosen": 0.02303619310259819,
      "logits/rejected": 0.04090576246380806,
      "logps/chosen": -198.5749969482422,
      "logps/rejected": -209.38436889648438,
      "logps/weighted_chosen": -0.764483630657196,
      "logps/weighted_rejected": -1.1029876470565796,
      "loss": 0.6775,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -79.39921569824219,
      "rewards/margins": 7.441015720367432,
      "rewards/rejected": -86.8218765258789,
      "rewards/weighted_accuracies": 0.5093749761581421,
      "rewards/weighted_chosen": -0.3830154538154602,
      "rewards/weighted_margins": 0.15334168076515198,
      "rewards/weighted_rejected": -0.5364456176757812,
      "step": 1670
    },
    {
      "epoch": 0.8978555681742267,
      "grad_norm": 21.397300720214844,
      "learning_rate": 3.206072848144176e-08,
      "logits/chosen": -0.00791015662252903,
      "logits/rejected": 0.04454975202679634,
      "logps/chosen": -196.8625030517578,
      "logps/rejected": -209.47030639648438,
      "logps/weighted_chosen": -1.026220679283142,
      "logps/weighted_rejected": -1.4850037097930908,
      "loss": 0.6715,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -79.52656555175781,
      "rewards/margins": 9.549219131469727,
      "rewards/rejected": -89.0843734741211,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.3757896423339844,
      "rewards/weighted_margins": 0.19904938340187073,
      "rewards/weighted_rejected": -0.5747489929199219,
      "step": 1680
    },
    {
      "epoch": 0.9031999465562162,
      "grad_norm": 7.6560378074646,
      "learning_rate": 2.8855959274217802e-08,
      "logits/chosen": 0.011219787411391735,
      "logits/rejected": 0.015446471981704235,
      "logps/chosen": -192.77188110351562,
      "logps/rejected": -202.75,
      "logps/weighted_chosen": -1.6045806407928467,
      "logps/weighted_rejected": -1.1945679187774658,
      "loss": 0.6618,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -78.68046569824219,
      "rewards/margins": 10.335546493530273,
      "rewards/rejected": -89.0546875,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.3234237730503082,
      "rewards/weighted_margins": 0.20303955674171448,
      "rewards/weighted_rejected": -0.5267227292060852,
      "step": 1690
    },
    {
      "epoch": 0.9085443249382056,
      "grad_norm": 7.867788314819336,
      "learning_rate": 2.5815157152700106e-08,
      "logits/chosen": -0.0039764405228197575,
      "logits/rejected": 0.052965544164180756,
      "logps/chosen": -198.703125,
      "logps/rejected": -227.3562469482422,
      "logps/weighted_chosen": -1.1858246326446533,
      "logps/weighted_rejected": -1.7776000499725342,
      "loss": 0.6681,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -81.5078125,
      "rewards/margins": 25.093358993530273,
      "rewards/rejected": -106.625,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.3344528079032898,
      "rewards/weighted_margins": 0.23516464233398438,
      "rewards/weighted_rejected": -0.5694076418876648,
      "step": 1700
    },
    {
      "epoch": 0.9138887033201951,
      "grad_norm": 31.351016998291016,
      "learning_rate": 2.2939380373852513e-08,
      "logits/chosen": 0.04047546535730362,
      "logits/rejected": 0.057331085205078125,
      "logps/chosen": -192.50155639648438,
      "logps/rejected": -199.328125,
      "logps/weighted_chosen": -1.187780737876892,
      "logps/weighted_rejected": -1.1523864269256592,
      "loss": 0.7191,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -80.76875305175781,
      "rewards/margins": 6.9296875,
      "rewards/rejected": -87.68281555175781,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.4314765930175781,
      "rewards/weighted_margins": 0.061667632311582565,
      "rewards/weighted_rejected": -0.4930679202079773,
      "step": 1710
    },
    {
      "epoch": 0.9192330817021845,
      "grad_norm": 12.911133766174316,
      "learning_rate": 2.0229629762681334e-08,
      "logits/chosen": -0.0029060363303869963,
      "logits/rejected": 0.054146576672792435,
      "logps/chosen": -202.1999969482422,
      "logps/rejected": -203.1953125,
      "logps/weighted_chosen": -1.305535912513733,
      "logps/weighted_rejected": -1.5592772960662842,
      "loss": 0.6688,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -80.04765319824219,
      "rewards/margins": 6.839453220367432,
      "rewards/rejected": -86.9046859741211,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.3944839537143707,
      "rewards/weighted_margins": 0.15517273545265198,
      "rewards/weighted_rejected": -0.5497649908065796,
      "step": 1720
    },
    {
      "epoch": 0.924577460084174,
      "grad_norm": 7.706750869750977,
      "learning_rate": 1.7686848363929363e-08,
      "logits/chosen": -0.020279694348573685,
      "logits/rejected": 0.029271889477968216,
      "logps/chosen": -200.734375,
      "logps/rejected": -208.2390594482422,
      "logps/weighted_chosen": -0.884796142578125,
      "logps/weighted_rejected": -1.5206177234649658,
      "loss": 0.6686,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -79.7640609741211,
      "rewards/margins": 7.583203315734863,
      "rewards/rejected": -87.37187194824219,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.4030723571777344,
      "rewards/weighted_margins": 0.17832335829734802,
      "rewards/weighted_rejected": -0.5815140008926392,
      "step": 1730
    },
    {
      "epoch": 0.9299218384661634,
      "grad_norm": 8.226058006286621,
      "learning_rate": 1.5311921113878355e-08,
      "logits/chosen": -0.0008687973022460938,
      "logits/rejected": 0.028296660631895065,
      "logps/chosen": -194.00155639648438,
      "logps/rejected": -203.703125,
      "logps/weighted_chosen": -1.365270972251892,
      "logps/weighted_rejected": -1.539056420326233,
      "loss": 0.6705,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -77.375,
      "rewards/margins": 8.365625381469727,
      "rewards/rejected": -85.73750305175781,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.45653611421585083,
      "rewards/weighted_margins": 0.19713135063648224,
      "rewards/weighted_rejected": -0.6534255743026733,
      "step": 1740
    },
    {
      "epoch": 0.9352662168481528,
      "grad_norm": 21.11993408203125,
      "learning_rate": 1.3105674532374212e-08,
      "logits/chosen": 0.022278213873505592,
      "logits/rejected": 0.06085510179400444,
      "logps/chosen": -189.58438110351562,
      "logps/rejected": -204.9343719482422,
      "logps/weighted_chosen": -0.7065078616142273,
      "logps/weighted_rejected": -1.562200903892517,
      "loss": 0.6816,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -78.2125015258789,
      "rewards/margins": 9.004687309265137,
      "rewards/rejected": -87.2289047241211,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.3986854553222656,
      "rewards/weighted_margins": 0.12375183403491974,
      "rewards/weighted_rejected": -0.5223197937011719,
      "step": 1750
    },
    {
      "epoch": 0.9406105952301423,
      "grad_norm": 7.333647727966309,
      "learning_rate": 1.106887643518256e-08,
      "logits/chosen": -0.011242675594985485,
      "logits/rejected": -0.014645385555922985,
      "logps/chosen": -207.7468719482422,
      "logps/rejected": -222.8937530517578,
      "logps/weighted_chosen": -1.1957489252090454,
      "logps/weighted_rejected": -1.2758667469024658,
      "loss": 0.6751,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -82.3929672241211,
      "rewards/margins": 12.168749809265137,
      "rewards/rejected": -94.54219055175781,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.3890426754951477,
      "rewards/weighted_margins": 0.12043914943933487,
      "rewards/weighted_rejected": -0.5094535946846008,
      "step": 1760
    },
    {
      "epoch": 0.9459549736121318,
      "grad_norm": 6.959185600280762,
      "learning_rate": 9.202235666774027e-09,
      "logits/chosen": -0.015836333855986595,
      "logits/rejected": -0.0019592284224927425,
      "logps/chosen": -208.76406860351562,
      "logps/rejected": -219.86874389648438,
      "logps/weighted_chosen": -1.0631835460662842,
      "logps/weighted_rejected": -1.0057373046875,
      "loss": 0.704,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -85.24140930175781,
      "rewards/margins": 8.126562118530273,
      "rewards/rejected": -93.36405944824219,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.43046340346336365,
      "rewards/weighted_margins": 0.13985595107078552,
      "rewards/weighted_rejected": -0.5703773498535156,
      "step": 1770
    },
    {
      "epoch": 0.9512993519941212,
      "grad_norm": 18.106613159179688,
      "learning_rate": 7.50640185363305e-09,
      "logits/chosen": 0.01944122277200222,
      "logits/rejected": 0.0510406494140625,
      "logps/chosen": -201.4640655517578,
      "logps/rejected": -213.0968780517578,
      "logps/weighted_chosen": -1.5302276611328125,
      "logps/weighted_rejected": -1.740319848060608,
      "loss": 0.699,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -82.03047180175781,
      "rewards/margins": 7.597265720367432,
      "rewards/rejected": -89.6078109741211,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.37673646211624146,
      "rewards/weighted_margins": 0.13721999526023865,
      "rewards/weighted_rejected": -0.5135101079940796,
      "step": 1780
    },
    {
      "epoch": 0.9566437303761106,
      "grad_norm": 11.966142654418945,
      "learning_rate": 5.981965178175563e-09,
      "logits/chosen": -0.07247161865234375,
      "logits/rejected": -0.02783966064453125,
      "logps/chosen": -205.18124389648438,
      "logps/rejected": -219.86874389648438,
      "logps/weighted_chosen": -0.6668335199356079,
      "logps/weighted_rejected": -1.101470947265625,
      "loss": 0.6685,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -85.0562515258789,
      "rewards/margins": 10.532031059265137,
      "rewards/rejected": -95.59062194824219,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.32790833711624146,
      "rewards/weighted_margins": 0.19003143906593323,
      "rewards/weighted_rejected": -0.517895519733429,
      "step": 1790
    },
    {
      "epoch": 0.9619881087581,
      "grad_norm": 9.166258811950684,
      "learning_rate": 4.629456173354196e-09,
      "logits/chosen": 0.03516960144042969,
      "logits/rejected": 0.053380586206912994,
      "logps/chosen": -197.13436889648438,
      "logps/rejected": -208.0656280517578,
      "logps/weighted_chosen": -1.8832824230194092,
      "logps/weighted_rejected": -1.6900818347930908,
      "loss": 0.6711,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -79.5218734741211,
      "rewards/margins": 8.657422065734863,
      "rewards/rejected": -88.18281555175781,
      "rewards/weighted_accuracies": 0.609375,
      "rewards/weighted_chosen": -0.3943443298339844,
      "rewards/weighted_margins": 0.23465195298194885,
      "rewards/weighted_rejected": -0.6290283203125,
      "step": 1800
    },
    {
      "epoch": 0.9673324871400896,
      "grad_norm": 7.5731706619262695,
      "learning_rate": 3.4493455380225634e-09,
      "logits/chosen": -0.02142333984375,
      "logits/rejected": 0.01344223041087389,
      "logps/chosen": -208.6281280517578,
      "logps/rejected": -217.1765594482422,
      "logps/weighted_chosen": -0.970935046672821,
      "logps/weighted_rejected": -1.060217261314392,
      "loss": 0.6414,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -80.3062515258789,
      "rewards/margins": 11.758593559265137,
      "rewards/rejected": -92.08125305175781,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.3379959166049957,
      "rewards/weighted_margins": 0.2506675720214844,
      "rewards/weighted_rejected": -0.5888374447822571,
      "step": 1810
    },
    {
      "epoch": 0.972676865522079,
      "grad_norm": 10.392830848693848,
      "learning_rate": 2.4420439731234754e-09,
      "logits/chosen": -0.02555542066693306,
      "logits/rejected": 0.033538818359375,
      "logps/chosen": -211.2375030517578,
      "logps/rejected": -217.2218780517578,
      "logps/weighted_chosen": -1.076715111732483,
      "logps/weighted_rejected": -1.1117675304412842,
      "loss": 0.658,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -86.1390609741211,
      "rewards/margins": 7.975390434265137,
      "rewards/rejected": -94.12968444824219,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": -0.387350469827652,
      "rewards/weighted_margins": 0.1678314208984375,
      "rewards/weighted_rejected": -0.5549789667129517,
      "step": 1820
    },
    {
      "epoch": 0.9780212439040684,
      "grad_norm": 14.731633186340332,
      "learning_rate": 1.607902038756881e-09,
      "logits/chosen": -0.03051147423684597,
      "logits/rejected": 0.03038787841796875,
      "logps/chosen": -201.2140655517578,
      "logps/rejected": -211.74063110351562,
      "logps/weighted_chosen": -1.221441626548767,
      "logps/weighted_rejected": -1.850244164466858,
      "loss": 0.698,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -82.11015319824219,
      "rewards/margins": 7.809765815734863,
      "rewards/rejected": -89.9296875,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.4083351194858551,
      "rewards/weighted_margins": 0.17291259765625,
      "rewards/weighted_rejected": -0.581036388874054,
      "step": 1830
    },
    {
      "epoch": 0.9833656222860578,
      "grad_norm": 16.210765838623047,
      "learning_rate": 9.472100321785137e-10,
      "logits/chosen": 0.01699371263384819,
      "logits/rejected": 0.042603302747011185,
      "logps/chosen": -193.3234405517578,
      "logps/rejected": -205.1609344482422,
      "logps/weighted_chosen": -1.5597137212753296,
      "logps/weighted_rejected": -1.877203345298767,
      "loss": 0.6571,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -80.43437194824219,
      "rewards/margins": 10.099609375,
      "rewards/rejected": -90.53593444824219,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.4782699644565582,
      "rewards/weighted_margins": 0.19840240478515625,
      "rewards/weighted_rejected": -0.6767608523368835,
      "step": 1840
    },
    {
      "epoch": 0.9887100006680473,
      "grad_norm": 15.271677017211914,
      "learning_rate": 4.601978867705969e-10,
      "logits/chosen": 0.00860519427806139,
      "logits/rejected": 0.059200286865234375,
      "logps/chosen": -193.88436889648438,
      "logps/rejected": -201.98593139648438,
      "logps/weighted_chosen": -1.474829077720642,
      "logps/weighted_rejected": -1.616754174232483,
      "loss": 0.6632,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -78.921875,
      "rewards/margins": 7.835546970367432,
      "rewards/rejected": -86.7671890258789,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.33439406752586365,
      "rewards/weighted_margins": 0.28738290071487427,
      "rewards/weighted_rejected": -0.6216728091239929,
      "step": 1850
    },
    {
      "epoch": 0.9940543790500368,
      "grad_norm": 37.25553512573242,
      "learning_rate": 1.4703509202118604e-10,
      "logits/chosen": -0.05092964321374893,
      "logits/rejected": -0.014559936709702015,
      "logps/chosen": -205.4250030517578,
      "logps/rejected": -215.70938110351562,
      "logps/weighted_chosen": -0.953631579875946,
      "logps/weighted_rejected": -1.1918243169784546,
      "loss": 0.697,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -83.56562805175781,
      "rewards/margins": 7.594531059265137,
      "rewards/rejected": -91.1812515258789,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.3130737245082855,
      "rewards/weighted_margins": 0.1581375151872635,
      "rewards/weighted_rejected": -0.47164687514305115,
      "step": 1860
    },
    {
      "epoch": 0.9993987574320262,
      "grad_norm": 7.526350021362305,
      "learning_rate": 7.830634538019686e-12,
      "logits/chosen": -0.017978286370635033,
      "logits/rejected": 0.02026519738137722,
      "logps/chosen": -205.7078094482422,
      "logps/rejected": -213.390625,
      "logps/weighted_chosen": -1.1760375499725342,
      "logps/weighted_rejected": -1.36285400390625,
      "loss": 0.6788,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -81.91874694824219,
      "rewards/margins": 8.227734565734863,
      "rewards/rejected": -90.15156555175781,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.3904052674770355,
      "rewards/weighted_margins": 0.1767173707485199,
      "rewards/weighted_rejected": -0.5669296383857727,
      "step": 1870
    }
  ],
  "logging_steps": 10,
  "max_steps": 1872,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}