{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5344378381989445,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005344378381989445,
      "grad_norm": 63.52920150756836,
      "learning_rate": 0.0,
      "logits/chosen": -0.144775390625,
      "logits/rejected": -0.1036376953125,
      "logps/chosen": -110.828125,
      "logps/rejected": -116.828125,
      "logps/weighted_chosen": -0.2086181640625,
      "logps/weighted_rejected": -1.271392822265625,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/weighted_accuracies": 0.0,
      "rewards/weighted_chosen": 0.0,
      "rewards/weighted_margins": 0.0,
      "rewards/weighted_rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005344378381989445,
      "grad_norm": 102.77810668945312,
      "learning_rate": 4.787234042553191e-08,
      "logits/chosen": -0.2549913227558136,
      "logits/rejected": -0.24070316553115845,
      "logps/chosen": -135.32899475097656,
      "logps/rejected": -138.99305725097656,
      "logps/weighted_chosen": -0.8547499179840088,
      "logps/weighted_rejected": -0.9027608036994934,
      "loss": 0.6928,
      "rewards/accuracies": 0.2951388955116272,
      "rewards/chosen": -0.0164930559694767,
      "rewards/margins": -0.0349392369389534,
      "rewards/rejected": 0.0184461809694767,
      "rewards/weighted_accuracies": 0.4027777910232544,
      "rewards/weighted_chosen": -0.002239651046693325,
      "rewards/weighted_margins": 0.0028160943184047937,
      "rewards/weighted_rejected": -0.0050794812850654125,
      "step": 10
    },
    {
      "epoch": 0.01068875676397889,
      "grad_norm": 9.201544761657715,
      "learning_rate": 1.0106382978723404e-07,
      "logits/chosen": -0.19488906860351562,
      "logits/rejected": -0.183399960398674,
      "logps/chosen": -116.0374984741211,
      "logps/rejected": -115.5054702758789,
      "logps/weighted_chosen": -0.6929916143417358,
      "logps/weighted_rejected": -0.6838119626045227,
      "loss": 0.6917,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.01796874962747097,
      "rewards/margins": 0.03007812425494194,
      "rewards/rejected": -0.01210937462747097,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": 0.0022293091751635075,
      "rewards/weighted_margins": 0.00339679722674191,
      "rewards/weighted_rejected": -0.0011629104847088456,
      "step": 20
    },
    {
      "epoch": 0.016033135145968335,
      "grad_norm": 7.109442234039307,
      "learning_rate": 1.5425531914893615e-07,
      "logits/chosen": -0.1776607483625412,
      "logits/rejected": -0.17884215712547302,
      "logps/chosen": -123.2867202758789,
      "logps/rejected": -129.75,
      "logps/weighted_chosen": -0.7567901611328125,
      "logps/weighted_rejected": -1.2207733392715454,
      "loss": 0.6913,
      "rewards/accuracies": 0.3968749940395355,
      "rewards/chosen": -0.0074218749068677425,
      "rewards/margins": 0.04843749850988388,
      "rewards/rejected": -0.05585937574505806,
      "rewards/weighted_accuracies": 0.5,
      "rewards/weighted_chosen": 0.0007101058727130294,
      "rewards/weighted_margins": 0.004384422209113836,
      "rewards/weighted_rejected": -0.0036745071411132812,
      "step": 30
    },
    {
      "epoch": 0.02137751352795778,
      "grad_norm": 6.0048956871032715,
      "learning_rate": 2.074468085106383e-07,
      "logits/chosen": -0.23842163383960724,
      "logits/rejected": -0.2298881560564041,
      "logps/chosen": -127.34062194824219,
      "logps/rejected": -128.1710968017578,
      "logps/weighted_chosen": -0.8051223754882812,
      "logps/weighted_rejected": -0.5736129879951477,
      "loss": 0.6934,
      "rewards/accuracies": 0.34062498807907104,
      "rewards/chosen": -0.01992187462747097,
      "rewards/margins": -0.05351562425494194,
      "rewards/rejected": 0.03359375149011612,
      "rewards/weighted_accuracies": 0.4593749940395355,
      "rewards/weighted_chosen": -7.83920258982107e-05,
      "rewards/weighted_margins": -0.0009832382202148438,
      "rewards/weighted_rejected": 0.000904083251953125,
      "step": 40
    },
    {
      "epoch": 0.026721891909947223,
      "grad_norm": 32.7233772277832,
      "learning_rate": 2.6063829787234044e-07,
      "logits/chosen": -0.18835143744945526,
      "logits/rejected": -0.170085147023201,
      "logps/chosen": -120.88594055175781,
      "logps/rejected": -121.3515625,
      "logps/weighted_chosen": -0.6867736577987671,
      "logps/weighted_rejected": -1.138249158859253,
      "loss": 0.6903,
      "rewards/accuracies": 0.3656249940395355,
      "rewards/chosen": 0.03554687649011612,
      "rewards/margins": -0.03164062649011612,
      "rewards/rejected": 0.06718750298023224,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.006287765689194202,
      "rewards/weighted_margins": 0.0064636231400072575,
      "rewards/weighted_rejected": -0.01274185162037611,
      "step": 50
    },
    {
      "epoch": 0.03206627029193667,
      "grad_norm": 55.560707092285156,
      "learning_rate": 3.1382978723404253e-07,
      "logits/chosen": -0.1308601349592209,
      "logits/rejected": -0.1308906525373459,
      "logps/chosen": -116.5875015258789,
      "logps/rejected": -119.5406265258789,
      "logps/weighted_chosen": -0.8019469976425171,
      "logps/weighted_rejected": -0.8834930658340454,
      "loss": 0.7002,
      "rewards/accuracies": 0.3343749940395355,
      "rewards/chosen": 0.03476562350988388,
      "rewards/margins": -0.06171875074505806,
      "rewards/rejected": 0.09648437798023224,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.01903533935546875,
      "rewards/weighted_margins": -2.880096508306451e-05,
      "rewards/weighted_rejected": -0.019007300958037376,
      "step": 60
    },
    {
      "epoch": 0.037410648673926114,
      "grad_norm": 20.22484588623047,
      "learning_rate": 3.6702127659574467e-07,
      "logits/chosen": -0.20572929084300995,
      "logits/rejected": -0.19457244873046875,
      "logps/chosen": -114.4507827758789,
      "logps/rejected": -118.63594055175781,
      "logps/weighted_chosen": -0.6070801019668579,
      "logps/weighted_rejected": -0.6768280267715454,
      "loss": 0.6911,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.16484375298023224,
      "rewards/margins": 0.07148437201976776,
      "rewards/rejected": 0.09335937350988388,
      "rewards/weighted_accuracies": 0.4749999940395355,
      "rewards/weighted_chosen": -0.010281562805175781,
      "rewards/weighted_margins": 0.007259177975356579,
      "rewards/weighted_rejected": -0.017521286383271217,
      "step": 70
    },
    {
      "epoch": 0.04275502705591556,
      "grad_norm": 15.305061340332031,
      "learning_rate": 4.202127659574468e-07,
      "logits/chosen": -0.19508513808250427,
      "logits/rejected": -0.12093047797679901,
      "logps/chosen": -112.57734680175781,
      "logps/rejected": -112.77734375,
      "logps/weighted_chosen": -0.4597320556640625,
      "logps/weighted_rejected": -0.9711883664131165,
      "loss": 0.6952,
      "rewards/accuracies": 0.37812501192092896,
      "rewards/chosen": 0.04570312425494194,
      "rewards/margins": 0.01914062537252903,
      "rewards/rejected": 0.02656250074505806,
      "rewards/weighted_accuracies": 0.4625000059604645,
      "rewards/weighted_chosen": -0.02759857103228569,
      "rewards/weighted_margins": 0.021048355847597122,
      "rewards/weighted_rejected": -0.04872245714068413,
      "step": 80
    },
    {
      "epoch": 0.048099405437905,
      "grad_norm": 11.991273880004883,
      "learning_rate": 4.734042553191489e-07,
      "logits/chosen": -0.17391815781593323,
      "logits/rejected": -0.11656417697668076,
      "logps/chosen": -119.3070297241211,
      "logps/rejected": -118.51249694824219,
      "logps/weighted_chosen": -1.0416854619979858,
      "logps/weighted_rejected": -1.5859466791152954,
      "loss": 0.6736,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08164062350988388,
      "rewards/margins": 0.341796875,
      "rewards/rejected": -0.4234375059604645,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.04084587097167969,
      "rewards/weighted_margins": 0.10567589104175568,
      "rewards/weighted_rejected": -0.14661255478858948,
      "step": 90
    },
    {
      "epoch": 0.053443783819894446,
      "grad_norm": 17.195886611938477,
      "learning_rate": 5.26595744680851e-07,
      "logits/chosen": -0.204203799366951,
      "logits/rejected": -0.14691162109375,
      "logps/chosen": -125.05155944824219,
      "logps/rejected": -125.3671875,
      "logps/weighted_chosen": -0.7374237179756165,
      "logps/weighted_rejected": -0.7348953485488892,
      "loss": 0.7064,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.3832031190395355,
      "rewards/margins": 0.15585938096046448,
      "rewards/rejected": -0.5390625,
      "rewards/weighted_accuracies": 0.46875,
      "rewards/weighted_chosen": -0.060018159449100494,
      "rewards/weighted_margins": 0.0024120330344885588,
      "rewards/weighted_rejected": -0.06242714077234268,
      "step": 100
    },
    {
      "epoch": 0.058788162201883896,
      "grad_norm": 9.103920936584473,
      "learning_rate": 5.797872340425531e-07,
      "logits/chosen": -0.2138931304216385,
      "logits/rejected": -0.19011077284812927,
      "logps/chosen": -118.6820297241211,
      "logps/rejected": -121.28593444824219,
      "logps/weighted_chosen": -0.4821228086948395,
      "logps/weighted_rejected": -0.7759445309638977,
      "loss": 0.6939,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.896484375,
      "rewards/margins": 0.09531249850988388,
      "rewards/rejected": -0.9917968511581421,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.05413341522216797,
      "rewards/weighted_margins": 0.04503212124109268,
      "rewards/weighted_rejected": -0.09918193519115448,
      "step": 110
    },
    {
      "epoch": 0.06413254058387334,
      "grad_norm": 8.539896011352539,
      "learning_rate": 6.329787234042553e-07,
      "logits/chosen": -0.17510680854320526,
      "logits/rejected": -0.14726562798023224,
      "logps/chosen": -122.58906555175781,
      "logps/rejected": -128.4656219482422,
      "logps/weighted_chosen": -0.9249496459960938,
      "logps/weighted_rejected": -0.762683093547821,
      "loss": 0.7325,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.068359375,
      "rewards/margins": 0.28398436307907104,
      "rewards/rejected": -1.3523437976837158,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.098027803003788,
      "rewards/weighted_margins": -0.029563140124082565,
      "rewards/weighted_rejected": -0.06836700439453125,
      "step": 120
    },
    {
      "epoch": 0.06947691896586278,
      "grad_norm": 15.564044952392578,
      "learning_rate": 6.861702127659574e-07,
      "logits/chosen": -0.20233002305030823,
      "logits/rejected": -0.18396148085594177,
      "logps/chosen": -122.2671890258789,
      "logps/rejected": -123.8382797241211,
      "logps/weighted_chosen": -0.7501281499862671,
      "logps/weighted_rejected": -0.821154773235321,
      "loss": 0.7078,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -1.45703125,
      "rewards/margins": 0.29179686307907104,
      "rewards/rejected": -1.7488281726837158,
      "rewards/weighted_accuracies": 0.5062500238418579,
      "rewards/weighted_chosen": -0.007884788326919079,
      "rewards/weighted_margins": 0.006314086727797985,
      "rewards/weighted_rejected": -0.014200759120285511,
      "step": 130
    },
    {
      "epoch": 0.07482129734785223,
      "grad_norm": 9.176558494567871,
      "learning_rate": 7.393617021276596e-07,
      "logits/chosen": -0.16455383598804474,
      "logits/rejected": -0.14660033583641052,
      "logps/chosen": -116.328125,
      "logps/rejected": -119.9664077758789,
      "logps/weighted_chosen": -0.9873703122138977,
      "logps/weighted_rejected": -0.6730011105537415,
      "loss": 0.7174,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.6007812023162842,
      "rewards/margins": 0.533203125,
      "rewards/rejected": -2.133984327316284,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": 0.02084655687212944,
      "rewards/weighted_margins": 0.05102996900677681,
      "rewards/weighted_rejected": -0.030106354504823685,
      "step": 140
    },
    {
      "epoch": 0.08016567572984168,
      "grad_norm": 10.172534942626953,
      "learning_rate": 7.925531914893616e-07,
      "logits/chosen": -0.1509498655796051,
      "logits/rejected": -0.13156357407569885,
      "logps/chosen": -120.0562515258789,
      "logps/rejected": -117.85546875,
      "logps/weighted_chosen": -0.8198806643486023,
      "logps/weighted_rejected": -0.7782562375068665,
      "loss": 0.7005,
      "rewards/accuracies": 0.46562498807907104,
      "rewards/chosen": -2.0234375,
      "rewards/margins": 0.607421875,
      "rewards/rejected": -2.630859375,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.0781441181898117,
      "rewards/weighted_margins": 0.034273337572813034,
      "rewards/weighted_rejected": -0.11248989403247833,
      "step": 150
    },
    {
      "epoch": 0.08551005411183112,
      "grad_norm": 25.879701614379883,
      "learning_rate": 8.457446808510637e-07,
      "logits/chosen": -0.17474937438964844,
      "logits/rejected": -0.17009583115577698,
      "logps/chosen": -121.2367172241211,
      "logps/rejected": -121.62109375,
      "logps/weighted_chosen": -0.8304061889648438,
      "logps/weighted_rejected": -0.866925060749054,
      "loss": 0.7072,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -2.219531297683716,
      "rewards/margins": 0.716796875,
      "rewards/rejected": -2.936328172683716,
      "rewards/weighted_accuracies": 0.47187501192092896,
      "rewards/weighted_chosen": -0.07932891696691513,
      "rewards/weighted_margins": 0.017035294324159622,
      "rewards/weighted_rejected": -0.09644661098718643,
      "step": 160
    },
    {
      "epoch": 0.09085443249382057,
      "grad_norm": 9.220558166503906,
      "learning_rate": 8.989361702127659e-07,
      "logits/chosen": -0.22574539482593536,
      "logits/rejected": -0.19882050156593323,
      "logps/chosen": -120.62968444824219,
      "logps/rejected": -124.78437805175781,
      "logps/weighted_chosen": -0.9401000738143921,
      "logps/weighted_rejected": -0.8424423336982727,
      "loss": 0.7174,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -2.4033203125,
      "rewards/margins": 0.613085925579071,
      "rewards/rejected": -3.016406297683716,
      "rewards/weighted_accuracies": 0.48750001192092896,
      "rewards/weighted_chosen": -0.04837837070226669,
      "rewards/weighted_margins": 0.04595222324132919,
      "rewards/weighted_rejected": -0.09437103569507599,
      "step": 170
    },
    {
      "epoch": 0.09619881087581,
      "grad_norm": 55.62498092651367,
      "learning_rate": 9.52127659574468e-07,
      "logits/chosen": -0.2216789275407791,
      "logits/rejected": -0.20373383164405823,
      "logps/chosen": -120.3851547241211,
      "logps/rejected": -121.2328109741211,
      "logps/weighted_chosen": -0.435232549905777,
      "logps/weighted_rejected": -0.5373809933662415,
      "loss": 0.7302,
      "rewards/accuracies": 0.47187501192092896,
      "rewards/chosen": -1.338281273841858,
      "rewards/margins": 0.3658203184604645,
      "rewards/rejected": -1.7041015625,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.08163909614086151,
      "rewards/weighted_margins": -0.008893203921616077,
      "rewards/weighted_rejected": -0.07281456142663956,
      "step": 180
    },
    {
      "epoch": 0.10154318925779945,
      "grad_norm": 39.38788986206055,
      "learning_rate": 9.99999129927477e-07,
      "logits/chosen": -0.19243164360523224,
      "logits/rejected": -0.15758666396141052,
      "logps/chosen": -121.8960952758789,
      "logps/rejected": -123.01094055175781,
      "logps/weighted_chosen": -0.953173816204071,
      "logps/weighted_rejected": -1.0853087902069092,
      "loss": 0.6702,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.100390672683716,
      "rewards/margins": 0.662890613079071,
      "rewards/rejected": -2.7632813453674316,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.09999503940343857,
      "rewards/weighted_margins": 0.12822990119457245,
      "rewards/weighted_rejected": -0.22818031907081604,
      "step": 190
    },
    {
      "epoch": 0.10688756763978889,
      "grad_norm": 12.883384704589844,
      "learning_rate": 9.99894724888679e-07,
      "logits/chosen": -0.17392578721046448,
      "logits/rejected": -0.147491455078125,
      "logps/chosen": -129.53750610351562,
      "logps/rejected": -128.95703125,
      "logps/weighted_chosen": -0.7322753667831421,
      "logps/weighted_rejected": -1.0728164911270142,
      "loss": 0.6777,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.215625047683716,
      "rewards/margins": 0.9507812261581421,
      "rewards/rejected": -4.166406154632568,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.1104557067155838,
      "rewards/weighted_margins": 0.13128623366355896,
      "rewards/weighted_rejected": -0.2415626496076584,
      "step": 200
    },
    {
      "epoch": 0.11223194602177834,
      "grad_norm": 17.96660041809082,
      "learning_rate": 9.996163469793475e-07,
      "logits/chosen": -0.18083801865577698,
      "logits/rejected": -0.12766647338867188,
      "logps/chosen": -128.1953125,
      "logps/rejected": -123.03047180175781,
      "logps/weighted_chosen": -0.8699920773506165,
      "logps/weighted_rejected": -0.8845611810684204,
      "loss": 0.7572,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.5113282203674316,
      "rewards/margins": 1.3683593273162842,
      "rewards/rejected": -4.879296779632568,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.07919616997241974,
      "rewards/weighted_margins": 0.04895629733800888,
      "rewards/weighted_rejected": -0.12815475463867188,
      "step": 210
    },
    {
      "epoch": 0.11757632440376779,
      "grad_norm": 63.790077209472656,
      "learning_rate": 9.991640930802883e-07,
      "logits/chosen": -0.14619751274585724,
      "logits/rejected": -0.13860473036766052,
      "logps/chosen": -126.61015319824219,
      "logps/rejected": -129.9773406982422,
      "logps/weighted_chosen": -1.0776779651641846,
      "logps/weighted_rejected": -0.9309722781181335,
      "loss": 0.7107,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -4.296484470367432,
      "rewards/margins": 1.10546875,
      "rewards/rejected": -5.40234375,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.013729858212172985,
      "rewards/weighted_margins": 0.14467468857765198,
      "rewards/weighted_rejected": -0.1582542359828949,
      "step": 220
    },
    {
      "epoch": 0.12292070278575723,
      "grad_norm": 9.634794235229492,
      "learning_rate": 9.98538120584459e-07,
      "logits/chosen": -0.1792152374982834,
      "logits/rejected": -0.1362098753452301,
      "logps/chosen": -133.7140655517578,
      "logps/rejected": -137.7546844482422,
      "logps/weighted_chosen": -0.6676223874092102,
      "logps/weighted_rejected": -0.9518234133720398,
      "loss": 0.7614,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -4.493554592132568,
      "rewards/margins": 1.0623047351837158,
      "rewards/rejected": -5.555859565734863,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.1120578795671463,
      "rewards/weighted_margins": 0.031700897961854935,
      "rewards/weighted_rejected": -0.14376182854175568,
      "step": 230
    },
    {
      "epoch": 0.12826508116774668,
      "grad_norm": 8.717682838439941,
      "learning_rate": 9.977386473421917e-07,
      "logits/chosen": -0.09295501559972763,
      "logits/rejected": -0.07836151123046875,
      "logps/chosen": -121.78202819824219,
      "logps/rejected": -126.39453125,
      "logps/weighted_chosen": -0.8116515874862671,
      "logps/weighted_rejected": -1.2165496349334717,
      "loss": 0.6731,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -7.553515434265137,
      "rewards/margins": 1.669921875,
      "rewards/rejected": -9.224218368530273,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.1755084991455078,
      "rewards/weighted_margins": 0.15414047241210938,
      "rewards/weighted_rejected": -0.32989272475242615,
      "step": 240
    },
    {
      "epoch": 0.13360945954973613,
      "grad_norm": 35.197269439697266,
      "learning_rate": 9.96765951585378e-07,
      "logits/chosen": -0.021549224853515625,
      "logits/rejected": 0.00345611572265625,
      "logps/chosen": -123.0296859741211,
      "logps/rejected": -127.9671859741211,
      "logps/weighted_chosen": -1.186151146888733,
      "logps/weighted_rejected": -1.386474609375,
      "loss": 0.7026,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -9.133593559265137,
      "rewards/margins": 1.150781273841858,
      "rewards/rejected": -10.283984184265137,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.3282794952392578,
      "rewards/weighted_margins": 0.1091407760977745,
      "rewards/weighted_rejected": -0.4371795654296875,
      "step": 250
    },
    {
      "epoch": 0.13895383793172555,
      "grad_norm": 84.2197494506836,
      "learning_rate": 9.956203718306388e-07,
      "logits/chosen": 0.025267411023378372,
      "logits/rejected": 0.06335143744945526,
      "logps/chosen": -128.8429718017578,
      "logps/rejected": -133.50625610351562,
      "logps/weighted_chosen": -0.9628280401229858,
      "logps/weighted_rejected": -1.2008453607559204,
      "loss": 0.7443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -10.861719131469727,
      "rewards/margins": 0.871874988079071,
      "rewards/rejected": -11.733593940734863,
      "rewards/weighted_accuracies": 0.49687498807907104,
      "rewards/weighted_chosen": -0.26259079575538635,
      "rewards/weighted_margins": 0.06770706176757812,
      "rewards/weighted_rejected": -0.3305191099643707,
      "step": 260
    },
    {
      "epoch": 0.144298216313715,
      "grad_norm": 57.09628677368164,
      "learning_rate": 9.943023067615136e-07,
      "logits/chosen": -0.017973709851503372,
      "logits/rejected": -0.0057998658157885075,
      "logps/chosen": -137.60311889648438,
      "logps/rejected": -140.91561889648438,
      "logps/weighted_chosen": -0.8694183230400085,
      "logps/weighted_rejected": -0.8910156488418579,
      "loss": 0.7462,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -10.946093559265137,
      "rewards/margins": 1.662109375,
      "rewards/rejected": -12.608202934265137,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.06702041625976562,
      "rewards/weighted_margins": 0.13949890434741974,
      "rewards/weighted_rejected": -0.20627327263355255,
      "step": 270
    },
    {
      "epoch": 0.14964259469570446,
      "grad_norm": 45.0294303894043,
      "learning_rate": 9.928122150897112e-07,
      "logits/chosen": -0.045014191418886185,
      "logits/rejected": 0.02437591552734375,
      "logps/chosen": -130.1125030517578,
      "logps/rejected": -131.6171875,
      "logps/weighted_chosen": -0.7659690976142883,
      "logps/weighted_rejected": -0.8499618768692017,
      "loss": 0.7079,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -12.664453506469727,
      "rewards/margins": 1.584375023841858,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.22265930473804474,
      "rewards/weighted_margins": 0.07556991279125214,
      "rewards/weighted_rejected": -0.298013299703598,
      "step": 280
    },
    {
      "epoch": 0.1549869730776939,
      "grad_norm": 9.370684623718262,
      "learning_rate": 9.9115061539547e-07,
      "logits/chosen": -0.02069244347512722,
      "logits/rejected": 0.01569976843893528,
      "logps/chosen": -139.4453125,
      "logps/rejected": -141.2578125,
      "logps/weighted_chosen": -0.8775177001953125,
      "logps/weighted_rejected": -1.2375701665878296,
      "loss": 0.6949,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -13.07421875,
      "rewards/margins": 1.177734375,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.18484345078468323,
      "rewards/weighted_margins": 0.12048187106847763,
      "rewards/weighted_rejected": -0.30528029799461365,
      "step": 290
    },
    {
      "epoch": 0.16033135145968336,
      "grad_norm": 30.68979263305664,
      "learning_rate": 9.893180859470818e-07,
      "logits/chosen": -0.0137786865234375,
      "logits/rejected": 0.041573334485292435,
      "logps/chosen": -130.22265625,
      "logps/rejected": -132.06094360351562,
      "logps/weighted_chosen": -1.1769134998321533,
      "logps/weighted_rejected": -1.2438628673553467,
      "loss": 0.7261,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -13.093358993530273,
      "rewards/margins": 2.108203172683716,
      "rewards/rejected": -15.200780868530273,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.2848831117153168,
      "rewards/weighted_margins": 0.07841148227453232,
      "rewards/weighted_rejected": -0.36317139863967896,
      "step": 300
    },
    {
      "epoch": 0.16567572984167278,
      "grad_norm": 71.56196594238281,
      "learning_rate": 9.873152644996424e-07,
      "logits/chosen": -0.11114044487476349,
      "logits/rejected": -0.11599807441234589,
      "logps/chosen": -131.01718139648438,
      "logps/rejected": -132.28125,
      "logps/weighted_chosen": -1.1032531261444092,
      "logps/weighted_rejected": -0.8420807123184204,
      "loss": 0.7872,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -12.979687690734863,
      "rewards/margins": 2.407421827316284,
      "rewards/rejected": -15.388280868530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": 0.05532989650964737,
      "rewards/weighted_margins": 0.2024696320295334,
      "rewards/weighted_rejected": -0.1469375640153885,
      "step": 310
    },
    {
      "epoch": 0.17102010822366223,
      "grad_norm": 12.055798530578613,
      "learning_rate": 9.85142848073103e-07,
      "logits/chosen": -0.10557708889245987,
      "logits/rejected": -0.06234588474035263,
      "logps/chosen": -133.39218139648438,
      "logps/rejected": -134.2109375,
      "logps/weighted_chosen": -0.700244128704071,
      "logps/weighted_rejected": -1.4902985095977783,
      "loss": 0.7558,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -14.2265625,
      "rewards/margins": 0.6207031011581421,
      "rewards/rejected": -14.846094131469727,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.2170669585466385,
      "rewards/weighted_margins": 0.03446922451257706,
      "rewards/weighted_rejected": -0.25187987089157104,
      "step": 320
    },
    {
      "epoch": 0.17636448660565168,
      "grad_norm": 41.381507873535156,
      "learning_rate": 9.828015927096914e-07,
      "logits/chosen": -0.085169218480587,
      "logits/rejected": -0.05568351596593857,
      "logps/chosen": -143.8234405517578,
      "logps/rejected": -142.39608764648438,
      "logps/weighted_chosen": -0.977459728717804,
      "logps/weighted_rejected": -0.7958831787109375,
      "loss": 0.6935,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -14.643750190734863,
      "rewards/margins": 1.578125,
      "rewards/rejected": -16.222265243530273,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.16206398606300354,
      "rewards/weighted_margins": 0.07153511047363281,
      "rewards/weighted_rejected": -0.23361511528491974,
      "step": 330
    },
    {
      "epoch": 0.18170886498764113,
      "grad_norm": 7.313599586486816,
      "learning_rate": 9.802923132107968e-07,
      "logits/chosen": -0.10784301906824112,
      "logits/rejected": -0.052645109593868256,
      "logps/chosen": -143.91561889648438,
      "logps/rejected": -147.7234344482422,
      "logps/weighted_chosen": -0.7267853021621704,
      "logps/weighted_rejected": -1.24786376953125,
      "loss": 0.7102,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -16.782812118530273,
      "rewards/margins": 1.23046875,
      "rewards/rejected": -18.012500762939453,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.1999809294939041,
      "rewards/weighted_margins": 0.056002043187618256,
      "rewards/weighted_rejected": -0.2561977505683899,
      "step": 340
    },
    {
      "epoch": 0.18705324336963056,
      "grad_norm": 66.15936279296875,
      "learning_rate": 9.776158828534024e-07,
      "logits/chosen": -0.12608642876148224,
      "logits/rejected": -0.09711913764476776,
      "logps/chosen": -143.48281860351562,
      "logps/rejected": -188.25936889648438,
      "logps/weighted_chosen": -1.260833740234375,
      "logps/weighted_rejected": -0.9638397097587585,
      "loss": 0.7338,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -16.727733612060547,
      "rewards/margins": 3.159374952316284,
      "rewards/rejected": -19.883594512939453,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.26622849702835083,
      "rewards/weighted_margins": 0.05975799635052681,
      "rewards/weighted_rejected": -0.3257743716239929,
      "step": 350
    },
    {
      "epoch": 0.19239762175162,
      "grad_norm": 10.4098539352417,
      "learning_rate": 9.747732330861695e-07,
      "logits/chosen": -0.05247039720416069,
      "logits/rejected": -0.024548912420868874,
      "logps/chosen": -135.78750610351562,
      "logps/rejected": -137.03671264648438,
      "logps/weighted_chosen": -0.8440704345703125,
      "logps/weighted_rejected": -0.8307411074638367,
      "loss": 0.7415,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -16.864452362060547,
      "rewards/margins": 2.5316405296325684,
      "rewards/rejected": -19.396093368530273,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": 0.005719375796616077,
      "rewards/weighted_margins": 0.17883948981761932,
      "rewards/weighted_rejected": -0.17323684692382812,
      "step": 360
    },
    {
      "epoch": 0.19774200013360946,
      "grad_norm": 8.8024263381958,
      "learning_rate": 9.717653532052742e-07,
      "logits/chosen": -0.0410335548222065,
      "logits/rejected": -0.049485016614198685,
      "logps/chosen": -130.2781219482422,
      "logps/rejected": -137.9343719482422,
      "logps/weighted_chosen": -1.1705901622772217,
      "logps/weighted_rejected": -0.7656036615371704,
      "loss": 0.7402,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -17.155858993530273,
      "rewards/margins": 2.863085985183716,
      "rewards/rejected": -20.016016006469727,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.07308082282543182,
      "rewards/weighted_margins": 0.04105224460363388,
      "rewards/weighted_rejected": -0.114354707300663,
      "step": 370
    },
    {
      "epoch": 0.2030863785155989,
      "grad_norm": 26.81167221069336,
      "learning_rate": 9.685932900101146e-07,
      "logits/chosen": -0.07604064792394638,
      "logits/rejected": -0.06302948296070099,
      "logps/chosen": -134.7859344482422,
      "logps/rejected": -137.11093139648438,
      "logps/weighted_chosen": -0.98638916015625,
      "logps/weighted_rejected": -0.6243560910224915,
      "loss": 0.7116,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.729297637939453,
      "rewards/margins": 1.8025391101837158,
      "rewards/rejected": -19.528514862060547,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.11945419013500214,
      "rewards/weighted_margins": 0.07185058295726776,
      "rewards/weighted_rejected": -0.1911766082048416,
      "step": 380
    },
    {
      "epoch": 0.20843075689758836,
      "grad_norm": 40.89613342285156,
      "learning_rate": 9.652581474390043e-07,
      "logits/chosen": -0.09274902194738388,
      "logits/rejected": -0.05452308803796768,
      "logps/chosen": -139.3249969482422,
      "logps/rejected": -141.53750610351562,
      "logps/weighted_chosen": -0.9199386835098267,
      "logps/weighted_rejected": -0.9217513799667358,
      "loss": 0.7323,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -19.379688262939453,
      "rewards/margins": 2.321093797683716,
      "rewards/rejected": -21.702733993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.05511016771197319,
      "rewards/weighted_margins": 0.04057302325963974,
      "rewards/weighted_rejected": -0.09564266353845596,
      "step": 390
    },
    {
      "epoch": 0.21377513527957778,
      "grad_norm": 51.860347747802734,
      "learning_rate": 9.61761086184981e-07,
      "logits/chosen": -0.10268402099609375,
      "logits/rejected": -0.05923309177160263,
      "logps/chosen": -139.50936889648438,
      "logps/rejected": -141.4015655517578,
      "logps/weighted_chosen": -0.9449096918106079,
      "logps/weighted_rejected": -1.177301049232483,
      "loss": 0.7387,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -20.024608612060547,
      "rewards/margins": 2.9205079078674316,
      "rewards/rejected": -22.942577362060547,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.26060789823532104,
      "rewards/weighted_margins": 0.017241667956113815,
      "rewards/weighted_rejected": -0.2779647707939148,
      "step": 400
    },
    {
      "epoch": 0.21911951366156723,
      "grad_norm": 7.539150238037109,
      "learning_rate": 9.581033232918629e-07,
      "logits/chosen": -0.015906523913145065,
      "logits/rejected": 0.02000865899026394,
      "logps/chosen": -138.39999389648438,
      "logps/rejected": -141.68905639648438,
      "logps/weighted_chosen": -1.0313812494277954,
      "logps/weighted_rejected": -1.1196472644805908,
      "loss": 0.7283,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -21.702342987060547,
      "rewards/margins": 2.826953172683716,
      "rewards/rejected": -24.53125,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": -0.2783859372138977,
      "rewards/weighted_margins": 0.06903991848230362,
      "rewards/weighted_rejected": -0.3477470278739929,
      "step": 410
    },
    {
      "epoch": 0.22446389204355668,
      "grad_norm": 6.638446807861328,
      "learning_rate": 9.542861317306952e-07,
      "logits/chosen": -0.05767669528722763,
      "logits/rejected": -0.03759918361902237,
      "logps/chosen": -140.64688110351562,
      "logps/rejected": -141.9718780517578,
      "logps/weighted_chosen": -1.5940430164337158,
      "logps/weighted_rejected": -1.458154320716858,
      "loss": 0.7371,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -22.33203125,
      "rewards/margins": 1.1359374523162842,
      "rewards/rejected": -23.46484375,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.299835205078125,
      "rewards/weighted_margins": 0.032170869410037994,
      "rewards/weighted_rejected": -0.3319748044013977,
      "step": 420
    },
    {
      "epoch": 0.22980827042554614,
      "grad_norm": 24.621816635131836,
      "learning_rate": 9.503108399567308e-07,
      "logits/chosen": -0.15150222182273865,
      "logits/rejected": -0.09339599311351776,
      "logps/chosen": -149.8671875,
      "logps/rejected": -153.6296844482422,
      "logps/weighted_chosen": -0.7455719113349915,
      "logps/weighted_rejected": -0.826556384563446,
      "loss": 0.7125,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -22.685546875,
      "rewards/margins": 1.7355468273162842,
      "rewards/rejected": -24.420312881469727,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.19403991103172302,
      "rewards/weighted_margins": 0.02692871168255806,
      "rewards/weighted_rejected": -0.2209148406982422,
      "step": 430
    },
    {
      "epoch": 0.23515264880753559,
      "grad_norm": 10.726137161254883,
      "learning_rate": 9.461788314471034e-07,
      "logits/chosen": -0.08421440422534943,
      "logits/rejected": -0.031351469457149506,
      "logps/chosen": -143.1171875,
      "logps/rejected": -146.890625,
      "logps/weighted_chosen": -0.7524811029434204,
      "logps/weighted_rejected": -0.8818389773368835,
      "loss": 0.7062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -24.516796112060547,
      "rewards/margins": 2.0640625953674316,
      "rewards/rejected": -26.577733993530273,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.22655639052391052,
      "rewards/weighted_margins": 0.07337188720703125,
      "rewards/weighted_rejected": -0.30005186796188354,
      "step": 440
    },
    {
      "epoch": 0.240497027189525,
      "grad_norm": 8.586684226989746,
      "learning_rate": 9.418915442193509e-07,
      "logits/chosen": -0.03446502611041069,
      "logits/rejected": 0.007968139834702015,
      "logps/chosen": -144.7843780517578,
      "logps/rejected": -155.96249389648438,
      "logps/weighted_chosen": -1.1978485584259033,
      "logps/weighted_rejected": -1.2447845935821533,
      "loss": 0.6841,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -26.516016006469727,
      "rewards/margins": 4.714453220367432,
      "rewards/rejected": -31.236328125,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.25520628690719604,
      "rewards/weighted_margins": 0.09325332939624786,
      "rewards/weighted_rejected": -0.3483436703681946,
      "step": 450
    },
    {
      "epoch": 0.24584140557151446,
      "grad_norm": 10.014894485473633,
      "learning_rate": 9.374504703309579e-07,
      "logits/chosen": -0.1379852294921875,
      "logits/rejected": -0.0861412063241005,
      "logps/chosen": -141.83828735351562,
      "logps/rejected": -143.34530639648438,
      "logps/weighted_chosen": -1.1420501470565796,
      "logps/weighted_rejected": -1.055877685546875,
      "loss": 0.6804,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -25.53515625,
      "rewards/margins": 2.038281202316284,
      "rewards/rejected": -27.571874618530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.21983489394187927,
      "rewards/weighted_margins": 0.17051848769187927,
      "rewards/weighted_rejected": -0.3904357850551605,
      "step": 460
    },
    {
      "epoch": 0.2511857839535039,
      "grad_norm": 10.024358749389648,
      "learning_rate": 9.328571553600915e-07,
      "logits/chosen": -0.1318313628435135,
      "logits/rejected": -0.0828731507062912,
      "logps/chosen": -144.39688110351562,
      "logps/rejected": -147.41796875,
      "logps/weighted_chosen": -0.9446808099746704,
      "logps/weighted_rejected": -1.138859510421753,
      "loss": 0.7137,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -25.697265625,
      "rewards/margins": 1.9656250476837158,
      "rewards/rejected": -27.665233612060547,
      "rewards/weighted_accuracies": 0.4906249940395355,
      "rewards/weighted_chosen": -0.2078697234392166,
      "rewards/weighted_margins": 0.052256010472774506,
      "rewards/weighted_rejected": -0.260000616312027,
      "step": 470
    },
    {
      "epoch": 0.25653016233549336,
      "grad_norm": 13.618422508239746,
      "learning_rate": 9.281131978677106e-07,
      "logits/chosen": -0.16686096787452698,
      "logits/rejected": -0.1246364563703537,
      "logps/chosen": -149.2546844482422,
      "logps/rejected": -152.47811889648438,
      "logps/weighted_chosen": -0.686419665813446,
      "logps/weighted_rejected": -1.0338714122772217,
      "loss": 0.6805,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -25.559375762939453,
      "rewards/margins": 3.1332030296325684,
      "rewards/rejected": -28.689844131469727,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.22538375854492188,
      "rewards/weighted_margins": 0.13199158012866974,
      "rewards/weighted_rejected": -0.35721856355667114,
      "step": 480
    },
    {
      "epoch": 0.2618745407174828,
      "grad_norm": 30.852863311767578,
      "learning_rate": 9.232202488412361e-07,
      "logits/chosen": -0.10437698662281036,
      "logits/rejected": -0.05600614473223686,
      "logps/chosen": -143.5671844482422,
      "logps/rejected": -149.11874389648438,
      "logps/weighted_chosen": -1.236029028892517,
      "logps/weighted_rejected": -1.6542327404022217,
      "loss": 0.6937,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -28.171483993530273,
      "rewards/margins": 2.311328172683716,
      "rewards/rejected": -30.477733612060547,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.27268296480178833,
      "rewards/weighted_margins": 0.08454742282629013,
      "rewards/weighted_rejected": -0.35735321044921875,
      "step": 490
    },
    {
      "epoch": 0.26721891909947226,
      "grad_norm": 9.151970863342285,
      "learning_rate": 9.181800111199766e-07,
      "logits/chosen": -0.1230342835187912,
      "logits/rejected": -0.0801507979631424,
      "logps/chosen": -147.54843139648438,
      "logps/rejected": -147.5437469482422,
      "logps/weighted_chosen": -1.5909912586212158,
      "logps/weighted_rejected": -1.286199927330017,
      "loss": 0.7005,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -29.640233993530273,
      "rewards/margins": 3.1507811546325684,
      "rewards/rejected": -32.789451599121094,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.34203338623046875,
      "rewards/weighted_margins": 0.12214355170726776,
      "rewards/weighted_rejected": -0.4641265869140625,
      "step": 500
    },
    {
      "epoch": 0.26721891909947226,
      "eval_logits/chosen": -0.29476672410964966,
      "eval_logits/rejected": -0.26891082525253296,
      "eval_logps/chosen": -149.5992889404297,
      "eval_logps/rejected": -153.5249481201172,
      "eval_logps/weighted_chosen": -0.8761399984359741,
      "eval_logps/weighted_rejected": -1.120854377746582,
      "eval_loss": 0.6999610066413879,
      "eval_rewards/accuracies": 0.5784114003181458,
      "eval_rewards/chosen": -28.267057418823242,
      "eval_rewards/margins": 2.7842414379119873,
      "eval_rewards/rejected": -31.05422592163086,
      "eval_rewards/weighted_accuracies": 0.5264765620231628,
      "eval_rewards/weighted_chosen": -0.23699289560317993,
      "eval_rewards/weighted_margins": 0.08454981446266174,
      "eval_rewards/weighted_rejected": -0.3215427100658417,
      "eval_runtime": 1248.3443,
      "eval_samples_per_second": 1.571,
      "eval_steps_per_second": 0.393,
      "step": 500
    },
    {
      "epoch": 0.2725632974814617,
      "grad_norm": 7.777375221252441,
      "learning_rate": 9.129942388025066e-07,
      "logits/chosen": -0.1614234894514084,
      "logits/rejected": -0.12539061903953552,
      "logps/chosen": -138.70468139648438,
      "logps/rejected": -144.50936889648438,
      "logps/weighted_chosen": -1.0583159923553467,
      "logps/weighted_rejected": -1.0353851318359375,
      "loss": 0.7783,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -27.291797637939453,
      "rewards/margins": 2.485156297683716,
      "rewards/rejected": -29.778514862060547,
      "rewards/weighted_accuracies": 0.5093749761581421,
      "rewards/weighted_chosen": -0.21576309204101562,
      "rewards/weighted_margins": 0.003093719482421875,
      "rewards/weighted_rejected": -0.219013974070549,
      "step": 510
    },
    {
      "epoch": 0.2779076758634511,
      "grad_norm": 42.880157470703125,
      "learning_rate": 9.076647366362082e-07,
      "logits/chosen": -0.0863800048828125,
      "logits/rejected": -0.04528322070837021,
      "logps/chosen": -148.21875,
      "logps/rejected": -155.3046875,
      "logps/weighted_chosen": -1.141107201576233,
      "logps/weighted_rejected": -1.2995758056640625,
      "loss": 0.7748,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -33.478515625,
      "rewards/margins": 4.057812690734863,
      "rewards/rejected": -37.53515625,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.2628006041049957,
      "rewards/weighted_margins": -0.00189971923828125,
      "rewards/weighted_rejected": -0.26109999418258667,
      "step": 520
    },
    {
      "epoch": 0.2832520542454406,
      "grad_norm": 24.728120803833008,
      "learning_rate": 9.021933593891841e-07,
      "logits/chosen": -0.020507430657744408,
      "logits/rejected": 0.01460342388600111,
      "logps/chosen": -154.15780639648438,
      "logps/rejected": -156.328125,
      "logps/weighted_chosen": -1.0978972911834717,
      "logps/weighted_rejected": -0.9736785888671875,
      "loss": 0.6998,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -40.849220275878906,
      "rewards/margins": 2.678515672683716,
      "rewards/rejected": -43.53046798706055,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.2724204957485199,
      "rewards/weighted_margins": 0.09940185397863388,
      "rewards/weighted_rejected": -0.3718253970146179,
      "step": 530
    },
    {
      "epoch": 0.28859643262743,
      "grad_norm": 11.090974807739258,
      "learning_rate": 8.965820112047629e-07,
      "logits/chosen": -0.07119598239660263,
      "logits/rejected": -0.04578246921300888,
      "logps/chosen": -163.8781280517578,
      "logps/rejected": -172.9187469482422,
      "logps/weighted_chosen": -0.9050750732421875,
      "logps/weighted_rejected": -1.05401611328125,
      "loss": 0.6621,
      "rewards/accuracies": 0.559374988079071,
      "rewards/chosen": -42.501564025878906,
      "rewards/margins": 4.053515434265137,
      "rewards/rejected": -46.54999923706055,
      "rewards/weighted_accuracies": 0.6000000238418579,
      "rewards/weighted_chosen": -0.1999252289533615,
      "rewards/weighted_margins": 0.1526111662387848,
      "rewards/weighted_rejected": -0.3521057069301605,
      "step": 540
    },
    {
      "epoch": 0.2939408110094195,
      "grad_norm": 13.709502220153809,
      "learning_rate": 8.908326449388213e-07,
      "logits/chosen": -0.07627105712890625,
      "logits/rejected": -0.03014526329934597,
      "logps/chosen": -156.328125,
      "logps/rejected": -161.06405639648438,
      "logps/weighted_chosen": -1.228173851966858,
      "logps/weighted_rejected": -1.3901275396347046,
      "loss": 0.7368,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -41.07109451293945,
      "rewards/margins": 4.337500095367432,
      "rewards/rejected": -45.392967224121094,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.172210693359375,
      "rewards/weighted_margins": 0.07322921603918076,
      "rewards/weighted_rejected": -0.2451316863298416,
      "step": 550
    },
    {
      "epoch": 0.2992851893914089,
      "grad_norm": 10.929818153381348,
      "learning_rate": 8.849472614801527e-07,
      "logits/chosen": -0.07251434028148651,
      "logits/rejected": -0.02443542517721653,
      "logps/chosen": -161.86874389648438,
      "logps/rejected": -165.0749969482422,
      "logps/weighted_chosen": -0.752777099609375,
      "logps/weighted_rejected": -1.0892517566680908,
      "loss": 0.6642,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.9609375,
      "rewards/margins": 3.3421874046325684,
      "rewards/rejected": -48.30546951293945,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.1927940398454666,
      "rewards/weighted_margins": 0.19287796318531036,
      "rewards/weighted_rejected": -0.38528138399124146,
      "step": 560
    },
    {
      "epoch": 0.30462956777339834,
      "grad_norm": 11.730274200439453,
      "learning_rate": 8.789279090541208e-07,
      "logits/chosen": -0.05120544508099556,
      "logits/rejected": 0.01478500384837389,
      "logps/chosen": -161.8796844482422,
      "logps/rejected": -172.51406860351562,
      "logps/weighted_chosen": -0.8590911626815796,
      "logps/weighted_rejected": -1.1839568614959717,
      "loss": 0.6821,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.99609375,
      "rewards/margins": 4.257031440734863,
      "rewards/rejected": -48.24609375,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.29502803087234497,
      "rewards/weighted_margins": 0.16868743300437927,
      "rewards/weighted_rejected": -0.4639228880405426,
      "step": 570
    },
    {
      "epoch": 0.3099739461553878,
      "grad_norm": 16.967859268188477,
      "learning_rate": 8.72776682509837e-07,
      "logits/chosen": 0.011944961734116077,
      "logits/rejected": 0.08212585747241974,
      "logps/chosen": -157.47500610351562,
      "logps/rejected": -163.43594360351562,
      "logps/weighted_chosen": -0.8815460205078125,
      "logps/weighted_rejected": -1.220306396484375,
      "loss": 0.6705,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.369529724121094,
      "rewards/margins": 4.614453315734863,
      "rewards/rejected": -47.984375,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.3169898986816406,
      "rewards/weighted_margins": 0.19199447333812714,
      "rewards/weighted_rejected": -0.5094040036201477,
      "step": 580
    },
    {
      "epoch": 0.31531832453737724,
      "grad_norm": 9.289381980895996,
      "learning_rate": 8.664957225911138e-07,
      "logits/chosen": -0.04825134202837944,
      "logits/rejected": -0.01284942589700222,
      "logps/chosen": -163.6437530517578,
      "logps/rejected": -184.9796905517578,
      "logps/weighted_chosen": -0.775604248046875,
      "logps/weighted_rejected": -1.1336090564727783,
      "loss": 0.6889,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -42.566795349121094,
      "rewards/margins": 3.8046875,
      "rewards/rejected": -46.36328125,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.30945053696632385,
      "rewards/weighted_margins": 0.116083525121212,
      "rewards/weighted_rejected": -0.4253120422363281,
      "step": 590
    },
    {
      "epoch": 0.3206627029193667,
      "grad_norm": 7.966893672943115,
      "learning_rate": 8.600872151914451e-07,
      "logits/chosen": 0.06866455078125,
      "logits/rejected": 0.09470061957836151,
      "logps/chosen": -158.2546844482422,
      "logps/rejected": -167.48281860351562,
      "logps/weighted_chosen": -1.3719971179962158,
      "logps/weighted_rejected": -1.7499603033065796,
      "loss": 0.7417,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -44.11249923706055,
      "rewards/margins": 3.3988280296325684,
      "rewards/rejected": -47.50468826293945,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.21634292602539062,
      "rewards/weighted_margins": 0.19443893432617188,
      "rewards/weighted_rejected": -0.41086500883102417,
      "step": 600
    },
    {
      "epoch": 0.32600708130135614,
      "grad_norm": 49.298519134521484,
      "learning_rate": 8.535533905932737e-07,
      "logits/chosen": 0.06249542161822319,
      "logits/rejected": 0.12158355861902237,
      "logps/chosen": -173.4250030517578,
      "logps/rejected": -183.015625,
      "logps/weighted_chosen": -0.9283081293106079,
      "logps/weighted_rejected": -1.2264496088027954,
      "loss": 0.6778,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -48.342185974121094,
      "rewards/margins": 5.698437690734863,
      "rewards/rejected": -54.049217224121094,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.3028602600097656,
      "rewards/weighted_margins": 0.1688835173845291,
      "rewards/weighted_rejected": -0.4717727601528168,
      "step": 610
    },
    {
      "epoch": 0.33135145968334556,
      "grad_norm": 19.377832412719727,
      "learning_rate": 8.468965226918105e-07,
      "logits/chosen": 0.037828825414180756,
      "logits/rejected": 0.0580875389277935,
      "logps/chosen": -170.1281280517578,
      "logps/rejected": -178.9093780517578,
      "logps/weighted_chosen": -1.4841537475585938,
      "logps/weighted_rejected": -1.118981957435608,
      "loss": 0.7542,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -46.46210861206055,
      "rewards/margins": 3.794921875,
      "rewards/rejected": -50.26640701293945,
      "rewards/weighted_accuracies": 0.596875011920929,
      "rewards/weighted_chosen": -0.40849533677101135,
      "rewards/weighted_margins": 0.0321044921875,
      "rewards/weighted_rejected": -0.4405921995639801,
      "step": 620
    },
    {
      "epoch": 0.33669583806533504,
      "grad_norm": 10.768741607666016,
      "learning_rate": 8.40118928203676e-07,
      "logits/chosen": 0.05730438232421875,
      "logits/rejected": 0.07560119777917862,
      "logps/chosen": -152.3718719482422,
      "logps/rejected": -164.1218719482422,
      "logps/weighted_chosen": -1.4566253423690796,
      "logps/weighted_rejected": -1.3677704334259033,
      "loss": 0.7025,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.721092224121094,
      "rewards/margins": 4.366015434265137,
      "rewards/rejected": -48.07500076293945,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.30116957426071167,
      "rewards/weighted_margins": 0.09331168979406357,
      "rewards/weighted_rejected": -0.39441680908203125,
      "step": 630
    },
    {
      "epoch": 0.34204021644732446,
      "grad_norm": 8.60303020477295,
      "learning_rate": 8.332229658606382e-07,
      "logits/chosen": -0.055472563952207565,
      "logits/rejected": -0.03988037258386612,
      "logps/chosen": -175.6671905517578,
      "logps/rejected": -180.06875610351562,
      "logps/weighted_chosen": -0.8608856201171875,
      "logps/weighted_rejected": -0.986541748046875,
      "loss": 0.7297,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -44.84062576293945,
      "rewards/margins": 4.931250095367432,
      "rewards/rejected": -49.77031326293945,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.23048630356788635,
      "rewards/weighted_margins": 0.10724487155675888,
      "rewards/weighted_rejected": -0.33786240220069885,
      "step": 640
    },
    {
      "epoch": 0.34738459482931394,
      "grad_norm": 26.148893356323242,
      "learning_rate": 8.262110355887302e-07,
      "logits/chosen": -0.055730439722537994,
      "logits/rejected": -0.00980529747903347,
      "logps/chosen": -166.38125610351562,
      "logps/rejected": -169.40469360351562,
      "logps/weighted_chosen": -0.7286010980606079,
      "logps/weighted_rejected": -0.8990936279296875,
      "loss": 0.7103,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -46.994529724121094,
      "rewards/margins": 3.8031249046325684,
      "rewards/rejected": -50.78515625,
      "rewards/weighted_accuracies": 0.518750011920929,
      "rewards/weighted_chosen": -0.2977249026298523,
      "rewards/weighted_margins": 0.08674392849206924,
      "rewards/weighted_rejected": -0.3846077024936676,
      "step": 650
    },
    {
      "epoch": 0.35272897321130336,
      "grad_norm": 11.823346138000488,
      "learning_rate": 8.190855776730293e-07,
      "logits/chosen": -0.07448883354663849,
      "logits/rejected": -0.014659118838608265,
      "logps/chosen": -171.80313110351562,
      "logps/rejected": -182.2687530517578,
      "logps/weighted_chosen": -0.9873107671737671,
      "logps/weighted_rejected": -1.4951263666152954,
      "loss": 0.7347,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -50.564064025878906,
      "rewards/margins": 4.532812595367432,
      "rewards/rejected": -55.109375,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.28445130586624146,
      "rewards/weighted_margins": 0.06389693915843964,
      "rewards/weighted_rejected": -0.34831467270851135,
      "step": 660
    },
    {
      "epoch": 0.3580733515932928,
      "grad_norm": 13.33145523071289,
      "learning_rate": 8.118490719083917e-07,
      "logits/chosen": -0.10270843654870987,
      "logits/rejected": -0.062209319323301315,
      "logps/chosen": -191.3000030517578,
      "logps/rejected": -193.4109344482422,
      "logps/weighted_chosen": -1.0586059093475342,
      "logps/weighted_rejected": -1.3695876598358154,
      "loss": 0.6575,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -58.06640625,
      "rewards/margins": 4.108984470367432,
      "rewards/rejected": -62.18359375,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.2945011258125305,
      "rewards/weighted_margins": 0.18829631805419922,
      "rewards/weighted_rejected": -0.4826883375644684,
      "step": 670
    },
    {
      "epoch": 0.36341772997528227,
      "grad_norm": 23.127918243408203,
      "learning_rate": 8.045040367364368e-07,
      "logits/chosen": 0.03829193115234375,
      "logits/rejected": 0.055706024169921875,
      "logps/chosen": -166.234375,
      "logps/rejected": -169.96249389648438,
      "logps/weighted_chosen": -1.025201439857483,
      "logps/weighted_rejected": -1.302404761314392,
      "loss": 0.7166,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -56.759376525878906,
      "rewards/margins": 4.711718559265137,
      "rewards/rejected": -61.46796798706055,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.4052978456020355,
      "rewards/weighted_margins": 0.09704723209142685,
      "rewards/weighted_rejected": -0.5025444030761719,
      "step": 680
    },
    {
      "epoch": 0.3687621083572717,
      "grad_norm": 7.137462615966797,
      "learning_rate": 7.970530283690819e-07,
      "logits/chosen": 0.04682769626379013,
      "logits/rejected": 0.066925048828125,
      "logps/chosen": -172.50155639648438,
      "logps/rejected": -180.54061889648438,
      "logps/weighted_chosen": -1.301055908203125,
      "logps/weighted_rejected": -1.128961205482483,
      "loss": 0.6864,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -58.544532775878906,
      "rewards/margins": 5.397265434265137,
      "rewards/rejected": -63.9453125,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.3540634214878082,
      "rewards/weighted_margins": 0.1536247283220291,
      "rewards/weighted_rejected": -0.5076873898506165,
      "step": 690
    },
    {
      "epoch": 0.3741064867392611,
      "grad_norm": 7.699697971343994,
      "learning_rate": 7.89498639898931e-07,
      "logits/chosen": 0.05636443942785263,
      "logits/rejected": 0.09223861992359161,
      "logps/chosen": -169.8562469482422,
      "logps/rejected": -173.65625,
      "logps/weighted_chosen": -0.9024292230606079,
      "logps/weighted_rejected": -1.1884033679962158,
      "loss": 0.6642,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -59.00078201293945,
      "rewards/margins": 4.887109279632568,
      "rewards/rejected": -63.889060974121094,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": -0.3017433285713196,
      "rewards/weighted_margins": 0.20005035400390625,
      "rewards/weighted_rejected": -0.5016593933105469,
      "step": 700
    },
    {
      "epoch": 0.3794508651212506,
      "grad_norm": 18.41610336303711,
      "learning_rate": 7.818435003968305e-07,
      "logits/chosen": -0.03687286376953125,
      "logits/rejected": -0.00025310515775345266,
      "logps/chosen": -181.0812530517578,
      "logps/rejected": -189.1828155517578,
      "logps/weighted_chosen": -1.0574157238006592,
      "logps/weighted_rejected": -1.2661011219024658,
      "loss": 0.6918,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -58.548439025878906,
      "rewards/margins": 5.687890529632568,
      "rewards/rejected": -64.234375,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.33240431547164917,
      "rewards/weighted_margins": 0.08642272651195526,
      "rewards/weighted_rejected": -0.4188430905342102,
      "step": 710
    },
    {
      "epoch": 0.38479524350324,
      "grad_norm": 26.664974212646484,
      "learning_rate": 7.740902739969008e-07,
      "logits/chosen": -0.08480529487133026,
      "logits/rejected": -0.07062835991382599,
      "logps/chosen": -171.16561889648438,
      "logps/rejected": -175.68905639648438,
      "logps/weighted_chosen": -0.828997790813446,
      "logps/weighted_rejected": -0.9821502566337585,
      "loss": 0.6819,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -49.982810974121094,
      "rewards/margins": 4.688672065734863,
      "rewards/rejected": -54.672264099121094,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.18975982069969177,
      "rewards/weighted_margins": 0.1927967071533203,
      "rewards/weighted_rejected": -0.38214111328125,
      "step": 720
    },
    {
      "epoch": 0.3901396218852295,
      "grad_norm": 8.73801040649414,
      "learning_rate": 7.662416589693695e-07,
      "logits/chosen": -0.00596694927662611,
      "logits/rejected": 0.019218826666474342,
      "logps/chosen": -161.8249969482422,
      "logps/rejected": -166.8312530517578,
      "logps/weighted_chosen": -1.3982207775115967,
      "logps/weighted_rejected": -1.7395598888397217,
      "loss": 0.7772,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -46.8515625,
      "rewards/margins": 5.096484184265137,
      "rewards/rejected": -51.95000076293945,
      "rewards/weighted_accuracies": 0.590624988079071,
      "rewards/weighted_chosen": -0.23512879014015198,
      "rewards/weighted_margins": 0.10187530517578125,
      "rewards/weighted_rejected": -0.3369201719760895,
      "step": 730
    },
    {
      "epoch": 0.3954840002672189,
      "grad_norm": 19.624563217163086,
      "learning_rate": 7.583003867815192e-07,
      "logits/chosen": -0.0639801025390625,
      "logits/rejected": -0.05755653232336044,
      "logps/chosen": -189.7468719482422,
      "logps/rejected": -196.7062530517578,
      "logps/weighted_chosen": -1.1809478998184204,
      "logps/weighted_rejected": -1.3041900396347046,
      "loss": 0.698,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -58.209373474121094,
      "rewards/margins": 4.971875190734863,
      "rewards/rejected": -63.1640625,
      "rewards/weighted_accuracies": 0.518750011920929,
      "rewards/weighted_chosen": -0.39410895109176636,
      "rewards/weighted_margins": 0.08784560859203339,
      "rewards/weighted_rejected": -0.4818374514579773,
      "step": 740
    },
    {
      "epoch": 0.40082837864920834,
      "grad_norm": 19.04839324951172,
      "learning_rate": 7.502692211470869e-07,
      "logits/chosen": -0.05941009521484375,
      "logits/rejected": -0.02816619910299778,
      "logps/chosen": -182.8984375,
      "logps/rejected": -190.1203155517578,
      "logps/weighted_chosen": -0.853833019733429,
      "logps/weighted_rejected": -1.1572723388671875,
      "loss": 0.7079,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -60.310157775878906,
      "rewards/margins": 6.424609184265137,
      "rewards/rejected": -66.73905944824219,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.4228218197822571,
      "rewards/weighted_margins": 0.10426559299230576,
      "rewards/weighted_rejected": -0.5270500183105469,
      "step": 750
    },
    {
      "epoch": 0.4061727570311978,
      "grad_norm": 12.947752952575684,
      "learning_rate": 7.421509570644387e-07,
      "logits/chosen": -0.05351562425494194,
      "logits/rejected": -0.0015235900646075606,
      "logps/chosen": -180.66250610351562,
      "logps/rejected": -185.6828155517578,
      "logps/weighted_chosen": -1.2698882818222046,
      "logps/weighted_rejected": -1.5226471424102783,
      "loss": 0.6831,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -62.2109375,
      "rewards/margins": 5.124218940734863,
      "rewards/rejected": -67.33280944824219,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.32732313871383667,
      "rewards/weighted_margins": 0.18027648329734802,
      "rewards/weighted_rejected": -0.5074516534805298,
      "step": 760
    },
    {
      "epoch": 0.41151713541318724,
      "grad_norm": 10.666881561279297,
      "learning_rate": 7.339484198438566e-07,
      "logits/chosen": 0.00444793701171875,
      "logits/rejected": 0.03400421142578125,
      "logps/chosen": -165.93124389648438,
      "logps/rejected": -172.2937469482422,
      "logps/weighted_chosen": -1.7782714366912842,
      "logps/weighted_rejected": -1.320520043373108,
      "loss": 0.6969,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -57.14374923706055,
      "rewards/margins": 7.307421684265137,
      "rewards/rejected": -64.4468765258789,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.3784065246582031,
      "rewards/weighted_margins": 0.17845840752124786,
      "rewards/weighted_rejected": -0.557085394859314,
      "step": 770
    },
    {
      "epoch": 0.4168615137951767,
      "grad_norm": 11.188801765441895,
      "learning_rate": 7.256644641242773e-07,
      "logits/chosen": -0.03662109375,
      "logits/rejected": 0.02340393140912056,
      "logps/chosen": -170.4765625,
      "logps/rejected": -175.390625,
      "logps/weighted_chosen": -1.057519555091858,
      "logps/weighted_rejected": -1.3154876232147217,
      "loss": 0.7128,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -56.12578201293945,
      "rewards/margins": 3.3226561546325684,
      "rewards/rejected": -59.43828201293945,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.37098997831344604,
      "rewards/weighted_margins": 0.07502517849206924,
      "rewards/weighted_rejected": -0.4461258053779602,
      "step": 780
    },
    {
      "epoch": 0.42220589217716614,
      "grad_norm": 9.07581615447998,
      "learning_rate": 7.173019728798234e-07,
      "logits/chosen": -0.03949751704931259,
      "logits/rejected": -0.02464141882956028,
      "logps/chosen": -169.55313110351562,
      "logps/rejected": -177.18124389648438,
      "logps/weighted_chosen": -0.8541290163993835,
      "logps/weighted_rejected": -1.1333099603652954,
      "loss": 0.6788,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -55.47734451293945,
      "rewards/margins": 6.255859375,
      "rewards/rejected": -61.71406173706055,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.307028204202652,
      "rewards/weighted_margins": 0.12036208808422089,
      "rewards/weighted_rejected": -0.4274864196777344,
      "step": 790
    },
    {
      "epoch": 0.42755027055915557,
      "grad_norm": 19.591575622558594,
      "learning_rate": 7.088638564164738e-07,
      "logits/chosen": -0.08669586479663849,
      "logits/rejected": -0.05663032457232475,
      "logps/chosen": -185.1593780517578,
      "logps/rejected": -195.95938110351562,
      "logps/weighted_chosen": -1.067468285560608,
      "logps/weighted_rejected": -1.238824486732483,
      "loss": 0.693,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -63.07421875,
      "rewards/margins": 6.429296970367432,
      "rewards/rejected": -69.49296569824219,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.3178604245185852,
      "rewards/weighted_margins": 0.1114906296133995,
      "rewards/weighted_rejected": -0.4294891357421875,
      "step": 800
    },
    {
      "epoch": 0.43289464894114504,
      "grad_norm": 13.074841499328613,
      "learning_rate": 7.003530513592215e-07,
      "logits/chosen": -0.05083770677447319,
      "logits/rejected": -0.0205841064453125,
      "logps/chosen": -177.95156860351562,
      "logps/rejected": -189.63436889648438,
      "logps/weighted_chosen": -1.188201904296875,
      "logps/weighted_rejected": -1.1756408214569092,
      "loss": 0.6926,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -62.192970275878906,
      "rewards/margins": 7.530859470367432,
      "rewards/rejected": -69.73905944824219,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.3717803955078125,
      "rewards/weighted_margins": 0.12430572509765625,
      "rewards/weighted_rejected": -0.49610137939453125,
      "step": 810
    },
    {
      "epoch": 0.43823902732313447,
      "grad_norm": 51.0267448425293,
      "learning_rate": 6.917725196300726e-07,
      "logits/chosen": -0.1438644379377365,
      "logits/rejected": -0.15563049912452698,
      "logps/chosen": -191.125,
      "logps/rejected": -203.52188110351562,
      "logps/weighted_chosen": -1.097863793373108,
      "logps/weighted_rejected": -0.971697986125946,
      "loss": 0.6974,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -63.764060974121094,
      "rewards/margins": 7.415234565734863,
      "rewards/rejected": -71.171875,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.3276786804199219,
      "rewards/weighted_margins": 0.08215713500976562,
      "rewards/weighted_rejected": -0.4100082516670227,
      "step": 820
    },
    {
      "epoch": 0.44358340570512395,
      "grad_norm": 20.933467864990234,
      "learning_rate": 6.831252474172411e-07,
      "logits/chosen": -0.05628051608800888,
      "logits/rejected": -0.03361053392291069,
      "logps/chosen": -176.0031280517578,
      "logps/rejected": -186.4343719482422,
      "logps/weighted_chosen": -1.6239410638809204,
      "logps/weighted_rejected": -1.3564453125,
      "loss": 0.6817,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -60.45234298706055,
      "rewards/margins": 6.214453220367432,
      "rewards/rejected": -66.66015625,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": -0.2794052064418793,
      "rewards/weighted_margins": 0.22185592353343964,
      "rewards/weighted_rejected": -0.500841498374939,
      "step": 830
    },
    {
      "epoch": 0.44892778408711337,
      "grad_norm": 11.393433570861816,
      "learning_rate": 6.74414244135898e-07,
      "logits/chosen": -0.15757599472999573,
      "logits/rejected": -0.116943359375,
      "logps/chosen": -177.0984344482422,
      "logps/rejected": -185.13436889648438,
      "logps/weighted_chosen": -0.666271984577179,
      "logps/weighted_rejected": -0.7827666997909546,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -60.349998474121094,
      "rewards/margins": 6.200390815734863,
      "rewards/rejected": -66.5289077758789,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.33733367919921875,
      "rewards/weighted_margins": 0.08320007473230362,
      "rewards/weighted_rejected": -0.42046278715133667,
      "step": 840
    },
    {
      "epoch": 0.4542721624691028,
      "grad_norm": 9.733312606811523,
      "learning_rate": 6.656425413808388e-07,
      "logits/chosen": -0.12353897094726562,
      "logits/rejected": -0.08713607490062714,
      "logps/chosen": -179.66561889648438,
      "logps/rejected": -183.109375,
      "logps/weighted_chosen": -1.0826873779296875,
      "logps/weighted_rejected": -1.106787085533142,
      "loss": 0.6875,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -60.806251525878906,
      "rewards/margins": 2.733203172683716,
      "rewards/rejected": -63.51250076293945,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.22345809638500214,
      "rewards/weighted_margins": 0.1886741667985916,
      "rewards/weighted_rejected": -0.4119575619697571,
      "step": 850
    },
    {
      "epoch": 0.45961654085109227,
      "grad_norm": 9.04237174987793,
      "learning_rate": 6.568131918714294e-07,
      "logits/chosen": -0.09643097221851349,
      "logits/rejected": -0.08455581963062286,
      "logps/chosen": -178.4453125,
      "logps/rejected": -187.99063110351562,
      "logps/weighted_chosen": -0.840087890625,
      "logps/weighted_rejected": -1.158935546875,
      "loss": 0.7138,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -61.162498474121094,
      "rewards/margins": 5.386328220367432,
      "rewards/rejected": -66.54765319824219,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.2982444763183594,
      "rewards/weighted_margins": 0.08280716091394424,
      "rewards/weighted_rejected": -0.38090819120407104,
      "step": 860
    },
    {
      "epoch": 0.4649609192330817,
      "grad_norm": 7.623125076293945,
      "learning_rate": 6.47929268389203e-07,
      "logits/chosen": -0.06428680568933487,
      "logits/rejected": -0.04297828674316406,
      "logps/chosen": -190.86093139648438,
      "logps/rejected": -205.9656219482422,
      "logps/weighted_chosen": -0.9338012933731079,
      "logps/weighted_rejected": -0.741748034954071,
      "loss": 0.6808,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -65.76640319824219,
      "rewards/margins": 7.994921684265137,
      "rewards/rejected": -73.75703430175781,
      "rewards/weighted_accuracies": 0.5,
      "rewards/weighted_chosen": -0.28187790513038635,
      "rewards/weighted_margins": 0.11656570434570312,
      "rewards/weighted_rejected": -0.39813536405563354,
      "step": 870
    },
    {
      "epoch": 0.47030529761507117,
      "grad_norm": 36.31256103515625,
      "learning_rate": 6.389938627084732e-07,
      "logits/chosen": -0.04478149488568306,
      "logits/rejected": -0.054534912109375,
      "logps/chosen": -181.5046844482422,
      "logps/rejected": -192.84375,
      "logps/weighted_chosen": -1.712792992591858,
      "logps/weighted_rejected": -1.4381287097930908,
      "loss": 0.7264,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -64.04609680175781,
      "rewards/margins": 6.507421970367432,
      "rewards/rejected": -70.56953430175781,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.3980148434638977,
      "rewards/weighted_margins": 0.10754700005054474,
      "rewards/weighted_rejected": -0.5054702758789062,
      "step": 880
    },
    {
      "epoch": 0.4756496759970606,
      "grad_norm": 6.0423126220703125,
      "learning_rate": 6.300100845203373e-07,
      "logits/chosen": -0.07687301933765411,
      "logits/rejected": -0.04276275634765625,
      "logps/chosen": -194.1609344482422,
      "logps/rejected": -200.828125,
      "logps/weighted_chosen": -0.6031554937362671,
      "logps/weighted_rejected": -0.86151123046875,
      "loss": 0.6793,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -63.5546875,
      "rewards/margins": 5.604296684265137,
      "rewards/rejected": -69.13984680175781,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.27074965834617615,
      "rewards/weighted_margins": 0.11608581244945526,
      "rewards/weighted_rejected": -0.38665771484375,
      "step": 890
    },
    {
      "epoch": 0.48099405437905,
      "grad_norm": 21.55549430847168,
      "learning_rate": 6.20981060350445e-07,
      "logits/chosen": -0.12602996826171875,
      "logits/rejected": -0.08534087985754013,
      "logps/chosen": -186.3312530517578,
      "logps/rejected": -190.0625,
      "logps/weighted_chosen": -0.8782455325126648,
      "logps/weighted_rejected": -1.0832672119140625,
      "loss": 0.6442,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -65.2984390258789,
      "rewards/margins": 6.321875095367432,
      "rewards/rejected": -71.61796569824219,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.3018638491630554,
      "rewards/weighted_margins": 0.19889220595359802,
      "rewards/weighted_rejected": -0.5003150701522827,
      "step": 900
    },
    {
      "epoch": 0.4863384327610395,
      "grad_norm": 15.255428314208984,
      "learning_rate": 6.119099324709087e-07,
      "logits/chosen": -0.08440399169921875,
      "logits/rejected": -0.03687591478228569,
      "logps/chosen": -183.1640625,
      "logps/rejected": -192.125,
      "logps/weighted_chosen": -0.7069732546806335,
      "logps/weighted_rejected": -1.348388671875,
      "loss": 0.7239,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -65.2796859741211,
      "rewards/margins": 7.408593654632568,
      "rewards/rejected": -72.6890640258789,
      "rewards/weighted_accuracies": 0.5093749761581421,
      "rewards/weighted_chosen": -0.3756294250488281,
      "rewards/weighted_margins": 0.05287017673254013,
      "rewards/weighted_rejected": -0.4286758303642273,
      "step": 910
    },
    {
      "epoch": 0.4916828111430289,
      "grad_norm": 10.355165481567383,
      "learning_rate": 6.027998578067316e-07,
      "logits/chosen": -0.06318511813879013,
      "logits/rejected": -0.02981414832174778,
      "logps/chosen": -199.36563110351562,
      "logps/rejected": -208.70938110351562,
      "logps/weighted_chosen": -0.6963897943496704,
      "logps/weighted_rejected": -0.969342052936554,
      "loss": 0.676,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -71.73905944824219,
      "rewards/margins": 6.481640815734863,
      "rewards/rejected": -78.2171859741211,
      "rewards/weighted_accuracies": 0.550000011920929,
      "rewards/weighted_chosen": -0.2880393862724304,
      "rewards/weighted_margins": 0.1389778107404709,
      "rewards/weighted_rejected": -0.426931768655777,
      "step": 920
    },
    {
      "epoch": 0.4970271895250184,
      "grad_norm": 5.530470371246338,
      "learning_rate": 5.936540068371394e-07,
      "logits/chosen": 0.006648254580795765,
      "logits/rejected": 0.005427551455795765,
      "logps/chosen": -193.99374389648438,
      "logps/rejected": -202.4406280517578,
      "logps/weighted_chosen": -1.107049584388733,
      "logps/weighted_rejected": -1.3529266119003296,
      "loss": 0.6926,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -74.79765319824219,
      "rewards/margins": 5.872656345367432,
      "rewards/rejected": -80.6624984741211,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.43673935532569885,
      "rewards/weighted_margins": 0.12621307373046875,
      "rewards/weighted_rejected": -0.5626624822616577,
      "step": 930
    },
    {
      "epoch": 0.5023715679070078,
      "grad_norm": 5.006593704223633,
      "learning_rate": 5.844755624921918e-07,
      "logits/chosen": -0.005947875790297985,
      "logits/rejected": 0.02864990197122097,
      "logps/chosen": -187.453125,
      "logps/rejected": -192.7375030517578,
      "logps/weighted_chosen": -1.132598876953125,
      "logps/weighted_rejected": -1.0424163341522217,
      "loss": 0.7378,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -70.77812194824219,
      "rewards/margins": 5.782812595367432,
      "rewards/rejected": -76.56562805175781,
      "rewards/weighted_accuracies": 0.512499988079071,
      "rewards/weighted_chosen": -0.44052886962890625,
      "rewards/weighted_margins": 0.07312469184398651,
      "rewards/weighted_rejected": -0.5137832760810852,
      "step": 940
    },
    {
      "epoch": 0.5077159462889973,
      "grad_norm": 16.502933502197266,
      "learning_rate": 5.752677190450634e-07,
      "logits/chosen": -0.021286392584443092,
      "logits/rejected": -0.00705032330006361,
      "logps/chosen": -186.49374389648438,
      "logps/rejected": -196.24844360351562,
      "logps/weighted_chosen": -1.068359375,
      "logps/weighted_rejected": -1.3232848644256592,
      "loss": 0.7251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -67.70234680175781,
      "rewards/margins": 7.561327934265137,
      "rewards/rejected": -75.28047180175781,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.390481561422348,
      "rewards/weighted_margins": 0.058994293212890625,
      "rewards/weighted_rejected": -0.44965285062789917,
      "step": 950
    },
    {
      "epoch": 0.5130603246709867,
      "grad_norm": 12.000792503356934,
      "learning_rate": 5.66033681000375e-07,
      "logits/chosen": -0.03893890231847763,
      "logits/rejected": 0.01745758019387722,
      "logps/chosen": -189.5234375,
      "logps/rejected": -201.2624969482422,
      "logps/weighted_chosen": -1.0573517084121704,
      "logps/weighted_rejected": -1.4250671863555908,
      "loss": 0.6656,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -69.40625,
      "rewards/margins": 8.408203125,
      "rewards/rejected": -77.78593444824219,
      "rewards/weighted_accuracies": 0.5718749761581421,
      "rewards/weighted_chosen": -0.2941024899482727,
      "rewards/weighted_margins": 0.1710365265607834,
      "rewards/weighted_rejected": -0.4653762876987457,
      "step": 960
    },
    {
      "epoch": 0.5184047030529761,
      "grad_norm": 8.829608917236328,
      "learning_rate": 5.567766619789658e-07,
      "logits/chosen": -0.03634033352136612,
      "logits/rejected": 0.0003803253057412803,
      "logps/chosen": -190.27969360351562,
      "logps/rejected": -199.0500030517578,
      "logps/weighted_chosen": -1.1300811767578125,
      "logps/weighted_rejected": -1.3106689453125,
      "loss": 0.6931,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -67.55078125,
      "rewards/margins": 7.454297065734863,
      "rewards/rejected": -74.9703140258789,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": -0.3190757632255554,
      "rewards/weighted_margins": 0.11787338554859161,
      "rewards/weighted_rejected": -0.4370986819267273,
      "step": 970
    },
    {
      "epoch": 0.5237490814349656,
      "grad_norm": 9.326337814331055,
      "learning_rate": 5.474998835994916e-07,
      "logits/chosen": 0.009844208136200905,
      "logits/rejected": 0.051239013671875,
      "logps/chosen": -195.78125,
      "logps/rejected": -208.5343780517578,
      "logps/weighted_chosen": -1.179107666015625,
      "logps/weighted_rejected": -1.546759009361267,
      "loss": 0.6677,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -70.6781234741211,
      "rewards/margins": 6.912109375,
      "rewards/rejected": -77.5609359741211,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.28845518827438354,
      "rewards/weighted_margins": 0.1378173828125,
      "rewards/weighted_rejected": -0.42629241943359375,
      "step": 980
    },
    {
      "epoch": 0.529093459816955,
      "grad_norm": 10.308549880981445,
      "learning_rate": 5.382065743572411e-07,
      "logits/chosen": -0.025909423828125,
      "logits/rejected": 0.01560287456959486,
      "logps/chosen": -192.8093719482422,
      "logps/rejected": -205.2843780517578,
      "logps/weighted_chosen": -0.8877533078193665,
      "logps/weighted_rejected": -1.2557891607284546,
      "loss": 0.6811,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -70.57890319824219,
      "rewards/margins": 8.194921493530273,
      "rewards/rejected": -78.80000305175781,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.3638969361782074,
      "rewards/weighted_margins": 0.13902893662452698,
      "rewards/weighted_rejected": -0.5028846859931946,
      "step": 990
    },
    {
      "epoch": 0.5344378381989445,
      "grad_norm": 6.612018585205078,
      "learning_rate": 5.288999685005571e-07,
      "logits/chosen": -0.03016967698931694,
      "logits/rejected": -0.0050643919967114925,
      "logps/chosen": -192.640625,
      "logps/rejected": -201.4656219482422,
      "logps/weighted_chosen": -0.626147449016571,
      "logps/weighted_rejected": -0.8550964593887329,
      "loss": 0.672,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -72.01640319824219,
      "rewards/margins": 6.489062309265137,
      "rewards/rejected": -78.5062484741211,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.2895965576171875,
      "rewards/weighted_margins": 0.1136070266366005,
      "rewards/weighted_rejected": -0.4031356871128082,
      "step": 1000
    },
    {
      "epoch": 0.5344378381989445,
      "eval_logits/chosen": -0.1623995155096054,
      "eval_logits/rejected": -0.13676175475120544,
      "eval_logps/chosen": -193.06211853027344,
      "eval_logps/rejected": -200.78919982910156,
      "eval_logps/weighted_chosen": -1.0030580759048462,
      "eval_logps/weighted_rejected": -1.3398514986038208,
      "eval_loss": 0.6716485619544983,
      "eval_rewards/accuracies": 0.5982688665390015,
      "eval_rewards/chosen": -71.73777770996094,
      "eval_rewards/margins": 6.556771755218506,
      "eval_rewards/rejected": -78.29378509521484,
      "eval_rewards/weighted_accuracies": 0.5570265054702759,
      "eval_rewards/weighted_chosen": -0.36391088366508484,
      "eval_rewards/weighted_margins": 0.17662912607192993,
      "eval_rewards/weighted_rejected": -0.5405399203300476,
      "eval_runtime": 1144.2369,
      "eval_samples_per_second": 1.714,
      "eval_steps_per_second": 0.429,
      "step": 1000
    }
  ],
  "logging_steps": 10,
  "max_steps": 1872,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}