{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.26721891909947226,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005344378381989445,
      "grad_norm": 63.52920150756836,
      "learning_rate": 0.0,
      "logits/chosen": -0.144775390625,
      "logits/rejected": -0.1036376953125,
      "logps/chosen": -110.828125,
      "logps/rejected": -116.828125,
      "logps/weighted_chosen": -0.2086181640625,
      "logps/weighted_rejected": -1.271392822265625,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/weighted_accuracies": 0.0,
      "rewards/weighted_chosen": 0.0,
      "rewards/weighted_margins": 0.0,
      "rewards/weighted_rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005344378381989445,
      "grad_norm": 102.77810668945312,
      "learning_rate": 4.787234042553191e-08,
      "logits/chosen": -0.2549913227558136,
      "logits/rejected": -0.24070316553115845,
      "logps/chosen": -135.32899475097656,
      "logps/rejected": -138.99305725097656,
      "logps/weighted_chosen": -0.8547499179840088,
      "logps/weighted_rejected": -0.9027608036994934,
      "loss": 0.6928,
      "rewards/accuracies": 0.2951388955116272,
      "rewards/chosen": -0.0164930559694767,
      "rewards/margins": -0.0349392369389534,
      "rewards/rejected": 0.0184461809694767,
      "rewards/weighted_accuracies": 0.4027777910232544,
      "rewards/weighted_chosen": -0.002239651046693325,
      "rewards/weighted_margins": 0.0028160943184047937,
      "rewards/weighted_rejected": -0.0050794812850654125,
      "step": 10
    },
    {
      "epoch": 0.01068875676397889,
      "grad_norm": 9.201544761657715,
      "learning_rate": 1.0106382978723404e-07,
      "logits/chosen": -0.19488906860351562,
      "logits/rejected": -0.183399960398674,
      "logps/chosen": -116.0374984741211,
      "logps/rejected": -115.5054702758789,
      "logps/weighted_chosen": -0.6929916143417358,
      "logps/weighted_rejected": -0.6838119626045227,
      "loss": 0.6917,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.01796874962747097,
      "rewards/margins": 0.03007812425494194,
      "rewards/rejected": -0.01210937462747097,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": 0.0022293091751635075,
      "rewards/weighted_margins": 0.00339679722674191,
      "rewards/weighted_rejected": -0.0011629104847088456,
      "step": 20
    },
    {
      "epoch": 0.016033135145968335,
      "grad_norm": 7.109442234039307,
      "learning_rate": 1.5425531914893615e-07,
      "logits/chosen": -0.1776607483625412,
      "logits/rejected": -0.17884215712547302,
      "logps/chosen": -123.2867202758789,
      "logps/rejected": -129.75,
      "logps/weighted_chosen": -0.7567901611328125,
      "logps/weighted_rejected": -1.2207733392715454,
      "loss": 0.6913,
      "rewards/accuracies": 0.3968749940395355,
      "rewards/chosen": -0.0074218749068677425,
      "rewards/margins": 0.04843749850988388,
      "rewards/rejected": -0.05585937574505806,
      "rewards/weighted_accuracies": 0.5,
      "rewards/weighted_chosen": 0.0007101058727130294,
      "rewards/weighted_margins": 0.004384422209113836,
      "rewards/weighted_rejected": -0.0036745071411132812,
      "step": 30
    },
    {
      "epoch": 0.02137751352795778,
      "grad_norm": 6.0048956871032715,
      "learning_rate": 2.074468085106383e-07,
      "logits/chosen": -0.23842163383960724,
      "logits/rejected": -0.2298881560564041,
      "logps/chosen": -127.34062194824219,
      "logps/rejected": -128.1710968017578,
      "logps/weighted_chosen": -0.8051223754882812,
      "logps/weighted_rejected": -0.5736129879951477,
      "loss": 0.6934,
      "rewards/accuracies": 0.34062498807907104,
      "rewards/chosen": -0.01992187462747097,
      "rewards/margins": -0.05351562425494194,
      "rewards/rejected": 0.03359375149011612,
      "rewards/weighted_accuracies": 0.4593749940395355,
      "rewards/weighted_chosen": -7.83920258982107e-05,
      "rewards/weighted_margins": -0.0009832382202148438,
      "rewards/weighted_rejected": 0.000904083251953125,
      "step": 40
    },
    {
      "epoch": 0.026721891909947223,
      "grad_norm": 32.7233772277832,
      "learning_rate": 2.6063829787234044e-07,
      "logits/chosen": -0.18835143744945526,
      "logits/rejected": -0.170085147023201,
      "logps/chosen": -120.88594055175781,
      "logps/rejected": -121.3515625,
      "logps/weighted_chosen": -0.6867736577987671,
      "logps/weighted_rejected": -1.138249158859253,
      "loss": 0.6903,
      "rewards/accuracies": 0.3656249940395355,
      "rewards/chosen": 0.03554687649011612,
      "rewards/margins": -0.03164062649011612,
      "rewards/rejected": 0.06718750298023224,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.006287765689194202,
      "rewards/weighted_margins": 0.0064636231400072575,
      "rewards/weighted_rejected": -0.01274185162037611,
      "step": 50
    },
    {
      "epoch": 0.03206627029193667,
      "grad_norm": 55.560707092285156,
      "learning_rate": 3.1382978723404253e-07,
      "logits/chosen": -0.1308601349592209,
      "logits/rejected": -0.1308906525373459,
      "logps/chosen": -116.5875015258789,
      "logps/rejected": -119.5406265258789,
      "logps/weighted_chosen": -0.8019469976425171,
      "logps/weighted_rejected": -0.8834930658340454,
      "loss": 0.7002,
      "rewards/accuracies": 0.3343749940395355,
      "rewards/chosen": 0.03476562350988388,
      "rewards/margins": -0.06171875074505806,
      "rewards/rejected": 0.09648437798023224,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.01903533935546875,
      "rewards/weighted_margins": -2.880096508306451e-05,
      "rewards/weighted_rejected": -0.019007300958037376,
      "step": 60
    },
    {
      "epoch": 0.037410648673926114,
      "grad_norm": 20.22484588623047,
      "learning_rate": 3.6702127659574467e-07,
      "logits/chosen": -0.20572929084300995,
      "logits/rejected": -0.19457244873046875,
      "logps/chosen": -114.4507827758789,
      "logps/rejected": -118.63594055175781,
      "logps/weighted_chosen": -0.6070801019668579,
      "logps/weighted_rejected": -0.6768280267715454,
      "loss": 0.6911,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.16484375298023224,
      "rewards/margins": 0.07148437201976776,
      "rewards/rejected": 0.09335937350988388,
      "rewards/weighted_accuracies": 0.4749999940395355,
      "rewards/weighted_chosen": -0.010281562805175781,
      "rewards/weighted_margins": 0.007259177975356579,
      "rewards/weighted_rejected": -0.017521286383271217,
      "step": 70
    },
    {
      "epoch": 0.04275502705591556,
      "grad_norm": 15.305061340332031,
      "learning_rate": 4.202127659574468e-07,
      "logits/chosen": -0.19508513808250427,
      "logits/rejected": -0.12093047797679901,
      "logps/chosen": -112.57734680175781,
      "logps/rejected": -112.77734375,
      "logps/weighted_chosen": -0.4597320556640625,
      "logps/weighted_rejected": -0.9711883664131165,
      "loss": 0.6952,
      "rewards/accuracies": 0.37812501192092896,
      "rewards/chosen": 0.04570312425494194,
      "rewards/margins": 0.01914062537252903,
      "rewards/rejected": 0.02656250074505806,
      "rewards/weighted_accuracies": 0.4625000059604645,
      "rewards/weighted_chosen": -0.02759857103228569,
      "rewards/weighted_margins": 0.021048355847597122,
      "rewards/weighted_rejected": -0.04872245714068413,
      "step": 80
    },
    {
      "epoch": 0.048099405437905,
      "grad_norm": 11.991273880004883,
      "learning_rate": 4.734042553191489e-07,
      "logits/chosen": -0.17391815781593323,
      "logits/rejected": -0.11656417697668076,
      "logps/chosen": -119.3070297241211,
      "logps/rejected": -118.51249694824219,
      "logps/weighted_chosen": -1.0416854619979858,
      "logps/weighted_rejected": -1.5859466791152954,
      "loss": 0.6736,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08164062350988388,
      "rewards/margins": 0.341796875,
      "rewards/rejected": -0.4234375059604645,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.04084587097167969,
      "rewards/weighted_margins": 0.10567589104175568,
      "rewards/weighted_rejected": -0.14661255478858948,
      "step": 90
    },
    {
      "epoch": 0.053443783819894446,
      "grad_norm": 17.195886611938477,
      "learning_rate": 5.26595744680851e-07,
      "logits/chosen": -0.204203799366951,
      "logits/rejected": -0.14691162109375,
      "logps/chosen": -125.05155944824219,
      "logps/rejected": -125.3671875,
      "logps/weighted_chosen": -0.7374237179756165,
      "logps/weighted_rejected": -0.7348953485488892,
      "loss": 0.7064,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.3832031190395355,
      "rewards/margins": 0.15585938096046448,
      "rewards/rejected": -0.5390625,
      "rewards/weighted_accuracies": 0.46875,
      "rewards/weighted_chosen": -0.060018159449100494,
      "rewards/weighted_margins": 0.0024120330344885588,
      "rewards/weighted_rejected": -0.06242714077234268,
      "step": 100
    },
    {
      "epoch": 0.058788162201883896,
      "grad_norm": 9.103920936584473,
      "learning_rate": 5.797872340425531e-07,
      "logits/chosen": -0.2138931304216385,
      "logits/rejected": -0.19011077284812927,
      "logps/chosen": -118.6820297241211,
      "logps/rejected": -121.28593444824219,
      "logps/weighted_chosen": -0.4821228086948395,
      "logps/weighted_rejected": -0.7759445309638977,
      "loss": 0.6939,
      "rewards/accuracies": 0.4281249940395355,
      "rewards/chosen": -0.896484375,
      "rewards/margins": 0.09531249850988388,
      "rewards/rejected": -0.9917968511581421,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.05413341522216797,
      "rewards/weighted_margins": 0.04503212124109268,
      "rewards/weighted_rejected": -0.09918193519115448,
      "step": 110
    },
    {
      "epoch": 0.06413254058387334,
      "grad_norm": 8.539896011352539,
      "learning_rate": 6.329787234042553e-07,
      "logits/chosen": -0.17510680854320526,
      "logits/rejected": -0.14726562798023224,
      "logps/chosen": -122.58906555175781,
      "logps/rejected": -128.4656219482422,
      "logps/weighted_chosen": -0.9249496459960938,
      "logps/weighted_rejected": -0.762683093547821,
      "loss": 0.7325,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.068359375,
      "rewards/margins": 0.28398436307907104,
      "rewards/rejected": -1.3523437976837158,
      "rewards/weighted_accuracies": 0.4781250059604645,
      "rewards/weighted_chosen": -0.098027803003788,
      "rewards/weighted_margins": -0.029563140124082565,
      "rewards/weighted_rejected": -0.06836700439453125,
      "step": 120
    },
    {
      "epoch": 0.06947691896586278,
      "grad_norm": 15.564044952392578,
      "learning_rate": 6.861702127659574e-07,
      "logits/chosen": -0.20233002305030823,
      "logits/rejected": -0.18396148085594177,
      "logps/chosen": -122.2671890258789,
      "logps/rejected": -123.8382797241211,
      "logps/weighted_chosen": -0.7501281499862671,
      "logps/weighted_rejected": -0.821154773235321,
      "loss": 0.7078,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -1.45703125,
      "rewards/margins": 0.29179686307907104,
      "rewards/rejected": -1.7488281726837158,
      "rewards/weighted_accuracies": 0.5062500238418579,
      "rewards/weighted_chosen": -0.007884788326919079,
      "rewards/weighted_margins": 0.006314086727797985,
      "rewards/weighted_rejected": -0.014200759120285511,
      "step": 130
    },
    {
      "epoch": 0.07482129734785223,
      "grad_norm": 9.176558494567871,
      "learning_rate": 7.393617021276596e-07,
      "logits/chosen": -0.16455383598804474,
      "logits/rejected": -0.14660033583641052,
      "logps/chosen": -116.328125,
      "logps/rejected": -119.9664077758789,
      "logps/weighted_chosen": -0.9873703122138977,
      "logps/weighted_rejected": -0.6730011105537415,
      "loss": 0.7174,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.6007812023162842,
      "rewards/margins": 0.533203125,
      "rewards/rejected": -2.133984327316284,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": 0.02084655687212944,
      "rewards/weighted_margins": 0.05102996900677681,
      "rewards/weighted_rejected": -0.030106354504823685,
      "step": 140
    },
    {
      "epoch": 0.08016567572984168,
      "grad_norm": 10.172534942626953,
      "learning_rate": 7.925531914893616e-07,
      "logits/chosen": -0.1509498655796051,
      "logits/rejected": -0.13156357407569885,
      "logps/chosen": -120.0562515258789,
      "logps/rejected": -117.85546875,
      "logps/weighted_chosen": -0.8198806643486023,
      "logps/weighted_rejected": -0.7782562375068665,
      "loss": 0.7005,
      "rewards/accuracies": 0.46562498807907104,
      "rewards/chosen": -2.0234375,
      "rewards/margins": 0.607421875,
      "rewards/rejected": -2.630859375,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.0781441181898117,
      "rewards/weighted_margins": 0.034273337572813034,
      "rewards/weighted_rejected": -0.11248989403247833,
      "step": 150
    },
    {
      "epoch": 0.08551005411183112,
      "grad_norm": 25.879701614379883,
      "learning_rate": 8.457446808510637e-07,
      "logits/chosen": -0.17474937438964844,
      "logits/rejected": -0.17009583115577698,
      "logps/chosen": -121.2367172241211,
      "logps/rejected": -121.62109375,
      "logps/weighted_chosen": -0.8304061889648438,
      "logps/weighted_rejected": -0.866925060749054,
      "loss": 0.7072,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -2.219531297683716,
      "rewards/margins": 0.716796875,
      "rewards/rejected": -2.936328172683716,
      "rewards/weighted_accuracies": 0.47187501192092896,
      "rewards/weighted_chosen": -0.07932891696691513,
      "rewards/weighted_margins": 0.017035294324159622,
      "rewards/weighted_rejected": -0.09644661098718643,
      "step": 160
    },
    {
      "epoch": 0.09085443249382057,
      "grad_norm": 9.220558166503906,
      "learning_rate": 8.989361702127659e-07,
      "logits/chosen": -0.22574539482593536,
      "logits/rejected": -0.19882050156593323,
      "logps/chosen": -120.62968444824219,
      "logps/rejected": -124.78437805175781,
      "logps/weighted_chosen": -0.9401000738143921,
      "logps/weighted_rejected": -0.8424423336982727,
      "loss": 0.7174,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -2.4033203125,
      "rewards/margins": 0.613085925579071,
      "rewards/rejected": -3.016406297683716,
      "rewards/weighted_accuracies": 0.48750001192092896,
      "rewards/weighted_chosen": -0.04837837070226669,
      "rewards/weighted_margins": 0.04595222324132919,
      "rewards/weighted_rejected": -0.09437103569507599,
      "step": 170
    },
    {
      "epoch": 0.09619881087581,
      "grad_norm": 55.62498092651367,
      "learning_rate": 9.52127659574468e-07,
      "logits/chosen": -0.2216789275407791,
      "logits/rejected": -0.20373383164405823,
      "logps/chosen": -120.3851547241211,
      "logps/rejected": -121.2328109741211,
      "logps/weighted_chosen": -0.435232549905777,
      "logps/weighted_rejected": -0.5373809933662415,
      "loss": 0.7302,
      "rewards/accuracies": 0.47187501192092896,
      "rewards/chosen": -1.338281273841858,
      "rewards/margins": 0.3658203184604645,
      "rewards/rejected": -1.7041015625,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.08163909614086151,
      "rewards/weighted_margins": -0.008893203921616077,
      "rewards/weighted_rejected": -0.07281456142663956,
      "step": 180
    },
    {
      "epoch": 0.10154318925779945,
      "grad_norm": 39.38788986206055,
      "learning_rate": 9.99999129927477e-07,
      "logits/chosen": -0.19243164360523224,
      "logits/rejected": -0.15758666396141052,
      "logps/chosen": -121.8960952758789,
      "logps/rejected": -123.01094055175781,
      "logps/weighted_chosen": -0.953173816204071,
      "logps/weighted_rejected": -1.0853087902069092,
      "loss": 0.6702,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.100390672683716,
      "rewards/margins": 0.662890613079071,
      "rewards/rejected": -2.7632813453674316,
      "rewards/weighted_accuracies": 0.578125,
      "rewards/weighted_chosen": -0.09999503940343857,
      "rewards/weighted_margins": 0.12822990119457245,
      "rewards/weighted_rejected": -0.22818031907081604,
      "step": 190
    },
    {
      "epoch": 0.10688756763978889,
      "grad_norm": 12.883384704589844,
      "learning_rate": 9.99894724888679e-07,
      "logits/chosen": -0.17392578721046448,
      "logits/rejected": -0.147491455078125,
      "logps/chosen": -129.53750610351562,
      "logps/rejected": -128.95703125,
      "logps/weighted_chosen": -0.7322753667831421,
      "logps/weighted_rejected": -1.0728164911270142,
      "loss": 0.6777,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.215625047683716,
      "rewards/margins": 0.9507812261581421,
      "rewards/rejected": -4.166406154632568,
      "rewards/weighted_accuracies": 0.5375000238418579,
      "rewards/weighted_chosen": -0.1104557067155838,
      "rewards/weighted_margins": 0.13128623366355896,
      "rewards/weighted_rejected": -0.2415626496076584,
      "step": 200
    },
    {
      "epoch": 0.11223194602177834,
      "grad_norm": 17.96660041809082,
      "learning_rate": 9.996163469793475e-07,
      "logits/chosen": -0.18083801865577698,
      "logits/rejected": -0.12766647338867188,
      "logps/chosen": -128.1953125,
      "logps/rejected": -123.03047180175781,
      "logps/weighted_chosen": -0.8699920773506165,
      "logps/weighted_rejected": -0.8845611810684204,
      "loss": 0.7572,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.5113282203674316,
      "rewards/margins": 1.3683593273162842,
      "rewards/rejected": -4.879296779632568,
      "rewards/weighted_accuracies": 0.5531250238418579,
      "rewards/weighted_chosen": -0.07919616997241974,
      "rewards/weighted_margins": 0.04895629733800888,
      "rewards/weighted_rejected": -0.12815475463867188,
      "step": 210
    },
    {
      "epoch": 0.11757632440376779,
      "grad_norm": 63.790077209472656,
      "learning_rate": 9.991640930802883e-07,
      "logits/chosen": -0.14619751274585724,
      "logits/rejected": -0.13860473036766052,
      "logps/chosen": -126.61015319824219,
      "logps/rejected": -129.9773406982422,
      "logps/weighted_chosen": -1.0776779651641846,
      "logps/weighted_rejected": -0.9309722781181335,
      "loss": 0.7107,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -4.296484470367432,
      "rewards/margins": 1.10546875,
      "rewards/rejected": -5.40234375,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.013729858212172985,
      "rewards/weighted_margins": 0.14467468857765198,
      "rewards/weighted_rejected": -0.1582542359828949,
      "step": 220
    },
    {
      "epoch": 0.12292070278575723,
      "grad_norm": 9.634794235229492,
      "learning_rate": 9.98538120584459e-07,
      "logits/chosen": -0.1792152374982834,
      "logits/rejected": -0.1362098753452301,
      "logps/chosen": -133.7140655517578,
      "logps/rejected": -137.7546844482422,
      "logps/weighted_chosen": -0.6676223874092102,
      "logps/weighted_rejected": -0.9518234133720398,
      "loss": 0.7614,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -4.493554592132568,
      "rewards/margins": 1.0623047351837158,
      "rewards/rejected": -5.555859565734863,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.1120578795671463,
      "rewards/weighted_margins": 0.031700897961854935,
      "rewards/weighted_rejected": -0.14376182854175568,
      "step": 230
    },
    {
      "epoch": 0.12826508116774668,
      "grad_norm": 8.717682838439941,
      "learning_rate": 9.977386473421917e-07,
      "logits/chosen": -0.09295501559972763,
      "logits/rejected": -0.07836151123046875,
      "logps/chosen": -121.78202819824219,
      "logps/rejected": -126.39453125,
      "logps/weighted_chosen": -0.8116515874862671,
      "logps/weighted_rejected": -1.2165496349334717,
      "loss": 0.6731,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -7.553515434265137,
      "rewards/margins": 1.669921875,
      "rewards/rejected": -9.224218368530273,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.1755084991455078,
      "rewards/weighted_margins": 0.15414047241210938,
      "rewards/weighted_rejected": -0.32989272475242615,
      "step": 240
    },
    {
      "epoch": 0.13360945954973613,
      "grad_norm": 35.197269439697266,
      "learning_rate": 9.96765951585378e-07,
      "logits/chosen": -0.021549224853515625,
      "logits/rejected": 0.00345611572265625,
      "logps/chosen": -123.0296859741211,
      "logps/rejected": -127.9671859741211,
      "logps/weighted_chosen": -1.186151146888733,
      "logps/weighted_rejected": -1.386474609375,
      "loss": 0.7026,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -9.133593559265137,
      "rewards/margins": 1.150781273841858,
      "rewards/rejected": -10.283984184265137,
      "rewards/weighted_accuracies": 0.59375,
      "rewards/weighted_chosen": -0.3282794952392578,
      "rewards/weighted_margins": 0.1091407760977745,
      "rewards/weighted_rejected": -0.4371795654296875,
      "step": 250
    },
    {
      "epoch": 0.13895383793172555,
      "grad_norm": 84.2197494506836,
      "learning_rate": 9.956203718306388e-07,
      "logits/chosen": 0.025267411023378372,
      "logits/rejected": 0.06335143744945526,
      "logps/chosen": -128.8429718017578,
      "logps/rejected": -133.50625610351562,
      "logps/weighted_chosen": -0.9628280401229858,
      "logps/weighted_rejected": -1.2008453607559204,
      "loss": 0.7443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -10.861719131469727,
      "rewards/margins": 0.871874988079071,
      "rewards/rejected": -11.733593940734863,
      "rewards/weighted_accuracies": 0.49687498807907104,
      "rewards/weighted_chosen": -0.26259079575538635,
      "rewards/weighted_margins": 0.06770706176757812,
      "rewards/weighted_rejected": -0.3305191099643707,
      "step": 260
    },
    {
      "epoch": 0.144298216313715,
      "grad_norm": 57.09628677368164,
      "learning_rate": 9.943023067615136e-07,
      "logits/chosen": -0.017973709851503372,
      "logits/rejected": -0.0057998658157885075,
      "logps/chosen": -137.60311889648438,
      "logps/rejected": -140.91561889648438,
      "logps/weighted_chosen": -0.8694183230400085,
      "logps/weighted_rejected": -0.8910156488418579,
      "loss": 0.7462,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -10.946093559265137,
      "rewards/margins": 1.662109375,
      "rewards/rejected": -12.608202934265137,
      "rewards/weighted_accuracies": 0.565625011920929,
      "rewards/weighted_chosen": -0.06702041625976562,
      "rewards/weighted_margins": 0.13949890434741974,
      "rewards/weighted_rejected": -0.20627327263355255,
      "step": 270
    },
    {
      "epoch": 0.14964259469570446,
      "grad_norm": 45.0294303894043,
      "learning_rate": 9.928122150897112e-07,
      "logits/chosen": -0.045014191418886185,
      "logits/rejected": 0.02437591552734375,
      "logps/chosen": -130.1125030517578,
      "logps/rejected": -131.6171875,
      "logps/weighted_chosen": -0.7659690976142883,
      "logps/weighted_rejected": -0.8499618768692017,
      "loss": 0.7079,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -12.664453506469727,
      "rewards/margins": 1.584375023841858,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.22265930473804474,
      "rewards/weighted_margins": 0.07556991279125214,
      "rewards/weighted_rejected": -0.298013299703598,
      "step": 280
    },
    {
      "epoch": 0.1549869730776939,
      "grad_norm": 9.370684623718262,
      "learning_rate": 9.9115061539547e-07,
      "logits/chosen": -0.02069244347512722,
      "logits/rejected": 0.01569976843893528,
      "logps/chosen": -139.4453125,
      "logps/rejected": -141.2578125,
      "logps/weighted_chosen": -0.8775177001953125,
      "logps/weighted_rejected": -1.2375701665878296,
      "loss": 0.6949,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -13.07421875,
      "rewards/margins": 1.177734375,
      "rewards/rejected": -14.249608993530273,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.18484345078468323,
      "rewards/weighted_margins": 0.12048187106847763,
      "rewards/weighted_rejected": -0.30528029799461365,
      "step": 290
    },
    {
      "epoch": 0.16033135145968336,
      "grad_norm": 30.68979263305664,
      "learning_rate": 9.893180859470818e-07,
      "logits/chosen": -0.0137786865234375,
      "logits/rejected": 0.041573334485292435,
      "logps/chosen": -130.22265625,
      "logps/rejected": -132.06094360351562,
      "logps/weighted_chosen": -1.1769134998321533,
      "logps/weighted_rejected": -1.2438628673553467,
      "loss": 0.7261,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -13.093358993530273,
      "rewards/margins": 2.108203172683716,
      "rewards/rejected": -15.200780868530273,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.2848831117153168,
      "rewards/weighted_margins": 0.07841148227453232,
      "rewards/weighted_rejected": -0.36317139863967896,
      "step": 300
    },
    {
      "epoch": 0.16567572984167278,
      "grad_norm": 71.56196594238281,
      "learning_rate": 9.873152644996424e-07,
      "logits/chosen": -0.11114044487476349,
      "logits/rejected": -0.11599807441234589,
      "logps/chosen": -131.01718139648438,
      "logps/rejected": -132.28125,
      "logps/weighted_chosen": -1.1032531261444092,
      "logps/weighted_rejected": -0.8420807123184204,
      "loss": 0.7872,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -12.979687690734863,
      "rewards/margins": 2.407421827316284,
      "rewards/rejected": -15.388280868530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": 0.05532989650964737,
      "rewards/weighted_margins": 0.2024696320295334,
      "rewards/weighted_rejected": -0.1469375640153885,
      "step": 310
    },
    {
      "epoch": 0.17102010822366223,
      "grad_norm": 12.055798530578613,
      "learning_rate": 9.85142848073103e-07,
      "logits/chosen": -0.10557708889245987,
      "logits/rejected": -0.06234588474035263,
      "logps/chosen": -133.39218139648438,
      "logps/rejected": -134.2109375,
      "logps/weighted_chosen": -0.700244128704071,
      "logps/weighted_rejected": -1.4902985095977783,
      "loss": 0.7558,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -14.2265625,
      "rewards/margins": 0.6207031011581421,
      "rewards/rejected": -14.846094131469727,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.2170669585466385,
      "rewards/weighted_margins": 0.03446922451257706,
      "rewards/weighted_rejected": -0.25187987089157104,
      "step": 320
    },
    {
      "epoch": 0.17636448660565168,
      "grad_norm": 41.381507873535156,
      "learning_rate": 9.828015927096914e-07,
      "logits/chosen": -0.085169218480587,
      "logits/rejected": -0.05568351596593857,
      "logps/chosen": -143.8234405517578,
      "logps/rejected": -142.39608764648438,
      "logps/weighted_chosen": -0.977459728717804,
      "logps/weighted_rejected": -0.7958831787109375,
      "loss": 0.6935,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -14.643750190734863,
      "rewards/margins": 1.578125,
      "rewards/rejected": -16.222265243530273,
      "rewards/weighted_accuracies": 0.581250011920929,
      "rewards/weighted_chosen": -0.16206398606300354,
      "rewards/weighted_margins": 0.07153511047363281,
      "rewards/weighted_rejected": -0.23361511528491974,
      "step": 330
    },
    {
      "epoch": 0.18170886498764113,
      "grad_norm": 7.313599586486816,
      "learning_rate": 9.802923132107968e-07,
      "logits/chosen": -0.10784301906824112,
      "logits/rejected": -0.052645109593868256,
      "logps/chosen": -143.91561889648438,
      "logps/rejected": -147.7234344482422,
      "logps/weighted_chosen": -0.7267853021621704,
      "logps/weighted_rejected": -1.24786376953125,
      "loss": 0.7102,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -16.782812118530273,
      "rewards/margins": 1.23046875,
      "rewards/rejected": -18.012500762939453,
      "rewards/weighted_accuracies": 0.5687500238418579,
      "rewards/weighted_chosen": -0.1999809294939041,
      "rewards/weighted_margins": 0.056002043187618256,
      "rewards/weighted_rejected": -0.2561977505683899,
      "step": 340
    },
    {
      "epoch": 0.18705324336963056,
      "grad_norm": 66.15936279296875,
      "learning_rate": 9.776158828534024e-07,
      "logits/chosen": -0.12608642876148224,
      "logits/rejected": -0.09711913764476776,
      "logps/chosen": -143.48281860351562,
      "logps/rejected": -188.25936889648438,
      "logps/weighted_chosen": -1.260833740234375,
      "logps/weighted_rejected": -0.9638397097587585,
      "loss": 0.7338,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -16.727733612060547,
      "rewards/margins": 3.159374952316284,
      "rewards/rejected": -19.883594512939453,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": -0.26622849702835083,
      "rewards/weighted_margins": 0.05975799635052681,
      "rewards/weighted_rejected": -0.3257743716239929,
      "step": 350
    },
    {
      "epoch": 0.19239762175162,
      "grad_norm": 10.4098539352417,
      "learning_rate": 9.747732330861695e-07,
      "logits/chosen": -0.05247039720416069,
      "logits/rejected": -0.024548912420868874,
      "logps/chosen": -135.78750610351562,
      "logps/rejected": -137.03671264648438,
      "logps/weighted_chosen": -0.8440704345703125,
      "logps/weighted_rejected": -0.8307411074638367,
      "loss": 0.7415,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -16.864452362060547,
      "rewards/margins": 2.5316405296325684,
      "rewards/rejected": -19.396093368530273,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": 0.005719375796616077,
      "rewards/weighted_margins": 0.17883948981761932,
      "rewards/weighted_rejected": -0.17323684692382812,
      "step": 360
    },
    {
      "epoch": 0.19774200013360946,
      "grad_norm": 8.8024263381958,
      "learning_rate": 9.717653532052742e-07,
      "logits/chosen": -0.0410335548222065,
      "logits/rejected": -0.049485016614198685,
      "logps/chosen": -130.2781219482422,
      "logps/rejected": -137.9343719482422,
      "logps/weighted_chosen": -1.1705901622772217,
      "logps/weighted_rejected": -0.7656036615371704,
      "loss": 0.7402,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -17.155858993530273,
      "rewards/margins": 2.863085985183716,
      "rewards/rejected": -20.016016006469727,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": -0.07308082282543182,
      "rewards/weighted_margins": 0.04105224460363388,
      "rewards/weighted_rejected": -0.114354707300663,
      "step": 370
    },
    {
      "epoch": 0.2030863785155989,
      "grad_norm": 26.81167221069336,
      "learning_rate": 9.685932900101146e-07,
      "logits/chosen": -0.07604064792394638,
      "logits/rejected": -0.06302948296070099,
      "logps/chosen": -134.7859344482422,
      "logps/rejected": -137.11093139648438,
      "logps/weighted_chosen": -0.98638916015625,
      "logps/weighted_rejected": -0.6243560910224915,
      "loss": 0.7116,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.729297637939453,
      "rewards/margins": 1.8025391101837158,
      "rewards/rejected": -19.528514862060547,
      "rewards/weighted_accuracies": 0.546875,
      "rewards/weighted_chosen": -0.11945419013500214,
      "rewards/weighted_margins": 0.07185058295726776,
      "rewards/weighted_rejected": -0.1911766082048416,
      "step": 380
    },
    {
      "epoch": 0.20843075689758836,
      "grad_norm": 40.89613342285156,
      "learning_rate": 9.652581474390043e-07,
      "logits/chosen": -0.09274902194738388,
      "logits/rejected": -0.05452308803796768,
      "logps/chosen": -139.3249969482422,
      "logps/rejected": -141.53750610351562,
      "logps/weighted_chosen": -0.9199386835098267,
      "logps/weighted_rejected": -0.9217513799667358,
      "loss": 0.7323,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -19.379688262939453,
      "rewards/margins": 2.321093797683716,
      "rewards/rejected": -21.702733993530273,
      "rewards/weighted_accuracies": 0.543749988079071,
      "rewards/weighted_chosen": -0.05511016771197319,
      "rewards/weighted_margins": 0.04057302325963974,
      "rewards/weighted_rejected": -0.09564266353845596,
      "step": 390
    },
    {
      "epoch": 0.21377513527957778,
      "grad_norm": 51.860347747802734,
      "learning_rate": 9.61761086184981e-07,
      "logits/chosen": -0.10268402099609375,
      "logits/rejected": -0.05923309177160263,
      "logps/chosen": -139.50936889648438,
      "logps/rejected": -141.4015655517578,
      "logps/weighted_chosen": -0.9449096918106079,
      "logps/weighted_rejected": -1.177301049232483,
      "loss": 0.7387,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -20.024608612060547,
      "rewards/margins": 2.9205079078674316,
      "rewards/rejected": -22.942577362060547,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": -0.26060789823532104,
      "rewards/weighted_margins": 0.017241667956113815,
      "rewards/weighted_rejected": -0.2779647707939148,
      "step": 400
    },
    {
      "epoch": 0.21911951366156723,
      "grad_norm": 7.539150238037109,
      "learning_rate": 9.581033232918629e-07,
      "logits/chosen": -0.015906523913145065,
      "logits/rejected": 0.02000865899026394,
      "logps/chosen": -138.39999389648438,
      "logps/rejected": -141.68905639648438,
      "logps/weighted_chosen": -1.0313812494277954,
      "logps/weighted_rejected": -1.1196472644805908,
      "loss": 0.7283,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -21.702342987060547,
      "rewards/margins": 2.826953172683716,
      "rewards/rejected": -24.53125,
      "rewards/weighted_accuracies": 0.5249999761581421,
      "rewards/weighted_chosen": -0.2783859372138977,
      "rewards/weighted_margins": 0.06903991848230362,
      "rewards/weighted_rejected": -0.3477470278739929,
      "step": 410
    },
    {
      "epoch": 0.22446389204355668,
      "grad_norm": 6.638446807861328,
      "learning_rate": 9.542861317306952e-07,
      "logits/chosen": -0.05767669528722763,
      "logits/rejected": -0.03759918361902237,
      "logps/chosen": -140.64688110351562,
      "logps/rejected": -141.9718780517578,
      "logps/weighted_chosen": -1.5940430164337158,
      "logps/weighted_rejected": -1.458154320716858,
      "loss": 0.7371,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -22.33203125,
      "rewards/margins": 1.1359374523162842,
      "rewards/rejected": -23.46484375,
      "rewards/weighted_accuracies": 0.515625,
      "rewards/weighted_chosen": -0.299835205078125,
      "rewards/weighted_margins": 0.032170869410037994,
      "rewards/weighted_rejected": -0.3319748044013977,
      "step": 420
    },
    {
      "epoch": 0.22980827042554614,
      "grad_norm": 24.621816635131836,
      "learning_rate": 9.503108399567308e-07,
      "logits/chosen": -0.15150222182273865,
      "logits/rejected": -0.09339599311351776,
      "logps/chosen": -149.8671875,
      "logps/rejected": -153.6296844482422,
      "logps/weighted_chosen": -0.7455719113349915,
      "logps/weighted_rejected": -0.826556384563446,
      "loss": 0.7125,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -22.685546875,
      "rewards/margins": 1.7355468273162842,
      "rewards/rejected": -24.420312881469727,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": -0.19403991103172302,
      "rewards/weighted_margins": 0.02692871168255806,
      "rewards/weighted_rejected": -0.2209148406982422,
      "step": 430
    },
    {
      "epoch": 0.23515264880753559,
      "grad_norm": 10.726137161254883,
      "learning_rate": 9.461788314471034e-07,
      "logits/chosen": -0.08421440422534943,
      "logits/rejected": -0.031351469457149506,
      "logps/chosen": -143.1171875,
      "logps/rejected": -146.890625,
      "logps/weighted_chosen": -0.7524811029434204,
      "logps/weighted_rejected": -0.8818389773368835,
      "loss": 0.7062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -24.516796112060547,
      "rewards/margins": 2.0640625953674316,
      "rewards/rejected": -26.577733993530273,
      "rewards/weighted_accuracies": 0.528124988079071,
      "rewards/weighted_chosen": -0.22655639052391052,
      "rewards/weighted_margins": 0.07337188720703125,
      "rewards/weighted_rejected": -0.30005186796188354,
      "step": 440
    },
    {
      "epoch": 0.240497027189525,
      "grad_norm": 8.586684226989746,
      "learning_rate": 9.418915442193509e-07,
      "logits/chosen": -0.03446502611041069,
      "logits/rejected": 0.007968139834702015,
      "logps/chosen": -144.7843780517578,
      "logps/rejected": -155.96249389648438,
      "logps/weighted_chosen": -1.1978485584259033,
      "logps/weighted_rejected": -1.2447845935821533,
      "loss": 0.6841,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -26.516016006469727,
      "rewards/margins": 4.714453220367432,
      "rewards/rejected": -31.236328125,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.25520628690719604,
      "rewards/weighted_margins": 0.09325332939624786,
      "rewards/weighted_rejected": -0.3483436703681946,
      "step": 450
    },
    {
      "epoch": 0.24584140557151446,
      "grad_norm": 10.014894485473633,
      "learning_rate": 9.374504703309579e-07,
      "logits/chosen": -0.1379852294921875,
      "logits/rejected": -0.0861412063241005,
      "logps/chosen": -141.83828735351562,
      "logps/rejected": -143.34530639648438,
      "logps/weighted_chosen": -1.1420501470565796,
      "logps/weighted_rejected": -1.055877685546875,
      "loss": 0.6804,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": -25.53515625,
      "rewards/margins": 2.038281202316284,
      "rewards/rejected": -27.571874618530273,
      "rewards/weighted_accuracies": 0.5218750238418579,
      "rewards/weighted_chosen": -0.21983489394187927,
      "rewards/weighted_margins": 0.17051848769187927,
      "rewards/weighted_rejected": -0.3904357850551605,
      "step": 460
    },
    {
      "epoch": 0.2511857839535039,
      "grad_norm": 10.024358749389648,
      "learning_rate": 9.328571553600915e-07,
      "logits/chosen": -0.1318313628435135,
      "logits/rejected": -0.0828731507062912,
      "logps/chosen": -144.39688110351562,
      "logps/rejected": -147.41796875,
      "logps/weighted_chosen": -0.9446808099746704,
      "logps/weighted_rejected": -1.138859510421753,
      "loss": 0.7137,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -25.697265625,
      "rewards/margins": 1.9656250476837158,
      "rewards/rejected": -27.665233612060547,
      "rewards/weighted_accuracies": 0.4906249940395355,
      "rewards/weighted_chosen": -0.2078697234392166,
      "rewards/weighted_margins": 0.052256010472774506,
      "rewards/weighted_rejected": -0.260000616312027,
      "step": 470
    },
    {
      "epoch": 0.25653016233549336,
      "grad_norm": 13.618422508239746,
      "learning_rate": 9.281131978677106e-07,
      "logits/chosen": -0.16686096787452698,
      "logits/rejected": -0.1246364563703537,
      "logps/chosen": -149.2546844482422,
      "logps/rejected": -152.47811889648438,
      "logps/weighted_chosen": -0.686419665813446,
      "logps/weighted_rejected": -1.0338714122772217,
      "loss": 0.6805,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -25.559375762939453,
      "rewards/margins": 3.1332030296325684,
      "rewards/rejected": -28.689844131469727,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.22538375854492188,
      "rewards/weighted_margins": 0.13199158012866974,
      "rewards/weighted_rejected": -0.35721856355667114,
      "step": 480
    },
    {
      "epoch": 0.2618745407174828,
      "grad_norm": 30.852863311767578,
      "learning_rate": 9.232202488412361e-07,
      "logits/chosen": -0.10437698662281036,
      "logits/rejected": -0.05600614473223686,
      "logps/chosen": -143.5671844482422,
      "logps/rejected": -149.11874389648438,
      "logps/weighted_chosen": -1.236029028892517,
      "logps/weighted_rejected": -1.6542327404022217,
      "loss": 0.6937,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -28.171483993530273,
      "rewards/margins": 2.311328172683716,
      "rewards/rejected": -30.477733612060547,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": -0.27268296480178833,
      "rewards/weighted_margins": 0.08454742282629013,
      "rewards/weighted_rejected": -0.35735321044921875,
      "step": 490
    },
    {
      "epoch": 0.26721891909947226,
      "grad_norm": 9.151970863342285,
      "learning_rate": 9.181800111199766e-07,
      "logits/chosen": -0.1230342835187912,
      "logits/rejected": -0.0801507979631424,
      "logps/chosen": -147.54843139648438,
      "logps/rejected": -147.5437469482422,
      "logps/weighted_chosen": -1.5909912586212158,
      "logps/weighted_rejected": -1.286199927330017,
      "loss": 0.7005,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -29.640233993530273,
      "rewards/margins": 3.1507811546325684,
      "rewards/rejected": -32.789451599121094,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": -0.34203338623046875,
      "rewards/weighted_margins": 0.12214355170726776,
      "rewards/weighted_rejected": -0.4641265869140625,
      "step": 500
    },
    {
      "epoch": 0.26721891909947226,
      "eval_logits/chosen": -0.29476672410964966,
      "eval_logits/rejected": -0.26891082525253296,
      "eval_logps/chosen": -149.5992889404297,
      "eval_logps/rejected": -153.5249481201172,
      "eval_logps/weighted_chosen": -0.8761399984359741,
      "eval_logps/weighted_rejected": -1.120854377746582,
      "eval_loss": 0.6999610066413879,
      "eval_rewards/accuracies": 0.5784114003181458,
      "eval_rewards/chosen": -28.267057418823242,
      "eval_rewards/margins": 2.7842414379119873,
      "eval_rewards/rejected": -31.05422592163086,
      "eval_rewards/weighted_accuracies": 0.5264765620231628,
      "eval_rewards/weighted_chosen": -0.23699289560317993,
      "eval_rewards/weighted_margins": 0.08454981446266174,
      "eval_rewards/weighted_rejected": -0.3215427100658417,
      "eval_runtime": 1248.3443,
      "eval_samples_per_second": 1.571,
      "eval_steps_per_second": 0.393,
      "step": 500
    }
  ],
  "logging_steps": 10,
  "max_steps": 1872,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}