{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0020942408376963353,
      "grad_norm": 6690.55859375,
      "learning_rate": 0.0,
      "logits/chosen": -0.5995081663131714,
      "logits/rejected": -0.6144353747367859,
      "logps/chosen": -1.1853606700897217,
      "logps/rejected": -1.4816904067993164,
      "loss": 20000.3594,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.3230741387815215e-05,
      "rewards/margins": 7.880535122239962e-06,
      "rewards/rejected": 5.350205356080551e-06,
      "step": 1
    },
    {
      "epoch": 0.020942408376963352,
      "grad_norm": 5719.5029296875,
      "learning_rate": 9.375e-08,
      "logits/chosen": -0.6324554681777954,
      "logits/rejected": -0.6374175548553467,
      "logps/chosen": -1.1415884494781494,
      "logps/rejected": -1.3229657411575317,
      "loss": 20000.1285,
      "rewards/accuracies": 0.5243055820465088,
      "rewards/chosen": -1.3922724519943586e-06,
      "rewards/margins": -3.091482767558773e-07,
      "rewards/rejected": -1.0831237204911304e-06,
      "step": 10
    },
    {
      "epoch": 0.041884816753926704,
      "grad_norm": 8928.5966796875,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": -0.597779393196106,
      "logits/rejected": -0.6287046670913696,
      "logps/chosen": -1.1651686429977417,
      "logps/rejected": -1.311543583869934,
      "loss": 19999.3844,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 1.0694350748963188e-05,
      "rewards/margins": 6.846777978353202e-06,
      "rewards/rejected": 3.847571861115284e-06,
      "step": 20
    },
    {
      "epoch": 0.06282722513089005,
      "grad_norm": 6757.13037109375,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": -0.6134611368179321,
      "logits/rejected": -0.605442225933075,
      "logps/chosen": -1.1174745559692383,
      "logps/rejected": -1.2003570795059204,
      "loss": 19997.9344,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 6.686393317067996e-05,
      "rewards/margins": 3.225356340408325e-05,
      "rewards/rejected": 3.461036249063909e-05,
      "step": 30
    },
    {
      "epoch": 0.08376963350785341,
      "grad_norm": 8304.0927734375,
      "learning_rate": 4.0625e-07,
      "logits/chosen": -0.6177842617034912,
      "logits/rejected": -0.6428981423377991,
      "logps/chosen": -1.1158215999603271,
      "logps/rejected": -1.2773981094360352,
      "loss": 19990.6594,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0002113355149049312,
      "rewards/margins": 8.95043122000061e-05,
      "rewards/rejected": 0.0001218312099808827,
      "step": 40
    },
    {
      "epoch": 0.10471204188481675,
      "grad_norm": 6644.90380859375,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": -0.6271435022354126,
      "logits/rejected": -0.6673277020454407,
      "logps/chosen": -1.047498345375061,
      "logps/rejected": -1.2514413595199585,
      "loss": 19978.0703,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 0.0006389970076270401,
      "rewards/margins": 0.0003597235190682113,
      "rewards/rejected": 0.00027927348855882883,
      "step": 50
    },
    {
      "epoch": 0.1256544502617801,
      "grad_norm": 9410.4248046875,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": -0.6456407308578491,
      "logits/rejected": -0.6601846814155579,
      "logps/chosen": -1.1118009090423584,
      "logps/rejected": -1.2955642938613892,
      "loss": 19958.8578,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0007649646140635014,
      "rewards/margins": 0.00041284109465777874,
      "rewards/rejected": 0.0003521234611980617,
      "step": 60
    },
    {
      "epoch": 0.14659685863874344,
      "grad_norm": 9895.3310546875,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": -0.710538923740387,
      "logits/rejected": -0.7164761424064636,
      "logps/chosen": -1.1343839168548584,
      "logps/rejected": -1.3560715913772583,
      "loss": 19939.225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00039382357499562204,
      "rewards/margins": 0.0006254783947952092,
      "rewards/rejected": -0.0002316548052476719,
      "step": 70
    },
    {
      "epoch": 0.16753926701570682,
      "grad_norm": 16232.4453125,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": -0.6893922090530396,
      "logits/rejected": -0.6864418387413025,
      "logps/chosen": -1.1864311695098877,
      "logps/rejected": -1.511588215827942,
      "loss": 19887.7375,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0005077069508843124,
      "rewards/margins": 0.0014377882471308112,
      "rewards/rejected": -0.0019454952562227845,
      "step": 80
    },
    {
      "epoch": 0.18848167539267016,
      "grad_norm": 15383.0810546875,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": -0.6936720609664917,
      "logits/rejected": -0.6995416879653931,
      "logps/chosen": -1.2832069396972656,
      "logps/rejected": -1.6835159063339233,
      "loss": 19835.5531,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0014368193224072456,
      "rewards/margins": 0.0021624970249831676,
      "rewards/rejected": -0.003599316580221057,
      "step": 90
    },
    {
      "epoch": 0.2094240837696335,
      "grad_norm": 51433.94921875,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": -0.6618175506591797,
      "logits/rejected": -0.6787184476852417,
      "logps/chosen": -1.3208234310150146,
      "logps/rejected": -1.6875203847885132,
      "loss": 19789.7328,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.001908238627947867,
      "rewards/margins": 0.0021552981343120337,
      "rewards/rejected": -0.004063536878675222,
      "step": 100
    },
    {
      "epoch": 0.23036649214659685,
      "grad_norm": 39666.109375,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": -0.6607747077941895,
      "logits/rejected": -0.6833058595657349,
      "logps/chosen": -1.3800714015960693,
      "logps/rejected": -2.1090340614318848,
      "loss": 19729.8531,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.002765479264780879,
      "rewards/margins": 0.004542567301541567,
      "rewards/rejected": -0.007308047264814377,
      "step": 110
    },
    {
      "epoch": 0.2513089005235602,
      "grad_norm": 26585.208984375,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": -0.6459494829177856,
      "logits/rejected": -0.6470843553543091,
      "logps/chosen": -1.5455596446990967,
      "logps/rejected": -1.9538530111312866,
      "loss": 19736.5594,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.004025847185403109,
      "rewards/margins": 0.0025805418845266104,
      "rewards/rejected": -0.006606388837099075,
      "step": 120
    },
    {
      "epoch": 0.27225130890052357,
      "grad_norm": 39655.12890625,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": -0.6332282423973083,
      "logits/rejected": -0.6239765286445618,
      "logps/chosen": -1.8061062097549438,
      "logps/rejected": -2.3397936820983887,
      "loss": 19656.4844,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.006146098021417856,
      "rewards/margins": 0.003909106366336346,
      "rewards/rejected": -0.010055203922092915,
      "step": 130
    },
    {
      "epoch": 0.2931937172774869,
      "grad_norm": 63126.5234375,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": -0.6292804479598999,
      "logits/rejected": -0.6358648538589478,
      "logps/chosen": -2.2388834953308105,
      "logps/rejected": -2.7073793411254883,
      "loss": 19726.4313,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.01039391104131937,
      "rewards/margins": 0.003122520400211215,
      "rewards/rejected": -0.013516431674361229,
      "step": 140
    },
    {
      "epoch": 0.31413612565445026,
      "grad_norm": 29782.576171875,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": -0.6361591815948486,
      "logits/rejected": -0.635583758354187,
      "logps/chosen": -2.1060826778411865,
      "logps/rejected": -3.041602373123169,
      "loss": 19551.1266,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009226142428815365,
      "rewards/margins": 0.007642732001841068,
      "rewards/rejected": -0.016868876293301582,
      "step": 150
    },
    {
      "epoch": 0.33507853403141363,
      "grad_norm": 37246.7578125,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": -0.6211342811584473,
      "logits/rejected": -0.6241915822029114,
      "logps/chosen": -2.1096720695495605,
      "logps/rejected": -2.964322566986084,
      "loss": 19569.3875,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -0.009453673847019672,
      "rewards/margins": 0.0064040967263281345,
      "rewards/rejected": -0.015857771039009094,
      "step": 160
    },
    {
      "epoch": 0.35602094240837695,
      "grad_norm": 46280.3671875,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": -0.6069876551628113,
      "logits/rejected": -0.6020215749740601,
      "logps/chosen": -2.012474536895752,
      "logps/rejected": -3.0016605854034424,
      "loss": 19453.225,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.008933757431805134,
      "rewards/margins": 0.008303998038172722,
      "rewards/rejected": -0.01723775453865528,
      "step": 170
    },
    {
      "epoch": 0.3769633507853403,
      "grad_norm": 47927.20703125,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": -0.5999588966369629,
      "logits/rejected": -0.6140414476394653,
      "logps/chosen": -2.4049391746520996,
      "logps/rejected": -3.287382125854492,
      "loss": 19432.0125,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.012067523784935474,
      "rewards/margins": 0.007707494311034679,
      "rewards/rejected": -0.019775018095970154,
      "step": 180
    },
    {
      "epoch": 0.39790575916230364,
      "grad_norm": 188406.578125,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": -0.6130908727645874,
      "logits/rejected": -0.6305940747261047,
      "logps/chosen": -2.832430362701416,
      "logps/rejected": -3.6175262928009033,
      "loss": 19449.6109,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.017013628035783768,
      "rewards/margins": 0.0060007283464074135,
      "rewards/rejected": -0.023014355450868607,
      "step": 190
    },
    {
      "epoch": 0.418848167539267,
      "grad_norm": 47073.16015625,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": -0.6221901178359985,
      "logits/rejected": -0.6376347541809082,
      "logps/chosen": -2.5921828746795654,
      "logps/rejected": -3.7902023792266846,
      "loss": 19275.1328,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.01452858280390501,
      "rewards/margins": 0.01014014147222042,
      "rewards/rejected": -0.024668725207448006,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_logits/chosen": -0.6352167725563049,
      "eval_logits/rejected": -0.6299898624420166,
      "eval_logps/chosen": -2.4551029205322266,
      "eval_logps/rejected": -3.4695351123809814,
      "eval_loss": 2417.49609375,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.013385402970016003,
      "eval_rewards/margins": 0.00833566952496767,
      "eval_rewards/rejected": -0.021721070632338524,
      "eval_runtime": 78.7567,
      "eval_samples_per_second": 25.395,
      "eval_steps_per_second": 1.587,
      "step": 200
    },
    {
      "epoch": 0.4397905759162304,
      "grad_norm": 27402.61328125,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": -0.6011684536933899,
      "logits/rejected": -0.6206346750259399,
      "logps/chosen": -2.257577419281006,
      "logps/rejected": -3.6189427375793457,
      "loss": 19377.5328,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.01108148880302906,
      "rewards/margins": 0.011725915595889091,
      "rewards/rejected": -0.022807404398918152,
      "step": 210
    },
    {
      "epoch": 0.4607329842931937,
      "grad_norm": 55464.56640625,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": -0.6579941511154175,
      "logits/rejected": -0.6557571291923523,
      "logps/chosen": -2.2686405181884766,
      "logps/rejected": -3.539849042892456,
      "loss": 19303.3125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0108075812458992,
      "rewards/margins": 0.011197235435247421,
      "rewards/rejected": -0.02200481668114662,
      "step": 220
    },
    {
      "epoch": 0.4816753926701571,
      "grad_norm": 66891.46875,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": -0.6366732716560364,
      "logits/rejected": -0.6232300996780396,
      "logps/chosen": -2.8289544582366943,
      "logps/rejected": -3.995779037475586,
      "loss": 19231.3969,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.016343776136636734,
      "rewards/margins": 0.01012236438691616,
      "rewards/rejected": -0.026466142386198044,
      "step": 230
    },
    {
      "epoch": 0.5026178010471204,
      "grad_norm": 69423.734375,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": -0.6015263795852661,
      "logits/rejected": -0.6166636943817139,
      "logps/chosen": -3.3009753227233887,
      "logps/rejected": -4.603926658630371,
      "loss": 19201.2047,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.021539034321904182,
      "rewards/margins": 0.011443909257650375,
      "rewards/rejected": -0.03298294544219971,
      "step": 240
    },
    {
      "epoch": 0.5235602094240838,
      "grad_norm": 64731.2421875,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": -0.6268733739852905,
      "logits/rejected": -0.6166061162948608,
      "logps/chosen": -2.9978890419006348,
      "logps/rejected": -4.9570441246032715,
      "loss": 18975.8219,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.018680796027183533,
      "rewards/margins": 0.01662489026784897,
      "rewards/rejected": -0.0353056825697422,
      "step": 250
    },
    {
      "epoch": 0.5445026178010471,
      "grad_norm": 64183.33203125,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": -0.6079710721969604,
      "logits/rejected": -0.5957759618759155,
      "logps/chosen": -3.0889039039611816,
      "logps/rejected": -4.906301021575928,
      "loss": 18967.3656,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.019036870449781418,
      "rewards/margins": 0.016856301575899124,
      "rewards/rejected": -0.03589317575097084,
      "step": 260
    },
    {
      "epoch": 0.5654450261780105,
      "grad_norm": 103451.53125,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": -0.5650458335876465,
      "logits/rejected": -0.5668447613716125,
      "logps/chosen": -3.9116883277893066,
      "logps/rejected": -5.719117164611816,
      "loss": 18905.8031,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.027667338028550148,
      "rewards/margins": 0.016292227432131767,
      "rewards/rejected": -0.043959565460681915,
      "step": 270
    },
    {
      "epoch": 0.5863874345549738,
      "grad_norm": 74060.140625,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": -0.562662661075592,
      "logits/rejected": -0.5547593235969543,
      "logps/chosen": -3.6009249687194824,
      "logps/rejected": -6.734810829162598,
      "loss": 18614.3531,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02406414784491062,
      "rewards/margins": 0.029486840590834618,
      "rewards/rejected": -0.05355098843574524,
      "step": 280
    },
    {
      "epoch": 0.6073298429319371,
      "grad_norm": 197744.75,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": -0.5442657470703125,
      "logits/rejected": -0.5202258825302124,
      "logps/chosen": -4.191218852996826,
      "logps/rejected": -5.84238862991333,
      "loss": 18823.9969,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -0.0303532425314188,
      "rewards/margins": 0.014872364699840546,
      "rewards/rejected": -0.045225612819194794,
      "step": 290
    },
    {
      "epoch": 0.6282722513089005,
      "grad_norm": 82288.1015625,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": -0.5231214165687561,
      "logits/rejected": -0.5048767328262329,
      "logps/chosen": -4.525745868682861,
      "logps/rejected": -5.8891825675964355,
      "loss": 18956.8344,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -0.03319484740495682,
      "rewards/margins": 0.013094061985611916,
      "rewards/rejected": -0.046288907527923584,
      "step": 300
    },
    {
      "epoch": 0.6492146596858639,
      "grad_norm": 83390.5,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": -0.47909316420555115,
      "logits/rejected": -0.4821901321411133,
      "logps/chosen": -3.890228271484375,
      "logps/rejected": -5.4383721351623535,
      "loss": 18785.9547,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.027516454458236694,
      "rewards/margins": 0.014197492972016335,
      "rewards/rejected": -0.04171394929289818,
      "step": 310
    },
    {
      "epoch": 0.6701570680628273,
      "grad_norm": 119347.234375,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": -0.47245222330093384,
      "logits/rejected": -0.48146170377731323,
      "logps/chosen": -4.161627769470215,
      "logps/rejected": -6.324838161468506,
      "loss": 18573.5031,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.030029457062482834,
      "rewards/margins": 0.020363491028547287,
      "rewards/rejected": -0.05039294809103012,
      "step": 320
    },
    {
      "epoch": 0.6910994764397905,
      "grad_norm": 108725.671875,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": -0.4325336515903473,
      "logits/rejected": -0.43659958243370056,
      "logps/chosen": -4.2359724044799805,
      "logps/rejected": -7.189882755279541,
      "loss": 18710.9344,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.03000812791287899,
      "rewards/margins": 0.028335105627775192,
      "rewards/rejected": -0.05834323167800903,
      "step": 330
    },
    {
      "epoch": 0.7120418848167539,
      "grad_norm": 250133.859375,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": -0.40766844153404236,
      "logits/rejected": -0.40503817796707153,
      "logps/chosen": -5.0073561668396,
      "logps/rejected": -7.256580352783203,
      "loss": 18789.4781,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.037794455885887146,
      "rewards/margins": 0.021081697195768356,
      "rewards/rejected": -0.0588761568069458,
      "step": 340
    },
    {
      "epoch": 0.7329842931937173,
      "grad_norm": 65695.078125,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": -0.402383416891098,
      "logits/rejected": -0.4145272374153137,
      "logps/chosen": -4.494850158691406,
      "logps/rejected": -6.888899803161621,
      "loss": 18565.8484,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.03312065079808235,
      "rewards/margins": 0.0219460166990757,
      "rewards/rejected": -0.05506666377186775,
      "step": 350
    },
    {
      "epoch": 0.7539267015706806,
      "grad_norm": 182331.0625,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": -0.3540952503681183,
      "logits/rejected": -0.36194995045661926,
      "logps/chosen": -4.524051189422607,
      "logps/rejected": -7.192444801330566,
      "loss": 18681.9938,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.03363392874598503,
      "rewards/margins": 0.024621980264782906,
      "rewards/rejected": -0.058255910873413086,
      "step": 360
    },
    {
      "epoch": 0.774869109947644,
      "grad_norm": 99043.015625,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": -0.38287925720214844,
      "logits/rejected": -0.3868221640586853,
      "logps/chosen": -4.889115810394287,
      "logps/rejected": -7.343722343444824,
      "loss": 18350.1594,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.036828476935625076,
      "rewards/margins": 0.02258094772696495,
      "rewards/rejected": -0.059409428387880325,
      "step": 370
    },
    {
      "epoch": 0.7958115183246073,
      "grad_norm": 118281.1953125,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": -0.29700514674186707,
      "logits/rejected": -0.3104197382926941,
      "logps/chosen": -4.535180568695068,
      "logps/rejected": -7.199211120605469,
      "loss": 18479.7844,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.033621106296777725,
      "rewards/margins": 0.02514580450952053,
      "rewards/rejected": -0.058766912668943405,
      "step": 380
    },
    {
      "epoch": 0.8167539267015707,
      "grad_norm": 116531.1015625,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": -0.28081822395324707,
      "logits/rejected": -0.29538464546203613,
      "logps/chosen": -5.330414772033691,
      "logps/rejected": -7.864521026611328,
      "loss": 18660.3094,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.041854970157146454,
      "rewards/margins": 0.023673858493566513,
      "rewards/rejected": -0.06552883237600327,
      "step": 390
    },
    {
      "epoch": 0.837696335078534,
      "grad_norm": 115313.1640625,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": -0.2553872764110565,
      "logits/rejected": -0.27801090478897095,
      "logps/chosen": -5.410425662994385,
      "logps/rejected": -7.3440046310424805,
      "loss": 18486.2438,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04237430915236473,
      "rewards/margins": 0.018501609563827515,
      "rewards/rejected": -0.060875922441482544,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_logits/chosen": -0.2619972229003906,
      "eval_logits/rejected": -0.27771249413490295,
      "eval_logps/chosen": -5.502514839172363,
      "eval_logps/rejected": -8.184652328491211,
      "eval_loss": 2313.8056640625,
      "eval_rewards/accuracies": 0.6800000071525574,
      "eval_rewards/chosen": -0.04385951906442642,
      "eval_rewards/margins": 0.025012729689478874,
      "eval_rewards/rejected": -0.06887225061655045,
      "eval_runtime": 78.5743,
      "eval_samples_per_second": 25.454,
      "eval_steps_per_second": 1.591,
      "step": 400
    },
    {
      "epoch": 0.8586387434554974,
      "grad_norm": 116385.09375,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": -0.24211814999580383,
      "logits/rejected": -0.2585967183113098,
      "logps/chosen": -5.910313606262207,
      "logps/rejected": -8.293985366821289,
      "loss": 18503.9844,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -0.047523993998765945,
      "rewards/margins": 0.02237233892083168,
      "rewards/rejected": -0.06989633291959763,
      "step": 410
    },
    {
      "epoch": 0.8795811518324608,
      "grad_norm": 134170.890625,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": -0.22771398723125458,
      "logits/rejected": -0.2378026694059372,
      "logps/chosen": -5.347130298614502,
      "logps/rejected": -8.04057788848877,
      "loss": 18328.1453,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.042228274047374725,
      "rewards/margins": 0.025333663448691368,
      "rewards/rejected": -0.06756193935871124,
      "step": 420
    },
    {
      "epoch": 0.900523560209424,
      "grad_norm": 140065.46875,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": -0.22656838595867157,
      "logits/rejected": -0.26688051223754883,
      "logps/chosen": -5.213972568511963,
      "logps/rejected": -8.997479438781738,
      "loss": 18304.7516,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.040835700929164886,
      "rewards/margins": 0.03499193489551544,
      "rewards/rejected": -0.07582763582468033,
      "step": 430
    },
    {
      "epoch": 0.9214659685863874,
      "grad_norm": 147417.296875,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": -0.21701665222644806,
      "logits/rejected": -0.22004380822181702,
      "logps/chosen": -5.577446937561035,
      "logps/rejected": -8.317059516906738,
      "loss": 18362.8578,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.04439837858080864,
      "rewards/margins": 0.025850927457213402,
      "rewards/rejected": -0.07024930417537689,
      "step": 440
    },
    {
      "epoch": 0.9424083769633508,
      "grad_norm": 187662.15625,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": -0.19742074608802795,
      "logits/rejected": -0.21276791393756866,
      "logps/chosen": -5.669443607330322,
      "logps/rejected": -8.552831649780273,
      "loss": 18458.9266,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0450492687523365,
      "rewards/margins": 0.027155417948961258,
      "rewards/rejected": -0.07220469415187836,
      "step": 450
    },
    {
      "epoch": 0.9633507853403142,
      "grad_norm": 124422.875,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": -0.18504954874515533,
      "logits/rejected": -0.22272753715515137,
      "logps/chosen": -5.632592678070068,
      "logps/rejected": -8.428214073181152,
      "loss": 18551.7969,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.04460041597485542,
      "rewards/margins": 0.026442039757966995,
      "rewards/rejected": -0.07104245573282242,
      "step": 460
    },
    {
      "epoch": 0.9842931937172775,
      "grad_norm": 157096.296875,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": -0.14469654858112335,
      "logits/rejected": -0.15391038358211517,
      "logps/chosen": -5.636763572692871,
      "logps/rejected": -8.912847518920898,
      "loss": 18241.4094,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.04426239803433418,
      "rewards/margins": 0.031570300459861755,
      "rewards/rejected": -0.07583270221948624,
      "step": 470
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 19137.84001572327,
      "train_runtime": 6782.201,
      "train_samples_per_second": 9.014,
      "train_steps_per_second": 0.07
    }
  ],
  "logging_steps": 10,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}