{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0020942408376963353,
      "grad_norm": 241.46054077148438,
      "learning_rate": 0.0,
      "logits/chosen": -0.615048885345459,
      "logits/rejected": -0.6184952855110168,
      "logps/chosen": -340.7542724609375,
      "logps/rejected": -224.71102905273438,
      "loss": 46.3402,
      "nll_loss": 1.1951930522918701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -34.0754280090332,
      "rewards/margins": -11.60432243347168,
      "rewards/rejected": -22.47110366821289,
      "step": 1
    },
    {
      "epoch": 0.020942408376963352,
      "grad_norm": 207.28042602539062,
      "learning_rate": 9.375e-08,
      "logits/chosen": -0.6425137519836426,
      "logits/rejected": -0.636458158493042,
      "logps/chosen": -259.6365051269531,
      "logps/rejected": -258.07281494140625,
      "loss": 42.6859,
      "nll_loss": 1.0321696996688843,
      "rewards/accuracies": 0.4930555522441864,
      "rewards/chosen": -25.96364974975586,
      "rewards/margins": -0.1563688963651657,
      "rewards/rejected": -25.807281494140625,
      "step": 10
    },
    {
      "epoch": 0.041884816753926704,
      "grad_norm": 241.39328002929688,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": -0.6058956384658813,
      "logits/rejected": -0.6296488046646118,
      "logps/chosen": -292.1470642089844,
      "logps/rejected": -243.1712188720703,
      "loss": 42.1261,
      "nll_loss": 1.0961360931396484,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -29.214702606201172,
      "rewards/margins": -4.897582530975342,
      "rewards/rejected": -24.317123413085938,
      "step": 20
    },
    {
      "epoch": 0.06282722513089005,
      "grad_norm": 214.8778839111328,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": -0.5835741758346558,
      "logits/rejected": -0.5625507831573486,
      "logps/chosen": -266.05560302734375,
      "logps/rejected": -251.7474822998047,
      "loss": 41.0696,
      "nll_loss": 1.0376346111297607,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -26.60555648803711,
      "rewards/margins": -1.4308110475540161,
      "rewards/rejected": -25.174747467041016,
      "step": 30
    },
    {
      "epoch": 0.08376963350785341,
      "grad_norm": 228.7670440673828,
      "learning_rate": 4.0625e-07,
      "logits/chosen": -0.6142115592956543,
      "logits/rejected": -0.6461960077285767,
      "logps/chosen": -280.4624938964844,
      "logps/rejected": -260.13128662109375,
      "loss": 42.8858,
      "nll_loss": 1.001647710800171,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -28.046253204345703,
      "rewards/margins": -2.0331199169158936,
      "rewards/rejected": -26.013132095336914,
      "step": 40
    },
    {
      "epoch": 0.10471204188481675,
      "grad_norm": 203.48428344726562,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": -0.6760377287864685,
      "logits/rejected": -0.6930453777313232,
      "logps/chosen": -257.87176513671875,
      "logps/rejected": -250.3403778076172,
      "loss": 39.7368,
      "nll_loss": 0.9915801286697388,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -25.78717613220215,
      "rewards/margins": -0.7531374096870422,
      "rewards/rejected": -25.034040451049805,
      "step": 50
    },
    {
      "epoch": 0.1256544502617801,
      "grad_norm": 204.22805786132812,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": -0.6431644558906555,
      "logits/rejected": -0.6466089487075806,
      "logps/chosen": -284.398681640625,
      "logps/rejected": -275.51971435546875,
      "loss": 40.0522,
      "nll_loss": 1.030461311340332,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -28.439865112304688,
      "rewards/margins": -0.8878933191299438,
      "rewards/rejected": -27.55197525024414,
      "step": 60
    },
    {
      "epoch": 0.14659685863874344,
      "grad_norm": 216.3765411376953,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": -0.7524436712265015,
      "logits/rejected": -0.7512598633766174,
      "logps/chosen": -288.4020080566406,
      "logps/rejected": -285.08770751953125,
      "loss": 37.5469,
      "nll_loss": 1.122981309890747,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -28.840198516845703,
      "rewards/margins": -0.3314289450645447,
      "rewards/rejected": -28.508769989013672,
      "step": 70
    },
    {
      "epoch": 0.16753926701570682,
      "grad_norm": 221.7291259765625,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": -0.7214940786361694,
      "logits/rejected": -0.7364234328269958,
      "logps/chosen": -263.50787353515625,
      "logps/rejected": -245.77099609375,
      "loss": 36.5621,
      "nll_loss": 1.0321334600448608,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -26.350788116455078,
      "rewards/margins": -1.7736867666244507,
      "rewards/rejected": -24.577102661132812,
      "step": 80
    },
    {
      "epoch": 0.18848167539267016,
      "grad_norm": 209.3531951904297,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": -0.6903802752494812,
      "logits/rejected": -0.7218402028083801,
      "logps/chosen": -283.48724365234375,
      "logps/rejected": -249.9278564453125,
      "loss": 37.2678,
      "nll_loss": 1.0136864185333252,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -28.348724365234375,
      "rewards/margins": -3.35593843460083,
      "rewards/rejected": -24.992786407470703,
      "step": 90
    },
    {
      "epoch": 0.2094240837696335,
      "grad_norm": 222.45443725585938,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": -0.6955899000167847,
      "logits/rejected": -0.7055763006210327,
      "logps/chosen": -265.42913818359375,
      "logps/rejected": -276.52716064453125,
      "loss": 36.8305,
      "nll_loss": 0.9817202687263489,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -26.542917251586914,
      "rewards/margins": 1.1097980737686157,
      "rewards/rejected": -27.6527156829834,
      "step": 100
    },
    {
      "epoch": 0.23036649214659685,
      "grad_norm": 233.69064331054688,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": -0.6523857116699219,
      "logits/rejected": -0.7047474980354309,
      "logps/chosen": -286.0872802734375,
      "logps/rejected": -300.1228942871094,
      "loss": 34.9865,
      "nll_loss": 0.9495639801025391,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -28.608728408813477,
      "rewards/margins": 1.403562307357788,
      "rewards/rejected": -30.012292861938477,
      "step": 110
    },
    {
      "epoch": 0.2513089005235602,
      "grad_norm": 489.7384338378906,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": -0.7176281213760376,
      "logits/rejected": -0.7226043939590454,
      "logps/chosen": -289.14080810546875,
      "logps/rejected": -291.45965576171875,
      "loss": 34.8572,
      "nll_loss": 1.205727458000183,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -28.914077758789062,
      "rewards/margins": 0.2318885773420334,
      "rewards/rejected": -29.14596939086914,
      "step": 120
    },
    {
      "epoch": 0.27225130890052357,
      "grad_norm": 344.9328308105469,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": -0.6907280683517456,
      "logits/rejected": -0.6784687042236328,
      "logps/chosen": -315.9068298339844,
      "logps/rejected": -321.81317138671875,
      "loss": 33.9604,
      "nll_loss": 1.2358386516571045,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -31.590686798095703,
      "rewards/margins": 0.5906292796134949,
      "rewards/rejected": -32.18132019042969,
      "step": 130
    },
    {
      "epoch": 0.2931937172774869,
      "grad_norm": 940.4654541015625,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": -0.7276722192764282,
      "logits/rejected": -0.6987274289131165,
      "logps/chosen": -334.77227783203125,
      "logps/rejected": -322.3913269042969,
      "loss": 31.7891,
      "nll_loss": 1.4334642887115479,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -33.47722625732422,
      "rewards/margins": -1.2380931377410889,
      "rewards/rejected": -32.239131927490234,
      "step": 140
    },
    {
      "epoch": 0.31413612565445026,
      "grad_norm": 532.178955078125,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": -0.7420992851257324,
      "logits/rejected": -0.7299410104751587,
      "logps/chosen": -352.385498046875,
      "logps/rejected": -325.99505615234375,
      "loss": 31.6743,
      "nll_loss": 1.2520462274551392,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -35.238548278808594,
      "rewards/margins": -2.63904070854187,
      "rewards/rejected": -32.599510192871094,
      "step": 150
    },
    {
      "epoch": 0.33507853403141363,
      "grad_norm": 791.0310668945312,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": -0.7037047147750854,
      "logits/rejected": -0.7137006521224976,
      "logps/chosen": -320.0517578125,
      "logps/rejected": -320.142578125,
      "loss": 30.3343,
      "nll_loss": 1.385374903678894,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -32.00517654418945,
      "rewards/margins": 0.009086417965590954,
      "rewards/rejected": -32.014259338378906,
      "step": 160
    },
    {
      "epoch": 0.35602094240837695,
      "grad_norm": 458.65313720703125,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": -0.7242492437362671,
      "logits/rejected": -0.7235726118087769,
      "logps/chosen": -355.2301330566406,
      "logps/rejected": -352.45684814453125,
      "loss": 31.6656,
      "nll_loss": 1.4098021984100342,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -35.52301025390625,
      "rewards/margins": -0.2773243486881256,
      "rewards/rejected": -35.24568557739258,
      "step": 170
    },
    {
      "epoch": 0.3769633507853403,
      "grad_norm": 332.9512023925781,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": -0.6966814994812012,
      "logits/rejected": -0.7147270441055298,
      "logps/chosen": -338.35791015625,
      "logps/rejected": -338.9778137207031,
      "loss": 28.4701,
      "nll_loss": 1.5228649377822876,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -33.835792541503906,
      "rewards/margins": 0.06198873370885849,
      "rewards/rejected": -33.89778137207031,
      "step": 180
    },
    {
      "epoch": 0.39790575916230364,
      "grad_norm": 504.8504333496094,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": -0.7642617225646973,
      "logits/rejected": -0.7676379084587097,
      "logps/chosen": -336.16644287109375,
      "logps/rejected": -335.4501953125,
      "loss": 28.6168,
      "nll_loss": 1.2395371198654175,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -33.61664581298828,
      "rewards/margins": -0.0716233029961586,
      "rewards/rejected": -33.545021057128906,
      "step": 190
    },
    {
      "epoch": 0.418848167539267,
      "grad_norm": 662.9274291992188,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": -0.7552827000617981,
      "logits/rejected": -0.778997540473938,
      "logps/chosen": -305.7959289550781,
      "logps/rejected": -326.4095764160156,
      "loss": 27.4789,
      "nll_loss": 1.2367076873779297,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -30.5795955657959,
      "rewards/margins": 2.0613608360290527,
      "rewards/rejected": -32.640953063964844,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_logits/chosen": -0.7581648826599121,
      "eval_logits/rejected": -0.7404313087463379,
      "eval_logps/chosen": -342.4026184082031,
      "eval_logps/rejected": -349.4523620605469,
      "eval_loss": 6.657596111297607,
      "eval_nll_loss": 1.230733036994934,
      "eval_rewards/accuracies": 0.5766128897666931,
      "eval_rewards/chosen": -34.240257263183594,
      "eval_rewards/margins": 0.704971432685852,
      "eval_rewards/rejected": -34.94523239135742,
      "eval_runtime": 23.2353,
      "eval_samples_per_second": 86.076,
      "eval_steps_per_second": 2.711,
      "step": 200
    },
    {
      "epoch": 0.4397905759162304,
      "grad_norm": 579.301025390625,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": -0.6766096949577332,
      "logits/rejected": -0.6979749798774719,
      "logps/chosen": -336.41845703125,
      "logps/rejected": -355.58294677734375,
      "loss": 26.676,
      "nll_loss": 1.2723300457000732,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -33.641845703125,
      "rewards/margins": 1.916452407836914,
      "rewards/rejected": -35.55829620361328,
      "step": 210
    },
    {
      "epoch": 0.4607329842931937,
      "grad_norm": 1089.4422607421875,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": -0.7449113130569458,
      "logits/rejected": -0.7312273979187012,
      "logps/chosen": -325.61102294921875,
      "logps/rejected": -380.57421875,
      "loss": 27.9027,
      "nll_loss": 1.3178092241287231,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -32.56110763549805,
      "rewards/margins": 5.496316432952881,
      "rewards/rejected": -38.05742263793945,
      "step": 220
    },
    {
      "epoch": 0.4816753926701571,
      "grad_norm": 529.4258422851562,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": -0.7479566335678101,
      "logits/rejected": -0.7232205271720886,
      "logps/chosen": -376.5633239746094,
      "logps/rejected": -377.84942626953125,
      "loss": 25.7607,
      "nll_loss": 1.54669189453125,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -37.65633773803711,
      "rewards/margins": 0.12861032783985138,
      "rewards/rejected": -37.784950256347656,
      "step": 230
    },
    {
      "epoch": 0.5026178010471204,
      "grad_norm": 497.5738525390625,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": -0.6627580523490906,
      "logits/rejected": -0.704414427280426,
      "logps/chosen": -320.0458679199219,
      "logps/rejected": -344.0610656738281,
      "loss": 27.43,
      "nll_loss": 1.3753007650375366,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -32.00458908081055,
      "rewards/margins": 2.401517391204834,
      "rewards/rejected": -34.406105041503906,
      "step": 240
    },
    {
      "epoch": 0.5235602094240838,
      "grad_norm": 388.00811767578125,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": -0.7279710173606873,
      "logits/rejected": -0.7237112522125244,
      "logps/chosen": -366.0332336425781,
      "logps/rejected": -404.2882080078125,
      "loss": 26.6833,
      "nll_loss": 1.5022847652435303,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -36.60332489013672,
      "rewards/margins": 3.8254940509796143,
      "rewards/rejected": -40.42881774902344,
      "step": 250
    },
    {
      "epoch": 0.5445026178010471,
      "grad_norm": 488.0087585449219,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": -0.7367153167724609,
      "logits/rejected": -0.7415083646774292,
      "logps/chosen": -342.8262023925781,
      "logps/rejected": -365.2310791015625,
      "loss": 26.6956,
      "nll_loss": 1.3738378286361694,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -34.282615661621094,
      "rewards/margins": 2.2404911518096924,
      "rewards/rejected": -36.523109436035156,
      "step": 260
    },
    {
      "epoch": 0.5654450261780105,
      "grad_norm": 516.7625732421875,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": -0.721376895904541,
      "logits/rejected": -0.7088552713394165,
      "logps/chosen": -339.37744140625,
      "logps/rejected": -377.49615478515625,
      "loss": 23.9279,
      "nll_loss": 1.59787917137146,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -33.937744140625,
      "rewards/margins": 3.811868667602539,
      "rewards/rejected": -37.749610900878906,
      "step": 270
    },
    {
      "epoch": 0.5863874345549738,
      "grad_norm": 540.905029296875,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": -0.7323180437088013,
      "logits/rejected": -0.7498946189880371,
      "logps/chosen": -361.0552673339844,
      "logps/rejected": -382.8980712890625,
      "loss": 23.6927,
      "nll_loss": 1.3734923601150513,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -36.10552978515625,
      "rewards/margins": 2.184277057647705,
      "rewards/rejected": -38.2898063659668,
      "step": 280
    },
    {
      "epoch": 0.6073298429319371,
      "grad_norm": 614.5367431640625,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": -0.7599818110466003,
      "logits/rejected": -0.7415232062339783,
      "logps/chosen": -390.739013671875,
      "logps/rejected": -402.10894775390625,
      "loss": 26.7315,
      "nll_loss": 1.4149469137191772,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.07390594482422,
      "rewards/margins": 1.1369867324829102,
      "rewards/rejected": -40.21089553833008,
      "step": 290
    },
    {
      "epoch": 0.6282722513089005,
      "grad_norm": 611.55517578125,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": -0.7755026817321777,
      "logits/rejected": -0.7531968355178833,
      "logps/chosen": -340.93896484375,
      "logps/rejected": -375.37054443359375,
      "loss": 23.0069,
      "nll_loss": 1.4688938856124878,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -34.093894958496094,
      "rewards/margins": 3.4431610107421875,
      "rewards/rejected": -37.53705596923828,
      "step": 300
    },
    {
      "epoch": 0.6492146596858639,
      "grad_norm": 559.4390258789062,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": -0.720358669757843,
      "logits/rejected": -0.7089865803718567,
      "logps/chosen": -381.8735656738281,
      "logps/rejected": -424.19805908203125,
      "loss": 23.6405,
      "nll_loss": 1.3657066822052002,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.187355041503906,
      "rewards/margins": 4.23245096206665,
      "rewards/rejected": -42.419803619384766,
      "step": 310
    },
    {
      "epoch": 0.6701570680628273,
      "grad_norm": 566.0950927734375,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": -0.751908004283905,
      "logits/rejected": -0.7763121724128723,
      "logps/chosen": -361.6825256347656,
      "logps/rejected": -368.67047119140625,
      "loss": 24.4483,
      "nll_loss": 1.596862554550171,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -36.168251037597656,
      "rewards/margins": 0.6988004446029663,
      "rewards/rejected": -36.8670539855957,
      "step": 320
    },
    {
      "epoch": 0.6910994764397905,
      "grad_norm": 776.851806640625,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": -0.7262955904006958,
      "logits/rejected": -0.7191631197929382,
      "logps/chosen": -372.2191162109375,
      "logps/rejected": -400.3269348144531,
      "loss": 23.3532,
      "nll_loss": 1.5126838684082031,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -37.22191619873047,
      "rewards/margins": 2.810784101486206,
      "rewards/rejected": -40.03269577026367,
      "step": 330
    },
    {
      "epoch": 0.7120418848167539,
      "grad_norm": 518.089111328125,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": -0.7572416067123413,
      "logits/rejected": -0.7408580780029297,
      "logps/chosen": -341.0003356933594,
      "logps/rejected": -368.7081604003906,
      "loss": 23.7244,
      "nll_loss": 1.5676231384277344,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -34.10003662109375,
      "rewards/margins": 2.7707810401916504,
      "rewards/rejected": -36.870811462402344,
      "step": 340
    },
    {
      "epoch": 0.7329842931937173,
      "grad_norm": 778.4837646484375,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": -0.7688989639282227,
      "logits/rejected": -0.7650505304336548,
      "logps/chosen": -354.57867431640625,
      "logps/rejected": -391.5706481933594,
      "loss": 22.8915,
      "nll_loss": 1.5152699947357178,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -35.45786666870117,
      "rewards/margins": 3.6991982460021973,
      "rewards/rejected": -39.157066345214844,
      "step": 350
    },
    {
      "epoch": 0.7539267015706806,
      "grad_norm": 949.5282592773438,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": -0.7362595796585083,
      "logits/rejected": -0.7329570651054382,
      "logps/chosen": -350.1195068359375,
      "logps/rejected": -367.3720703125,
      "loss": 23.4713,
      "nll_loss": 1.4863172769546509,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -35.01194763183594,
      "rewards/margins": 1.7252594232559204,
      "rewards/rejected": -36.73720932006836,
      "step": 360
    },
    {
      "epoch": 0.774869109947644,
      "grad_norm": 623.1497802734375,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": -0.7591558694839478,
      "logits/rejected": -0.7718418836593628,
      "logps/chosen": -376.29229736328125,
      "logps/rejected": -411.29266357421875,
      "loss": 23.6771,
      "nll_loss": 1.399916410446167,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.629234313964844,
      "rewards/margins": 3.500037431716919,
      "rewards/rejected": -41.12926483154297,
      "step": 370
    },
    {
      "epoch": 0.7958115183246073,
      "grad_norm": 551.5185546875,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": -0.7095741033554077,
      "logits/rejected": -0.7066128849983215,
      "logps/chosen": -371.6845703125,
      "logps/rejected": -378.3497619628906,
      "loss": 24.297,
      "nll_loss": 1.507645845413208,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.16845703125,
      "rewards/margins": 0.6665191054344177,
      "rewards/rejected": -37.8349723815918,
      "step": 380
    },
    {
      "epoch": 0.8167539267015707,
      "grad_norm": 537.383056640625,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": -0.7830756902694702,
      "logits/rejected": -0.7760699987411499,
      "logps/chosen": -338.2967224121094,
      "logps/rejected": -373.68524169921875,
      "loss": 20.8009,
      "nll_loss": 1.4299148321151733,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -33.829673767089844,
      "rewards/margins": 3.538851261138916,
      "rewards/rejected": -37.36852264404297,
      "step": 390
    },
    {
      "epoch": 0.837696335078534,
      "grad_norm": 562.61572265625,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": -0.7369313836097717,
      "logits/rejected": -0.7653532028198242,
      "logps/chosen": -361.5699462890625,
      "logps/rejected": -360.4308166503906,
      "loss": 23.6103,
      "nll_loss": 1.4506025314331055,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -36.1569938659668,
      "rewards/margins": -0.11391093581914902,
      "rewards/rejected": -36.04308319091797,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_logits/chosen": -0.7660086750984192,
      "eval_logits/rejected": -0.7502105832099915,
      "eval_logps/chosen": -368.87127685546875,
      "eval_logps/rejected": -387.3282775878906,
      "eval_loss": 5.662460803985596,
      "eval_nll_loss": 1.3492006063461304,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": -36.88712692260742,
      "eval_rewards/margins": 1.845699429512024,
      "eval_rewards/rejected": -38.732826232910156,
      "eval_runtime": 23.0343,
      "eval_samples_per_second": 86.827,
      "eval_steps_per_second": 2.735,
      "step": 400
    },
    {
      "epoch": 0.8586387434554974,
      "grad_norm": 519.74755859375,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": -0.7296323180198669,
      "logits/rejected": -0.7103043794631958,
      "logps/chosen": -362.31890869140625,
      "logps/rejected": -402.84063720703125,
      "loss": 23.8299,
      "nll_loss": 1.475502371788025,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -36.231895446777344,
      "rewards/margins": 4.052168369293213,
      "rewards/rejected": -40.284061431884766,
      "step": 410
    },
    {
      "epoch": 0.8795811518324608,
      "grad_norm": 791.4617309570312,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": -0.7181447744369507,
      "logits/rejected": -0.7083781957626343,
      "logps/chosen": -403.479248046875,
      "logps/rejected": -397.3186950683594,
      "loss": 22.7265,
      "nll_loss": 1.512879490852356,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.347923278808594,
      "rewards/margins": -0.6160532236099243,
      "rewards/rejected": -39.731868743896484,
      "step": 420
    },
    {
      "epoch": 0.900523560209424,
      "grad_norm": 559.8237915039062,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": -0.7705979943275452,
      "logits/rejected": -0.7675026655197144,
      "logps/chosen": -378.0232238769531,
      "logps/rejected": -406.818115234375,
      "loss": 23.3836,
      "nll_loss": 1.365422248840332,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -37.80231857299805,
      "rewards/margins": 2.879490613937378,
      "rewards/rejected": -40.68181610107422,
      "step": 430
    },
    {
      "epoch": 0.9214659685863874,
      "grad_norm": 655.5603637695312,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": -0.7454264760017395,
      "logits/rejected": -0.7567831873893738,
      "logps/chosen": -374.8846130371094,
      "logps/rejected": -412.54425048828125,
      "loss": 23.5113,
      "nll_loss": 1.4483903646469116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -37.488460540771484,
      "rewards/margins": 3.765960216522217,
      "rewards/rejected": -41.25442123413086,
      "step": 440
    },
    {
      "epoch": 0.9424083769633508,
      "grad_norm": 1095.772216796875,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": -0.7458157539367676,
      "logits/rejected": -0.7518310546875,
      "logps/chosen": -365.1365051269531,
      "logps/rejected": -369.6548156738281,
      "loss": 22.6642,
      "nll_loss": 1.3850022554397583,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -36.513648986816406,
      "rewards/margins": 0.4518311023712158,
      "rewards/rejected": -36.96548080444336,
      "step": 450
    },
    {
      "epoch": 0.9633507853403142,
      "grad_norm": 877.9326782226562,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": -0.7453271150588989,
      "logits/rejected": -0.7776076197624207,
      "logps/chosen": -409.984375,
      "logps/rejected": -432.01141357421875,
      "loss": 23.1426,
      "nll_loss": 1.4942188262939453,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.998443603515625,
      "rewards/margins": 2.202702760696411,
      "rewards/rejected": -43.201148986816406,
      "step": 460
    },
    {
      "epoch": 0.9842931937172775,
      "grad_norm": 576.8656616210938,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": -0.7500009536743164,
      "logits/rejected": -0.7585066556930542,
      "logps/chosen": -346.47332763671875,
      "logps/rejected": -393.7072448730469,
      "loss": 24.6934,
      "nll_loss": 1.4887354373931885,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -34.64733123779297,
      "rewards/margins": 4.723390102386475,
      "rewards/rejected": -39.37071990966797,
      "step": 470
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 28.95567873588898,
      "train_runtime": 2691.8939,
      "train_samples_per_second": 22.711,
      "train_steps_per_second": 0.177
    }
  ],
  "logging_steps": 10,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}