{
  "best_global_step": 2500,
  "best_metric": 0.45733407139778137,
  "best_model_checkpoint": "models/dpo_default/checkpoint-2500",
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2757,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010884353741496598,
      "grad_norm": 0.7077187895774841,
      "learning_rate": 2.168674698795181e-05,
      "logits/chosen": -2.2123303413391113,
      "logits/rejected": -2.1015005111694336,
      "logps/chosen": -76.98332214355469,
      "logps/rejected": -83.68041229248047,
      "loss": 0.6929964542388916,
      "rewards/accuracies": 0.3492187559604645,
      "rewards/chosen": 0.0012271858286112547,
      "rewards/margins": 0.003317171009257436,
      "rewards/rejected": -0.0020899856463074684,
      "step": 10
    },
    {
      "epoch": 0.021768707482993196,
      "grad_norm": 0.9575150012969971,
      "learning_rate": 4.578313253012048e-05,
      "logits/chosen": -2.3612475395202637,
      "logits/rejected": -2.317596435546875,
      "logps/chosen": -79.90132904052734,
      "logps/rejected": -86.09098815917969,
      "loss": 0.6921308994293213,
      "rewards/accuracies": 0.4671874940395355,
      "rewards/chosen": 0.0005894556525163352,
      "rewards/margins": 0.00607589865103364,
      "rewards/rejected": -0.005486442707479,
      "step": 20
    },
    {
      "epoch": 0.0326530612244898,
      "grad_norm": 0.6923499703407288,
      "learning_rate": 6.987951807228917e-05,
      "logits/chosen": -2.2964720726013184,
      "logits/rejected": -2.2370924949645996,
      "logps/chosen": -80.2668228149414,
      "logps/rejected": -87.2217788696289,
      "loss": 0.6918315887451172,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.019290009513497353,
      "rewards/margins": 0.00765924621373415,
      "rewards/rejected": -0.026949256658554077,
      "step": 30
    },
    {
      "epoch": 0.04353741496598639,
      "grad_norm": 0.6989305019378662,
      "learning_rate": 9.397590361445784e-05,
      "logits/chosen": -2.3046586513519287,
      "logits/rejected": -2.2639377117156982,
      "logps/chosen": -84.65824890136719,
      "logps/rejected": -91.65088653564453,
      "loss": 0.6887531280517578,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08198987692594528,
      "rewards/margins": 0.017511751502752304,
      "rewards/rejected": -0.09950163215398788,
      "step": 40
    },
    {
      "epoch": 0.05442176870748299,
      "grad_norm": 0.7514657378196716,
      "learning_rate": 0.00011807228915662652,
      "logits/chosen": -2.576209545135498,
      "logits/rejected": -2.5158941745758057,
      "logps/chosen": -86.98737335205078,
      "logps/rejected": -93.26823425292969,
      "loss": 0.6795042991638184,
      "rewards/accuracies": 0.5015624761581421,
      "rewards/chosen": -0.16211052238941193,
      "rewards/margins": 0.05032004788517952,
      "rewards/rejected": -0.21243056654930115,
      "step": 50
    },
    {
      "epoch": 0.0653061224489796,
      "grad_norm": 0.8640028834342957,
      "learning_rate": 0.00014216867469879518,
      "logits/chosen": -2.4957079887390137,
      "logits/rejected": -2.3503634929656982,
      "logps/chosen": -83.06327819824219,
      "logps/rejected": -88.82142639160156,
      "loss": 0.6818838596343995,
      "rewards/accuracies": 0.51171875,
      "rewards/chosen": -0.2020430564880371,
      "rewards/margins": 0.05818678066134453,
      "rewards/rejected": -0.26022982597351074,
      "step": 60
    },
    {
      "epoch": 0.0761904761904762,
      "grad_norm": 0.8207036256790161,
      "learning_rate": 0.00016626506024096388,
      "logits/chosen": -2.699256420135498,
      "logits/rejected": -2.621330499649048,
      "logps/chosen": -81.74055480957031,
      "logps/rejected": -87.18543243408203,
      "loss": 0.6767777442932129,
      "rewards/accuracies": 0.520312488079071,
      "rewards/chosen": -0.19228772819042206,
      "rewards/margins": 0.07377694547176361,
      "rewards/rejected": -0.26606467366218567,
      "step": 70
    },
    {
      "epoch": 0.08707482993197278,
      "grad_norm": 0.8356528878211975,
      "learning_rate": 0.00019036144578313252,
      "logits/chosen": -2.595083236694336,
      "logits/rejected": -2.484344005584717,
      "logps/chosen": -82.08795166015625,
      "logps/rejected": -89.19499206542969,
      "loss": 0.6667680740356445,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": -0.2063063681125641,
      "rewards/margins": 0.11366782337427139,
      "rewards/rejected": -0.3199741840362549,
      "step": 80
    },
    {
      "epoch": 0.09795918367346938,
      "grad_norm": 0.7759003639221191,
      "learning_rate": 0.00019955123410620793,
      "logits/chosen": -2.369516611099243,
      "logits/rejected": -2.255380392074585,
      "logps/chosen": -80.26573944091797,
      "logps/rejected": -88.53955841064453,
      "loss": 0.6667191505432128,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.1989096850156784,
      "rewards/margins": 0.11919046938419342,
      "rewards/rejected": -0.31810012459754944,
      "step": 90
    },
    {
      "epoch": 0.10884353741496598,
      "grad_norm": 0.7274847626686096,
      "learning_rate": 0.0001988032909498878,
      "logits/chosen": -2.4584925174713135,
      "logits/rejected": -2.306166172027588,
      "logps/chosen": -85.4821548461914,
      "logps/rejected": -92.23262786865234,
      "loss": 0.6666709899902343,
      "rewards/accuracies": 0.5492187738418579,
      "rewards/chosen": -0.2420760691165924,
      "rewards/margins": 0.1285211443901062,
      "rewards/rejected": -0.370597243309021,
      "step": 100
    },
    {
      "epoch": 0.11972789115646258,
      "grad_norm": 0.8684931397438049,
      "learning_rate": 0.0001980553477935677,
      "logits/chosen": -2.1937153339385986,
      "logits/rejected": -2.128317356109619,
      "logps/chosen": -81.6711654663086,
      "logps/rejected": -89.73558807373047,
      "loss": 0.6597628593444824,
      "rewards/accuracies": 0.5523437261581421,
      "rewards/chosen": -0.1942557394504547,
      "rewards/margins": 0.15067201852798462,
      "rewards/rejected": -0.34492772817611694,
      "step": 110
    },
    {
      "epoch": 0.1306122448979592,
      "grad_norm": 0.7956843972206116,
      "learning_rate": 0.00019730740463724756,
      "logits/chosen": -2.446894884109497,
      "logits/rejected": -2.3522589206695557,
      "logps/chosen": -84.79292297363281,
      "logps/rejected": -90.49881744384766,
      "loss": 0.6632381439208984,
      "rewards/accuracies": 0.542187511920929,
      "rewards/chosen": -0.19174879789352417,
      "rewards/margins": 0.14180642366409302,
      "rewards/rejected": -0.3335552215576172,
      "step": 120
    },
    {
      "epoch": 0.1414965986394558,
      "grad_norm": 0.6242148876190186,
      "learning_rate": 0.00019655946148092746,
      "logits/chosen": -2.457613945007324,
      "logits/rejected": -2.388310670852661,
      "logps/chosen": -79.19390869140625,
      "logps/rejected": -85.8818130493164,
      "loss": 0.6509382724761963,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": -0.1272696703672409,
      "rewards/margins": 0.17109361290931702,
      "rewards/rejected": -0.29836326837539673,
      "step": 130
    },
    {
      "epoch": 0.1523809523809524,
      "grad_norm": 0.6959019899368286,
      "learning_rate": 0.00019581151832460733,
      "logits/chosen": -2.645115375518799,
      "logits/rejected": -2.5745694637298584,
      "logps/chosen": -82.52925109863281,
      "logps/rejected": -89.37605285644531,
      "loss": 0.6571790695190429,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.11609281599521637,
      "rewards/margins": 0.16595318913459778,
      "rewards/rejected": -0.28204596042633057,
      "step": 140
    },
    {
      "epoch": 0.16326530612244897,
      "grad_norm": 0.7039981484413147,
      "learning_rate": 0.00019506357516828722,
      "logits/chosen": -2.6011252403259277,
      "logits/rejected": -2.5297441482543945,
      "logps/chosen": -84.36079406738281,
      "logps/rejected": -91.32760620117188,
      "loss": 0.659122371673584,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.04060233011841774,
      "rewards/margins": 0.16302046179771423,
      "rewards/rejected": -0.20362277328968048,
      "step": 150
    },
    {
      "epoch": 0.17414965986394557,
      "grad_norm": 0.6617714166641235,
      "learning_rate": 0.0001943156320119671,
      "logits/chosen": -2.715470790863037,
      "logits/rejected": -2.5970466136932373,
      "logps/chosen": -83.14956665039062,
      "logps/rejected": -89.39061737060547,
      "loss": 0.6440523147583008,
      "rewards/accuracies": 0.5882812738418579,
      "rewards/chosen": -0.012846514582633972,
      "rewards/margins": 0.18811455368995667,
      "rewards/rejected": -0.20096108317375183,
      "step": 160
    },
    {
      "epoch": 0.18503401360544217,
      "grad_norm": 0.6483933925628662,
      "learning_rate": 0.000193567688855647,
      "logits/chosen": -2.807894468307495,
      "logits/rejected": -2.7104663848876953,
      "logps/chosen": -79.6620864868164,
      "logps/rejected": -85.63630676269531,
      "loss": 0.6529532909393311,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.05956472083926201,
      "rewards/margins": 0.190535768866539,
      "rewards/rejected": -0.2501004636287689,
      "step": 170
    },
    {
      "epoch": 0.19591836734693877,
      "grad_norm": 0.6357870697975159,
      "learning_rate": 0.00019281974569932686,
      "logits/chosen": -2.7263660430908203,
      "logits/rejected": -2.632416248321533,
      "logps/chosen": -86.24573516845703,
      "logps/rejected": -91.7590560913086,
      "loss": 0.6710307121276855,
      "rewards/accuracies": 0.5570312738418579,
      "rewards/chosen": -0.18387791514396667,
      "rewards/margins": 0.14046183228492737,
      "rewards/rejected": -0.32433977723121643,
      "step": 180
    },
    {
      "epoch": 0.20680272108843537,
      "grad_norm": 0.7087427973747253,
      "learning_rate": 0.00019207180254300675,
      "logits/chosen": -2.575040817260742,
      "logits/rejected": -2.4759504795074463,
      "logps/chosen": -83.10100555419922,
      "logps/rejected": -90.81092071533203,
      "loss": 0.6495565414428711,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": -0.2590435743331909,
      "rewards/margins": 0.18424804508686066,
      "rewards/rejected": -0.44329166412353516,
      "step": 190
    },
    {
      "epoch": 0.21768707482993196,
      "grad_norm": 0.7276835441589355,
      "learning_rate": 0.00019132385938668662,
      "logits/chosen": -2.6646244525909424,
      "logits/rejected": -2.642310857772827,
      "logps/chosen": -84.37451934814453,
      "logps/rejected": -92.43025970458984,
      "loss": 0.652790880203247,
      "rewards/accuracies": 0.5523437261581421,
      "rewards/chosen": -0.28780898451805115,
      "rewards/margins": 0.2015586793422699,
      "rewards/rejected": -0.48936766386032104,
      "step": 200
    },
    {
      "epoch": 0.22857142857142856,
      "grad_norm": 0.704118549823761,
      "learning_rate": 0.0001905759162303665,
      "logits/chosen": -2.7964985370635986,
      "logits/rejected": -2.7038846015930176,
      "logps/chosen": -83.10550689697266,
      "logps/rejected": -90.61512756347656,
      "loss": 0.6555557250976562,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": -0.17501111328601837,
      "rewards/margins": 0.1853359341621399,
      "rewards/rejected": -0.36034709215164185,
      "step": 210
    },
    {
      "epoch": 0.23945578231292516,
      "grad_norm": 0.6894403100013733,
      "learning_rate": 0.00018982797307404639,
      "logits/chosen": -2.694242000579834,
      "logits/rejected": -2.6799864768981934,
      "logps/chosen": -78.91683959960938,
      "logps/rejected": -84.78871154785156,
      "loss": 0.6462616920471191,
      "rewards/accuracies": 0.5804687738418579,
      "rewards/chosen": 0.06024743244051933,
      "rewards/margins": 0.19001427292823792,
      "rewards/rejected": -0.12976683676242828,
      "step": 220
    },
    {
      "epoch": 0.2503401360544218,
      "grad_norm": 0.6098469495773315,
      "learning_rate": 0.00018908002991772625,
      "logits/chosen": -2.9486870765686035,
      "logits/rejected": -2.8814079761505127,
      "logps/chosen": -80.17509460449219,
      "logps/rejected": -86.77090454101562,
      "loss": 0.631169319152832,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": 0.039627805352211,
      "rewards/margins": 0.2388683557510376,
      "rewards/rejected": -0.1992405354976654,
      "step": 230
    },
    {
      "epoch": 0.2612244897959184,
      "grad_norm": 0.6648709774017334,
      "learning_rate": 0.00018833208676140615,
      "logits/chosen": -3.144280195236206,
      "logits/rejected": -3.156015396118164,
      "logps/chosen": -82.75923156738281,
      "logps/rejected": -92.11023712158203,
      "loss": 0.6405491828918457,
      "rewards/accuracies": 0.592968761920929,
      "rewards/chosen": -0.15283265709877014,
      "rewards/margins": 0.2397342175245285,
      "rewards/rejected": -0.39256685972213745,
      "step": 240
    },
    {
      "epoch": 0.272108843537415,
      "grad_norm": 0.7529364824295044,
      "learning_rate": 0.00018758414360508602,
      "logits/chosen": -3.0414438247680664,
      "logits/rejected": -3.0043177604675293,
      "logps/chosen": -88.2968521118164,
      "logps/rejected": -97.04661560058594,
      "loss": 0.6290472984313965,
      "rewards/accuracies": 0.5914062261581421,
      "rewards/chosen": -0.38561299443244934,
      "rewards/margins": 0.28672346472740173,
      "rewards/rejected": -0.6723364591598511,
      "step": 250
    },
    {
      "epoch": 0.2829931972789116,
      "grad_norm": 0.627895176410675,
      "learning_rate": 0.00018683620044876591,
      "logits/chosen": -3.091475009918213,
      "logits/rejected": -2.9503979682922363,
      "logps/chosen": -87.58129119873047,
      "logps/rejected": -93.70463562011719,
      "loss": 0.6199825286865235,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -0.43795761466026306,
      "rewards/margins": 0.28398531675338745,
      "rewards/rejected": -0.7219429612159729,
      "step": 260
    },
    {
      "epoch": 0.2938775510204082,
      "grad_norm": 0.562925398349762,
      "learning_rate": 0.00018608825729244578,
      "logits/chosen": -3.0558292865753174,
      "logits/rejected": -2.9512274265289307,
      "logps/chosen": -80.02171325683594,
      "logps/rejected": -88.43952941894531,
      "loss": 0.6295814514160156,
      "rewards/accuracies": 0.5960937738418579,
      "rewards/chosen": -0.35493478178977966,
      "rewards/margins": 0.2739941477775574,
      "rewards/rejected": -0.6289288997650146,
      "step": 270
    },
    {
      "epoch": 0.3047619047619048,
      "grad_norm": 0.62668776512146,
      "learning_rate": 0.00018534031413612568,
      "logits/chosen": -3.073023557662964,
      "logits/rejected": -2.9786252975463867,
      "logps/chosen": -79.61070251464844,
      "logps/rejected": -87.7259750366211,
      "loss": 0.6259790420532226,
      "rewards/accuracies": 0.5914062261581421,
      "rewards/chosen": -0.3159467875957489,
      "rewards/margins": 0.2681388556957245,
      "rewards/rejected": -0.5840856432914734,
      "step": 280
    },
    {
      "epoch": 0.31564625850340133,
      "grad_norm": 0.6235923767089844,
      "learning_rate": 0.00018459237097980555,
      "logits/chosen": -3.129662036895752,
      "logits/rejected": -3.073840379714966,
      "logps/chosen": -81.99674987792969,
      "logps/rejected": -90.57108306884766,
      "loss": 0.6291594982147217,
      "rewards/accuracies": 0.585156261920929,
      "rewards/chosen": -0.2529909014701843,
      "rewards/margins": 0.28501176834106445,
      "rewards/rejected": -0.5380026698112488,
      "step": 290
    },
    {
      "epoch": 0.32653061224489793,
      "grad_norm": 0.6637018322944641,
      "learning_rate": 0.00018384442782348544,
      "logits/chosen": -3.1975247859954834,
      "logits/rejected": -3.1137585639953613,
      "logps/chosen": -80.12820434570312,
      "logps/rejected": -88.26316833496094,
      "loss": 0.6194732189178467,
      "rewards/accuracies": 0.600781261920929,
      "rewards/chosen": -0.0812927708029747,
      "rewards/margins": 0.3114483058452606,
      "rewards/rejected": -0.3927411139011383,
      "step": 300
    },
    {
      "epoch": 0.33741496598639453,
      "grad_norm": 0.6672607660293579,
      "learning_rate": 0.0001830964846671653,
      "logits/chosen": -2.985579252243042,
      "logits/rejected": -2.9056222438812256,
      "logps/chosen": -86.5948257446289,
      "logps/rejected": -95.234619140625,
      "loss": 0.6183744430541992,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": -0.3788287043571472,
      "rewards/margins": 0.31600421667099,
      "rewards/rejected": -0.694832980632782,
      "step": 310
    },
    {
      "epoch": 0.34829931972789113,
      "grad_norm": 0.6171224117279053,
      "learning_rate": 0.0001823485415108452,
      "logits/chosen": -3.0042669773101807,
      "logits/rejected": -2.951925754547119,
      "logps/chosen": -88.96502685546875,
      "logps/rejected": -97.89619445800781,
      "loss": 0.6247763633728027,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -0.5990578532218933,
      "rewards/margins": 0.300613671541214,
      "rewards/rejected": -0.8996715545654297,
      "step": 320
    },
    {
      "epoch": 0.35918367346938773,
      "grad_norm": 0.7893000245094299,
      "learning_rate": 0.00018160059835452508,
      "logits/chosen": -3.1310553550720215,
      "logits/rejected": -3.1132781505584717,
      "logps/chosen": -84.33443450927734,
      "logps/rejected": -94.07998657226562,
      "loss": 0.6248671531677246,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -0.6389909982681274,
      "rewards/margins": 0.29910796880722046,
      "rewards/rejected": -0.9380990266799927,
      "step": 330
    },
    {
      "epoch": 0.37006802721088433,
      "grad_norm": 0.5814021825790405,
      "learning_rate": 0.00018085265519820494,
      "logits/chosen": -3.1776278018951416,
      "logits/rejected": -3.08312726020813,
      "logps/chosen": -86.03544616699219,
      "logps/rejected": -93.88951110839844,
      "loss": 0.6233505249023438,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": -0.6253015398979187,
      "rewards/margins": 0.3192492425441742,
      "rewards/rejected": -0.9445506930351257,
      "step": 340
    },
    {
      "epoch": 0.38095238095238093,
      "grad_norm": 0.63541179895401,
      "learning_rate": 0.0001801047120418848,
      "logits/chosen": -2.9307355880737305,
      "logits/rejected": -2.9118292331695557,
      "logps/chosen": -89.96821594238281,
      "logps/rejected": -98.78160095214844,
      "loss": 0.6067781925201416,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": -0.5151289701461792,
      "rewards/margins": 0.34935957193374634,
      "rewards/rejected": -0.8644886016845703,
      "step": 350
    },
    {
      "epoch": 0.39183673469387753,
      "grad_norm": 0.695551872253418,
      "learning_rate": 0.0001793567688855647,
      "logits/chosen": -2.7680797576904297,
      "logits/rejected": -2.694873332977295,
      "logps/chosen": -84.47620391845703,
      "logps/rejected": -91.43312072753906,
      "loss": 0.6245352268218994,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -0.5855204463005066,
      "rewards/margins": 0.2961046099662781,
      "rewards/rejected": -0.8816250562667847,
      "step": 360
    },
    {
      "epoch": 0.40272108843537413,
      "grad_norm": 0.5492284297943115,
      "learning_rate": 0.00017860882572924458,
      "logits/chosen": -2.8768391609191895,
      "logits/rejected": -2.7919318675994873,
      "logps/chosen": -88.09062194824219,
      "logps/rejected": -94.99168395996094,
      "loss": 0.6229678630828858,
      "rewards/accuracies": 0.602343738079071,
      "rewards/chosen": -0.6255002021789551,
      "rewards/margins": 0.2889278531074524,
      "rewards/rejected": -0.9144280552864075,
      "step": 370
    },
    {
      "epoch": 0.41360544217687073,
      "grad_norm": 0.627325713634491,
      "learning_rate": 0.00017786088257292445,
      "logits/chosen": -2.6934731006622314,
      "logits/rejected": -2.595776319503784,
      "logps/chosen": -89.07003021240234,
      "logps/rejected": -97.8858871459961,
      "loss": 0.6102587699890136,
      "rewards/accuracies": 0.6078125238418579,
      "rewards/chosen": -0.5627188682556152,
      "rewards/margins": 0.3145248293876648,
      "rewards/rejected": -0.87724369764328,
      "step": 380
    },
    {
      "epoch": 0.42448979591836733,
      "grad_norm": 0.6781789064407349,
      "learning_rate": 0.00017711293941660434,
      "logits/chosen": -2.902066946029663,
      "logits/rejected": -2.792140245437622,
      "logps/chosen": -85.96211242675781,
      "logps/rejected": -94.57718658447266,
      "loss": 0.6128390789031982,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -0.5092566013336182,
      "rewards/margins": 0.33838534355163574,
      "rewards/rejected": -0.8476420640945435,
      "step": 390
    },
    {
      "epoch": 0.43537414965986393,
      "grad_norm": 0.6318673491477966,
      "learning_rate": 0.0001763649962602842,
      "logits/chosen": -2.978280544281006,
      "logits/rejected": -2.9661154747009277,
      "logps/chosen": -82.64006805419922,
      "logps/rejected": -93.13764953613281,
      "loss": 0.6212802410125733,
      "rewards/accuracies": 0.610156238079071,
      "rewards/chosen": -0.354736864566803,
      "rewards/margins": 0.31057295203208923,
      "rewards/rejected": -0.6653097867965698,
      "step": 400
    },
    {
      "epoch": 0.44625850340136053,
      "grad_norm": 0.6015628576278687,
      "learning_rate": 0.0001756170531039641,
      "logits/chosen": -3.077155590057373,
      "logits/rejected": -3.0079360008239746,
      "logps/chosen": -84.97288513183594,
      "logps/rejected": -93.01554870605469,
      "loss": 0.6140561580657959,
      "rewards/accuracies": 0.594531238079071,
      "rewards/chosen": -0.3532416820526123,
      "rewards/margins": 0.34475868940353394,
      "rewards/rejected": -0.6980003714561462,
      "step": 410
    },
    {
      "epoch": 0.45714285714285713,
      "grad_norm": 0.6051854491233826,
      "learning_rate": 0.00017486910994764398,
      "logits/chosen": -3.0866477489471436,
      "logits/rejected": -2.9812045097351074,
      "logps/chosen": -79.9186019897461,
      "logps/rejected": -88.40470886230469,
      "loss": 0.6004165649414063,
      "rewards/accuracies": 0.625781238079071,
      "rewards/chosen": -0.21888461709022522,
      "rewards/margins": 0.36811959743499756,
      "rewards/rejected": -0.5870041847229004,
      "step": 420
    },
    {
      "epoch": 0.46802721088435373,
      "grad_norm": 0.7300223708152771,
      "learning_rate": 0.00017412116679132387,
      "logits/chosen": -3.060652017593384,
      "logits/rejected": -3.027360439300537,
      "logps/chosen": -84.05428314208984,
      "logps/rejected": -91.3875503540039,
      "loss": 0.6200145244598388,
      "rewards/accuracies": 0.604687511920929,
      "rewards/chosen": -0.22208240628242493,
      "rewards/margins": 0.3296840786933899,
      "rewards/rejected": -0.5517665147781372,
      "step": 430
    },
    {
      "epoch": 0.47891156462585033,
      "grad_norm": 0.5995689630508423,
      "learning_rate": 0.00017337322363500374,
      "logits/chosen": -2.9303958415985107,
      "logits/rejected": -2.8347811698913574,
      "logps/chosen": -88.16230773925781,
      "logps/rejected": -94.56944274902344,
      "loss": 0.6032320499420166,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": -0.506262481212616,
      "rewards/margins": 0.3537197411060333,
      "rewards/rejected": -0.8599823117256165,
      "step": 440
    },
    {
      "epoch": 0.4897959183673469,
      "grad_norm": 0.6093873381614685,
      "learning_rate": 0.00017262528047868364,
      "logits/chosen": -3.054971218109131,
      "logits/rejected": -2.9332528114318848,
      "logps/chosen": -85.0113525390625,
      "logps/rejected": -94.11463928222656,
      "loss": 0.5989380359649659,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": -0.5802456140518188,
      "rewards/margins": 0.3973398804664612,
      "rewards/rejected": -0.97758549451828,
      "step": 450
    },
    {
      "epoch": 0.5006802721088436,
      "grad_norm": 0.6394651532173157,
      "learning_rate": 0.0001718773373223635,
      "logits/chosen": -2.9823951721191406,
      "logits/rejected": -2.9353485107421875,
      "logps/chosen": -86.30118560791016,
      "logps/rejected": -94.50347900390625,
      "loss": 0.6100581169128418,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -0.5156108140945435,
      "rewards/margins": 0.34870487451553345,
      "rewards/rejected": -0.8643158078193665,
      "step": 460
    },
    {
      "epoch": 0.5115646258503401,
      "grad_norm": 0.5491212010383606,
      "learning_rate": 0.0001711293941660434,
      "logits/chosen": -2.926840305328369,
      "logits/rejected": -2.8748295307159424,
      "logps/chosen": -81.23180389404297,
      "logps/rejected": -88.91163635253906,
      "loss": 0.601882791519165,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.35410138964653015,
      "rewards/margins": 0.377445787191391,
      "rewards/rejected": -0.7315471768379211,
      "step": 470
    },
    {
      "epoch": 0.5224489795918368,
      "grad_norm": 0.6345949172973633,
      "learning_rate": 0.00017038145100972327,
      "logits/chosen": -2.8460724353790283,
      "logits/rejected": -2.8185269832611084,
      "logps/chosen": -82.52976989746094,
      "logps/rejected": -91.4373779296875,
      "loss": 0.5966329097747802,
      "rewards/accuracies": 0.60546875,
      "rewards/chosen": -0.2420049011707306,
      "rewards/margins": 0.3614209294319153,
      "rewards/rejected": -0.6034258604049683,
      "step": 480
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 0.6204262971878052,
      "learning_rate": 0.00016963350785340316,
      "logits/chosen": -3.037978410720825,
      "logits/rejected": -2.9520788192749023,
      "logps/chosen": -83.21837615966797,
      "logps/rejected": -91.0192642211914,
      "loss": 0.5877750396728516,
      "rewards/accuracies": 0.6429687738418579,
      "rewards/chosen": -0.35447391867637634,
      "rewards/margins": 0.4061393737792969,
      "rewards/rejected": -0.7606132626533508,
      "step": 490
    },
    {
      "epoch": 0.54421768707483,
      "grad_norm": 0.6997068524360657,
      "learning_rate": 0.00016888556469708303,
      "logits/chosen": -3.0016021728515625,
      "logits/rejected": -2.9584157466888428,
      "logps/chosen": -88.25440979003906,
      "logps/rejected": -98.35990142822266,
      "loss": 0.5975300312042237,
      "rewards/accuracies": 0.6109374761581421,
      "rewards/chosen": -0.6453801393508911,
      "rewards/margins": 0.40526071190834045,
      "rewards/rejected": -1.0506408214569092,
      "step": 500
    },
    {
      "epoch": 0.54421768707483,
      "eval_logits/chosen": -3.0864946842193604,
      "eval_logits/rejected": -3.039473056793213,
      "eval_logps/chosen": -85.64629364013672,
      "eval_logps/rejected": -96.53662872314453,
      "eval_loss": 0.5770927667617798,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": -0.604114294052124,
      "eval_rewards/margins": 0.4646129906177521,
      "eval_rewards/rejected": -1.0687271356582642,
      "eval_runtime": 104.8956,
      "eval_samples_per_second": 22.88,
      "eval_steps_per_second": 2.86,
      "step": 500
    },
    {
      "epoch": 0.5551020408163265,
      "grad_norm": 0.5718573331832886,
      "learning_rate": 0.0001681376215407629,
      "logits/chosen": -2.98121976852417,
      "logits/rejected": -3.0054566860198975,
      "logps/chosen": -84.3095932006836,
      "logps/rejected": -96.11637115478516,
      "loss": 0.5776225090026855,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.6291953921318054,
      "rewards/margins": 0.4740595817565918,
      "rewards/rejected": -1.103255033493042,
      "step": 510
    },
    {
      "epoch": 0.5659863945578232,
      "grad_norm": 0.6719241738319397,
      "learning_rate": 0.0001673896783844428,
      "logits/chosen": -3.1065757274627686,
      "logits/rejected": -3.057704448699951,
      "logps/chosen": -84.51625061035156,
      "logps/rejected": -95.70087432861328,
      "loss": 0.5797244071960449,
      "rewards/accuracies": 0.6539062261581421,
      "rewards/chosen": -0.6672269105911255,
      "rewards/margins": 0.4843328595161438,
      "rewards/rejected": -1.1515597105026245,
      "step": 520
    },
    {
      "epoch": 0.5768707482993197,
      "grad_norm": 0.6521556377410889,
      "learning_rate": 0.00016664173522812267,
      "logits/chosen": -3.0161333084106445,
      "logits/rejected": -3.000890016555786,
      "logps/chosen": -85.24427795410156,
      "logps/rejected": -94.50189208984375,
      "loss": 0.5926599979400635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6613038778305054,
      "rewards/margins": 0.4472618103027344,
      "rewards/rejected": -1.1085655689239502,
      "step": 530
    },
    {
      "epoch": 0.5877551020408164,
      "grad_norm": 0.9451216459274292,
      "learning_rate": 0.00016589379207180256,
      "logits/chosen": -2.8275389671325684,
      "logits/rejected": -2.8195090293884277,
      "logps/chosen": -87.73820495605469,
      "logps/rejected": -98.05986785888672,
      "loss": 0.6018657684326172,
      "rewards/accuracies": 0.6265624761581421,
      "rewards/chosen": -0.6853042840957642,
      "rewards/margins": 0.4554131031036377,
      "rewards/rejected": -1.1407173871994019,
      "step": 540
    },
    {
      "epoch": 0.5986394557823129,
      "grad_norm": 0.610352098941803,
      "learning_rate": 0.00016514584891548243,
      "logits/chosen": -2.712162733078003,
      "logits/rejected": -2.6361680030822754,
      "logps/chosen": -86.30110168457031,
      "logps/rejected": -96.72479248046875,
      "loss": 0.5942647457122803,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7625668048858643,
      "rewards/margins": 0.41246652603149414,
      "rewards/rejected": -1.1750333309173584,
      "step": 550
    },
    {
      "epoch": 0.6095238095238096,
      "grad_norm": 0.5952523946762085,
      "learning_rate": 0.00016439790575916233,
      "logits/chosen": -2.773089647293091,
      "logits/rejected": -2.675023078918457,
      "logps/chosen": -85.59429168701172,
      "logps/rejected": -94.7846908569336,
      "loss": 0.5996862411499023,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": -0.7641364336013794,
      "rewards/margins": 0.39174142479896545,
      "rewards/rejected": -1.1558778285980225,
      "step": 560
    },
    {
      "epoch": 0.6204081632653061,
      "grad_norm": 0.6582514047622681,
      "learning_rate": 0.0001636499626028422,
      "logits/chosen": -2.78428316116333,
      "logits/rejected": -2.6511282920837402,
      "logps/chosen": -90.34364318847656,
      "logps/rejected": -99.95307159423828,
      "loss": 0.5831423759460449,
      "rewards/accuracies": 0.625781238079071,
      "rewards/chosen": -0.7876542210578918,
      "rewards/margins": 0.46621885895729065,
      "rewards/rejected": -1.2538731098175049,
      "step": 570
    },
    {
      "epoch": 0.6312925170068027,
      "grad_norm": 0.5589143633842468,
      "learning_rate": 0.0001629020194465221,
      "logits/chosen": -2.9068286418914795,
      "logits/rejected": -2.8854146003723145,
      "logps/chosen": -83.24221801757812,
      "logps/rejected": -91.76869201660156,
      "loss": 0.5974715709686279,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.520553469657898,
      "rewards/margins": 0.39965444803237915,
      "rewards/rejected": -0.9202079772949219,
      "step": 580
    },
    {
      "epoch": 0.6421768707482993,
      "grad_norm": 0.6037158370018005,
      "learning_rate": 0.00016215407629020196,
      "logits/chosen": -2.9200706481933594,
      "logits/rejected": -2.8421108722686768,
      "logps/chosen": -83.75981140136719,
      "logps/rejected": -94.21287536621094,
      "loss": 0.5780706405639648,
      "rewards/accuracies": 0.641406238079071,
      "rewards/chosen": -0.40980464220046997,
      "rewards/margins": 0.4663706421852112,
      "rewards/rejected": -0.8761752843856812,
      "step": 590
    },
    {
      "epoch": 0.6530612244897959,
      "grad_norm": 0.6423654556274414,
      "learning_rate": 0.00016140613313388185,
      "logits/chosen": -2.929079532623291,
      "logits/rejected": -2.9537947177886963,
      "logps/chosen": -86.27957916259766,
      "logps/rejected": -96.39942932128906,
      "loss": 0.5839208602905274,
      "rewards/accuracies": 0.6585937738418579,
      "rewards/chosen": -0.5493398904800415,
      "rewards/margins": 0.4438067376613617,
      "rewards/rejected": -0.9931465983390808,
      "step": 600
    },
    {
      "epoch": 0.6639455782312925,
      "grad_norm": 0.5478850603103638,
      "learning_rate": 0.00016065818997756172,
      "logits/chosen": -2.835813283920288,
      "logits/rejected": -2.7998528480529785,
      "logps/chosen": -87.89683532714844,
      "logps/rejected": -96.35087585449219,
      "loss": 0.5724361896514892,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5621733665466309,
      "rewards/margins": 0.4720228612422943,
      "rewards/rejected": -1.034196138381958,
      "step": 610
    },
    {
      "epoch": 0.6748299319727891,
      "grad_norm": 0.671947717666626,
      "learning_rate": 0.0001599102468212416,
      "logits/chosen": -2.9626426696777344,
      "logits/rejected": -2.8480992317199707,
      "logps/chosen": -88.1216812133789,
      "logps/rejected": -98.25172424316406,
      "loss": 0.5745330333709717,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -0.7491940259933472,
      "rewards/margins": 0.4780084490776062,
      "rewards/rejected": -1.2272025346755981,
      "step": 620
    },
    {
      "epoch": 0.6857142857142857,
      "grad_norm": 0.6402953267097473,
      "learning_rate": 0.00015916230366492146,
      "logits/chosen": -2.9432570934295654,
      "logits/rejected": -2.97039794921875,
      "logps/chosen": -90.78302001953125,
      "logps/rejected": -103.5708236694336,
      "loss": 0.5667342185974121,
      "rewards/accuracies": 0.66015625,
      "rewards/chosen": -0.9368416666984558,
      "rewards/margins": 0.5184968709945679,
      "rewards/rejected": -1.455338478088379,
      "step": 630
    },
    {
      "epoch": 0.6965986394557823,
      "grad_norm": 0.6474503874778748,
      "learning_rate": 0.00015841436050860136,
      "logits/chosen": -2.978628635406494,
      "logits/rejected": -2.9383959770202637,
      "logps/chosen": -89.43360900878906,
      "logps/rejected": -100.71207427978516,
      "loss": 0.5545726776123047,
      "rewards/accuracies": 0.6460937261581421,
      "rewards/chosen": -0.9920727610588074,
      "rewards/margins": 0.5569532513618469,
      "rewards/rejected": -1.5490261316299438,
      "step": 640
    },
    {
      "epoch": 0.7074829931972789,
      "grad_norm": 0.7136415243148804,
      "learning_rate": 0.00015766641735228122,
      "logits/chosen": -3.020536184310913,
      "logits/rejected": -2.9737136363983154,
      "logps/chosen": -91.73661041259766,
      "logps/rejected": -102.14755249023438,
      "loss": 0.5740270614624023,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.027769923210144,
      "rewards/margins": 0.5058225393295288,
      "rewards/rejected": -1.5335925817489624,
      "step": 650
    },
    {
      "epoch": 0.7183673469387755,
      "grad_norm": 0.601219892501831,
      "learning_rate": 0.0001569184741959611,
      "logits/chosen": -3.1565613746643066,
      "logits/rejected": -3.1246376037597656,
      "logps/chosen": -90.3838119506836,
      "logps/rejected": -101.56874084472656,
      "loss": 0.5599509239196777,
      "rewards/accuracies": 0.655468761920929,
      "rewards/chosen": -1.081892490386963,
      "rewards/margins": 0.559756875038147,
      "rewards/rejected": -1.6416492462158203,
      "step": 660
    },
    {
      "epoch": 0.7292517006802721,
      "grad_norm": 0.6408318877220154,
      "learning_rate": 0.000156170531039641,
      "logits/chosen": -3.2291476726531982,
      "logits/rejected": -3.2356181144714355,
      "logps/chosen": -89.50052642822266,
      "logps/rejected": -100.52101135253906,
      "loss": 0.5731996536254883,
      "rewards/accuracies": 0.647656261920929,
      "rewards/chosen": -1.1054667234420776,
      "rewards/margins": 0.5072154998779297,
      "rewards/rejected": -1.6126823425292969,
      "step": 670
    },
    {
      "epoch": 0.7401360544217687,
      "grad_norm": 0.6599347591400146,
      "learning_rate": 0.00015542258788332086,
      "logits/chosen": -3.2114415168762207,
      "logits/rejected": -3.172548532485962,
      "logps/chosen": -91.56523132324219,
      "logps/rejected": -102.18913269042969,
      "loss": 0.5616633892059326,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -0.8735089302062988,
      "rewards/margins": 0.5582183599472046,
      "rewards/rejected": -1.4317272901535034,
      "step": 680
    },
    {
      "epoch": 0.7510204081632653,
      "grad_norm": 0.668644905090332,
      "learning_rate": 0.00015467464472700075,
      "logits/chosen": -3.2348804473876953,
      "logits/rejected": -3.097151041030884,
      "logps/chosen": -90.90170288085938,
      "logps/rejected": -102.48905944824219,
      "loss": 0.549819803237915,
      "rewards/accuracies": 0.6695312261581421,
      "rewards/chosen": -0.6966196894645691,
      "rewards/margins": 0.6086488366127014,
      "rewards/rejected": -1.3052685260772705,
      "step": 690
    },
    {
      "epoch": 0.7619047619047619,
      "grad_norm": 0.776343822479248,
      "learning_rate": 0.00015392670157068062,
      "logits/chosen": -3.261051893234253,
      "logits/rejected": -3.150120258331299,
      "logps/chosen": -85.77153015136719,
      "logps/rejected": -96.22288513183594,
      "loss": 0.5865112781524658,
      "rewards/accuracies": 0.6539062261581421,
      "rewards/chosen": -0.6080220937728882,
      "rewards/margins": 0.5111768841743469,
      "rewards/rejected": -1.1191989183425903,
      "step": 700
    },
    {
      "epoch": 0.7727891156462585,
      "grad_norm": 0.6534438729286194,
      "learning_rate": 0.00015317875841436052,
      "logits/chosen": -2.9951682090759277,
      "logits/rejected": -2.9624991416931152,
      "logps/chosen": -88.7505874633789,
      "logps/rejected": -99.82380676269531,
      "loss": 0.5636815071105957,
      "rewards/accuracies": 0.6507812738418579,
      "rewards/chosen": -0.7708442807197571,
      "rewards/margins": 0.5377318263053894,
      "rewards/rejected": -1.308576226234436,
      "step": 710
    },
    {
      "epoch": 0.7836734693877551,
      "grad_norm": 0.5064298510551453,
      "learning_rate": 0.0001524308152580404,
      "logits/chosen": -3.1614372730255127,
      "logits/rejected": -3.1034646034240723,
      "logps/chosen": -84.72537994384766,
      "logps/rejected": -96.15172576904297,
      "loss": 0.5600537300109864,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8120764493942261,
      "rewards/margins": 0.5258986353874207,
      "rewards/rejected": -1.337975263595581,
      "step": 720
    },
    {
      "epoch": 0.7945578231292517,
      "grad_norm": 0.6030212640762329,
      "learning_rate": 0.00015168287210172028,
      "logits/chosen": -3.1010847091674805,
      "logits/rejected": -3.0516085624694824,
      "logps/chosen": -85.84700775146484,
      "logps/rejected": -96.8555908203125,
      "loss": 0.5588539123535157,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7111436128616333,
      "rewards/margins": 0.5404245853424072,
      "rewards/rejected": -1.251568078994751,
      "step": 730
    },
    {
      "epoch": 0.8054421768707483,
      "grad_norm": 0.6080058217048645,
      "learning_rate": 0.00015093492894540015,
      "logits/chosen": -3.1770665645599365,
      "logits/rejected": -3.198453187942505,
      "logps/chosen": -91.16377258300781,
      "logps/rejected": -101.21217346191406,
      "loss": 0.5411731719970703,
      "rewards/accuracies": 0.667187511920929,
      "rewards/chosen": -0.7363663911819458,
      "rewards/margins": 0.5946365594863892,
      "rewards/rejected": -1.331002950668335,
      "step": 740
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 0.6249299645423889,
      "learning_rate": 0.00015018698578908005,
      "logits/chosen": -3.2993998527526855,
      "logits/rejected": -3.160282611846924,
      "logps/chosen": -85.68218231201172,
      "logps/rejected": -97.8273696899414,
      "loss": 0.5476407051086426,
      "rewards/accuracies": 0.66796875,
      "rewards/chosen": -0.6863436102867126,
      "rewards/margins": 0.6311149597167969,
      "rewards/rejected": -1.3174583911895752,
      "step": 750
    },
    {
      "epoch": 0.8272108843537415,
      "grad_norm": 0.7027069926261902,
      "learning_rate": 0.00014943904263275992,
      "logits/chosen": -3.371492862701416,
      "logits/rejected": -3.292865037918091,
      "logps/chosen": -87.8956298828125,
      "logps/rejected": -101.7688980102539,
      "loss": 0.5395628929138183,
      "rewards/accuracies": 0.6695312261581421,
      "rewards/chosen": -0.8498029708862305,
      "rewards/margins": 0.6512821912765503,
      "rewards/rejected": -1.5010850429534912,
      "step": 760
    },
    {
      "epoch": 0.8380952380952381,
      "grad_norm": 0.6294671893119812,
      "learning_rate": 0.0001486910994764398,
      "logits/chosen": -3.389573574066162,
      "logits/rejected": -3.3465969562530518,
      "logps/chosen": -89.55952453613281,
      "logps/rejected": -102.7020263671875,
      "loss": 0.5348502635955811,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -1.0625698566436768,
      "rewards/margins": 0.6796306371688843,
      "rewards/rejected": -1.742200493812561,
      "step": 770
    },
    {
      "epoch": 0.8489795918367347,
      "grad_norm": 0.637783944606781,
      "learning_rate": 0.00014794315632011968,
      "logits/chosen": -3.226806163787842,
      "logits/rejected": -3.1543614864349365,
      "logps/chosen": -92.14598846435547,
      "logps/rejected": -104.5304946899414,
      "loss": 0.5414380073547364,
      "rewards/accuracies": 0.667187511920929,
      "rewards/chosen": -1.2139638662338257,
      "rewards/margins": 0.6445013284683228,
      "rewards/rejected": -1.8584649562835693,
      "step": 780
    },
    {
      "epoch": 0.8598639455782313,
      "grad_norm": 0.7089376449584961,
      "learning_rate": 0.00014719521316379955,
      "logits/chosen": -3.0788469314575195,
      "logits/rejected": -3.0586276054382324,
      "logps/chosen": -92.21556854248047,
      "logps/rejected": -102.19468688964844,
      "loss": 0.5625462532043457,
      "rewards/accuracies": 0.6351562738418579,
      "rewards/chosen": -1.1437532901763916,
      "rewards/margins": 0.5667166709899902,
      "rewards/rejected": -1.7104698419570923,
      "step": 790
    },
    {
      "epoch": 0.8707482993197279,
      "grad_norm": 0.6719876527786255,
      "learning_rate": 0.00014644727000747944,
      "logits/chosen": -3.21879506111145,
      "logits/rejected": -3.1802637577056885,
      "logps/chosen": -87.95653533935547,
      "logps/rejected": -99.78177642822266,
      "loss": 0.5563519954681396,
      "rewards/accuracies": 0.6585937738418579,
      "rewards/chosen": -1.027986764907837,
      "rewards/margins": 0.5857411623001099,
      "rewards/rejected": -1.6137279272079468,
      "step": 800
    },
    {
      "epoch": 0.8816326530612245,
      "grad_norm": 0.7437763810157776,
      "learning_rate": 0.0001456993268511593,
      "logits/chosen": -3.17462158203125,
      "logits/rejected": -3.1289212703704834,
      "logps/chosen": -90.2762680053711,
      "logps/rejected": -101.78657531738281,
      "loss": 0.5628280162811279,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -1.0404772758483887,
      "rewards/margins": 0.5995458364486694,
      "rewards/rejected": -1.6400229930877686,
      "step": 810
    },
    {
      "epoch": 0.8925170068027211,
      "grad_norm": 0.7401530146598816,
      "learning_rate": 0.0001449513836948392,
      "logits/chosen": -3.1435704231262207,
      "logits/rejected": -3.0372138023376465,
      "logps/chosen": -92.34797668457031,
      "logps/rejected": -103.2850341796875,
      "loss": 0.5592099189758301,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.03983473777771,
      "rewards/margins": 0.5651776194572449,
      "rewards/rejected": -1.60501229763031,
      "step": 820
    },
    {
      "epoch": 0.9034013605442177,
      "grad_norm": 0.6176671385765076,
      "learning_rate": 0.00014420344053851908,
      "logits/chosen": -3.2093448638916016,
      "logits/rejected": -3.1419150829315186,
      "logps/chosen": -92.94093322753906,
      "logps/rejected": -102.44952392578125,
      "loss": 0.5435313224792481,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9639943838119507,
      "rewards/margins": 0.598076343536377,
      "rewards/rejected": -1.562070608139038,
      "step": 830
    },
    {
      "epoch": 0.9142857142857143,
      "grad_norm": 0.6202688217163086,
      "learning_rate": 0.00014345549738219897,
      "logits/chosen": -3.3524563312530518,
      "logits/rejected": -3.2346653938293457,
      "logps/chosen": -91.24100494384766,
      "logps/rejected": -101.37155151367188,
      "loss": 0.5487663269042968,
      "rewards/accuracies": 0.678906261920929,
      "rewards/chosen": -0.9898284673690796,
      "rewards/margins": 0.5904334783554077,
      "rewards/rejected": -1.5802619457244873,
      "step": 840
    },
    {
      "epoch": 0.9251700680272109,
      "grad_norm": 0.6689825057983398,
      "learning_rate": 0.00014270755422587884,
      "logits/chosen": -3.454129695892334,
      "logits/rejected": -3.3540592193603516,
      "logps/chosen": -89.66830444335938,
      "logps/rejected": -102.6904525756836,
      "loss": 0.5206645011901856,
      "rewards/accuracies": 0.68359375,
      "rewards/chosen": -0.9211218953132629,
      "rewards/margins": 0.658340573310852,
      "rewards/rejected": -1.5794624090194702,
      "step": 850
    },
    {
      "epoch": 0.9360544217687075,
      "grad_norm": 0.738223671913147,
      "learning_rate": 0.00014195961106955874,
      "logits/chosen": -3.5349831581115723,
      "logits/rejected": -3.452519655227661,
      "logps/chosen": -92.4320297241211,
      "logps/rejected": -103.63383483886719,
      "loss": 0.5230117797851562,
      "rewards/accuracies": 0.6976562738418579,
      "rewards/chosen": -0.9207289814949036,
      "rewards/margins": 0.6930528879165649,
      "rewards/rejected": -1.6137816905975342,
      "step": 860
    },
    {
      "epoch": 0.9469387755102041,
      "grad_norm": 0.702201247215271,
      "learning_rate": 0.0001412116679132386,
      "logits/chosen": -3.439744234085083,
      "logits/rejected": -3.3527960777282715,
      "logps/chosen": -91.38029479980469,
      "logps/rejected": -104.20161437988281,
      "loss": 0.5320132732391357,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -1.1446105241775513,
      "rewards/margins": 0.6743196845054626,
      "rewards/rejected": -1.8189302682876587,
      "step": 870
    },
    {
      "epoch": 0.9578231292517007,
      "grad_norm": 0.6528463363647461,
      "learning_rate": 0.0001404637247569185,
      "logits/chosen": -3.513301134109497,
      "logits/rejected": -3.5274531841278076,
      "logps/chosen": -90.98865509033203,
      "logps/rejected": -103.4592056274414,
      "loss": 0.5318188667297363,
      "rewards/accuracies": 0.6773437261581421,
      "rewards/chosen": -1.1643035411834717,
      "rewards/margins": 0.6567374467849731,
      "rewards/rejected": -1.8210411071777344,
      "step": 880
    },
    {
      "epoch": 0.9687074829931973,
      "grad_norm": 0.6700842976570129,
      "learning_rate": 0.00013971578160059837,
      "logits/chosen": -3.6123263835906982,
      "logits/rejected": -3.54679536819458,
      "logps/chosen": -89.24981689453125,
      "logps/rejected": -101.41873931884766,
      "loss": 0.5319746017456055,
      "rewards/accuracies": 0.678906261920929,
      "rewards/chosen": -1.0091780424118042,
      "rewards/margins": 0.707198977470398,
      "rewards/rejected": -1.7163772583007812,
      "step": 890
    },
    {
      "epoch": 0.9795918367346939,
      "grad_norm": 0.6881840229034424,
      "learning_rate": 0.00013896783844427824,
      "logits/chosen": -3.528007984161377,
      "logits/rejected": -3.411106824874878,
      "logps/chosen": -88.0205078125,
      "logps/rejected": -99.57227325439453,
      "loss": 0.5377126693725586,
      "rewards/accuracies": 0.680468738079071,
      "rewards/chosen": -0.9363842010498047,
      "rewards/margins": 0.6687086820602417,
      "rewards/rejected": -1.605093002319336,
      "step": 900
    },
    {
      "epoch": 0.9904761904761905,
      "grad_norm": 0.6982113718986511,
      "learning_rate": 0.0001382198952879581,
      "logits/chosen": -3.544215440750122,
      "logits/rejected": -3.499542236328125,
      "logps/chosen": -87.49764251708984,
      "logps/rejected": -98.82611083984375,
      "loss": 0.5362007141113281,
      "rewards/accuracies": 0.65234375,
      "rewards/chosen": -0.8832570314407349,
      "rewards/margins": 0.6539750695228577,
      "rewards/rejected": -1.5372319221496582,
      "step": 910
    },
    {
      "epoch": 1.0010884353741496,
      "grad_norm": 0.49051275849342346,
      "learning_rate": 0.000137471952131638,
      "logits/chosen": -3.4829673767089844,
      "logits/rejected": -3.455536365509033,
      "logps/chosen": -86.75341796875,
      "logps/rejected": -98.81475067138672,
      "loss": 0.5034448623657226,
      "rewards/accuracies": 0.7075320482254028,
      "rewards/chosen": -0.9121702909469604,
      "rewards/margins": 0.770575225353241,
      "rewards/rejected": -1.6827455759048462,
      "step": 920
    },
    {
      "epoch": 1.0119727891156463,
      "grad_norm": 0.5276588201522827,
      "learning_rate": 0.00013672400897531787,
      "logits/chosen": -3.5929553508758545,
      "logits/rejected": -3.5475852489471436,
      "logps/chosen": -92.64207458496094,
      "logps/rejected": -107.84423828125,
      "loss": 0.4122049808502197,
      "rewards/accuracies": 0.7789062261581421,
      "rewards/chosen": -1.0530273914337158,
      "rewards/margins": 1.0623085498809814,
      "rewards/rejected": -2.1153359413146973,
      "step": 930
    },
    {
      "epoch": 1.022857142857143,
      "grad_norm": 0.5503054261207581,
      "learning_rate": 0.00013597606581899777,
      "logits/chosen": -3.6768276691436768,
      "logits/rejected": -3.5878639221191406,
      "logps/chosen": -94.07461547851562,
      "logps/rejected": -112.07344055175781,
      "loss": 0.40670342445373536,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -1.2350085973739624,
      "rewards/margins": 1.135968804359436,
      "rewards/rejected": -2.3709776401519775,
      "step": 940
    },
    {
      "epoch": 1.0337414965986396,
      "grad_norm": 0.5421497225761414,
      "learning_rate": 0.00013522812266267764,
      "logits/chosen": -3.7791919708251953,
      "logits/rejected": -3.726738452911377,
      "logps/chosen": -90.40796661376953,
      "logps/rejected": -106.10577392578125,
      "loss": 0.4037026882171631,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -1.1298649311065674,
      "rewards/margins": 1.157552719116211,
      "rewards/rejected": -2.2874176502227783,
      "step": 950
    },
    {
      "epoch": 1.044625850340136,
      "grad_norm": 0.5323552489280701,
      "learning_rate": 0.0001344801795063575,
      "logits/chosen": -3.955470323562622,
      "logits/rejected": -3.9261558055877686,
      "logps/chosen": -92.28099060058594,
      "logps/rejected": -108.50651550292969,
      "loss": 0.4047850608825684,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.9677878618240356,
      "rewards/margins": 1.1882990598678589,
      "rewards/rejected": -2.1560869216918945,
      "step": 960
    },
    {
      "epoch": 1.0555102040816327,
      "grad_norm": 0.4538813829421997,
      "learning_rate": 0.0001337322363500374,
      "logits/chosen": -3.7241110801696777,
      "logits/rejected": -3.7324581146240234,
      "logps/chosen": -90.46250915527344,
      "logps/rejected": -109.62055969238281,
      "loss": 0.39860632419586184,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -0.9383746981620789,
      "rewards/margins": 1.2237045764923096,
      "rewards/rejected": -2.162079334259033,
      "step": 970
    },
    {
      "epoch": 1.0663945578231293,
      "grad_norm": 0.5154798030853271,
      "learning_rate": 0.00013298429319371727,
      "logits/chosen": -3.8078360557556152,
      "logits/rejected": -3.7505202293395996,
      "logps/chosen": -88.5647201538086,
      "logps/rejected": -105.87337493896484,
      "loss": 0.4086859703063965,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -1.0960875749588013,
      "rewards/margins": 1.1370435953140259,
      "rewards/rejected": -2.2331314086914062,
      "step": 980
    },
    {
      "epoch": 1.0772789115646257,
      "grad_norm": 0.5626422166824341,
      "learning_rate": 0.00013223635003739717,
      "logits/chosen": -3.832609176635742,
      "logits/rejected": -3.7331223487854004,
      "logps/chosen": -93.8613052368164,
      "logps/rejected": -112.10726165771484,
      "loss": 0.38766965866088865,
      "rewards/accuracies": 0.7945312261581421,
      "rewards/chosen": -1.118674397468567,
      "rewards/margins": 1.2444055080413818,
      "rewards/rejected": -2.36307954788208,
      "step": 990
    },
    {
      "epoch": 1.0881632653061224,
      "grad_norm": 0.4980772137641907,
      "learning_rate": 0.00013148840688107703,
      "logits/chosen": -3.9342827796936035,
      "logits/rejected": -3.891047239303589,
      "logps/chosen": -90.54890441894531,
      "logps/rejected": -110.37210845947266,
      "loss": 0.38568527698516847,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -1.2535765171051025,
      "rewards/margins": 1.2768959999084473,
      "rewards/rejected": -2.5304722785949707,
      "step": 1000
    },
    {
      "epoch": 1.0881632653061224,
      "eval_logits/chosen": -4.035802364349365,
      "eval_logits/rejected": -3.9840340614318848,
      "eval_logps/chosen": -94.60836791992188,
      "eval_logps/rejected": -109.67314910888672,
      "eval_loss": 0.5218217968940735,
      "eval_rewards/accuracies": 0.6712499856948853,
      "eval_rewards/chosen": -1.5003221035003662,
      "eval_rewards/margins": 0.8820583820343018,
      "eval_rewards/rejected": -2.382380723953247,
      "eval_runtime": 104.4812,
      "eval_samples_per_second": 22.971,
      "eval_steps_per_second": 2.871,
      "step": 1000
    },
    {
      "epoch": 1.099047619047619,
      "grad_norm": 0.5595287084579468,
      "learning_rate": 0.00013074046372475693,
      "logits/chosen": -3.9862780570983887,
      "logits/rejected": -3.910135269165039,
      "logps/chosen": -99.29423522949219,
      "logps/rejected": -118.37043762207031,
      "loss": 0.382628059387207,
      "rewards/accuracies": 0.7914062738418579,
      "rewards/chosen": -1.4844170808792114,
      "rewards/margins": 1.2958667278289795,
      "rewards/rejected": -2.7802836894989014,
      "step": 1010
    },
    {
      "epoch": 1.1099319727891157,
      "grad_norm": 0.5150955319404602,
      "learning_rate": 0.0001299925205684368,
      "logits/chosen": -3.9361672401428223,
      "logits/rejected": -3.924912929534912,
      "logps/chosen": -95.98111724853516,
      "logps/rejected": -114.6279525756836,
      "loss": 0.37689783573150637,
      "rewards/accuracies": 0.785937488079071,
      "rewards/chosen": -1.5440380573272705,
      "rewards/margins": 1.3521003723144531,
      "rewards/rejected": -2.8961384296417236,
      "step": 1020
    },
    {
      "epoch": 1.1208163265306124,
      "grad_norm": 0.5913591384887695,
      "learning_rate": 0.0001292445774121167,
      "logits/chosen": -3.943892002105713,
      "logits/rejected": -3.976665496826172,
      "logps/chosen": -95.18277740478516,
      "logps/rejected": -113.27921295166016,
      "loss": 0.39126296043395997,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -1.530815839767456,
      "rewards/margins": 1.3069045543670654,
      "rewards/rejected": -2.8377201557159424,
      "step": 1030
    },
    {
      "epoch": 1.1317006802721088,
      "grad_norm": 0.6089534163475037,
      "learning_rate": 0.00012849663425579656,
      "logits/chosen": -3.9231574535369873,
      "logits/rejected": -3.8793838024139404,
      "logps/chosen": -91.93929290771484,
      "logps/rejected": -109.3221664428711,
      "loss": 0.4064298629760742,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.3621362447738647,
      "rewards/margins": 1.2190048694610596,
      "rewards/rejected": -2.5811409950256348,
      "step": 1040
    },
    {
      "epoch": 1.1425850340136054,
      "grad_norm": 0.5580401420593262,
      "learning_rate": 0.00012774869109947646,
      "logits/chosen": -4.0801682472229,
      "logits/rejected": -4.014742851257324,
      "logps/chosen": -93.5271987915039,
      "logps/rejected": -110.57737731933594,
      "loss": 0.39088118076324463,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -1.1289399862289429,
      "rewards/margins": 1.294062852859497,
      "rewards/rejected": -2.4230027198791504,
      "step": 1050
    },
    {
      "epoch": 1.153469387755102,
      "grad_norm": 0.6687185764312744,
      "learning_rate": 0.00012700074794315633,
      "logits/chosen": -3.948697566986084,
      "logits/rejected": -3.8599441051483154,
      "logps/chosen": -92.7010269165039,
      "logps/rejected": -109.04603576660156,
      "loss": 0.4010897159576416,
      "rewards/accuracies": 0.76953125,
      "rewards/chosen": -1.2695072889328003,
      "rewards/margins": 1.2557579278945923,
      "rewards/rejected": -2.5252652168273926,
      "step": 1060
    },
    {
      "epoch": 1.1643537414965985,
      "grad_norm": 0.5772605538368225,
      "learning_rate": 0.00012625280478683622,
      "logits/chosen": -4.051478385925293,
      "logits/rejected": -3.9212899208068848,
      "logps/chosen": -93.09245300292969,
      "logps/rejected": -112.09034729003906,
      "loss": 0.3885282754898071,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -1.3699064254760742,
      "rewards/margins": 1.302438497543335,
      "rewards/rejected": -2.67234468460083,
      "step": 1070
    },
    {
      "epoch": 1.1752380952380952,
      "grad_norm": 0.55010986328125,
      "learning_rate": 0.0001255048616305161,
      "logits/chosen": -3.901043653488159,
      "logits/rejected": -3.7977395057678223,
      "logps/chosen": -93.72676086425781,
      "logps/rejected": -112.05916595458984,
      "loss": 0.39031736850738524,
      "rewards/accuracies": 0.7796875238418579,
      "rewards/chosen": -1.4006497859954834,
      "rewards/margins": 1.2986645698547363,
      "rewards/rejected": -2.6993141174316406,
      "step": 1080
    },
    {
      "epoch": 1.1861224489795918,
      "grad_norm": 0.6019951105117798,
      "learning_rate": 0.00012475691847419596,
      "logits/chosen": -3.874293804168701,
      "logits/rejected": -3.8425419330596924,
      "logps/chosen": -97.39969635009766,
      "logps/rejected": -116.4852523803711,
      "loss": 0.38108861446380615,
      "rewards/accuracies": 0.785937488079071,
      "rewards/chosen": -1.5423808097839355,
      "rewards/margins": 1.3624523878097534,
      "rewards/rejected": -2.9048333168029785,
      "step": 1090
    },
    {
      "epoch": 1.1970068027210885,
      "grad_norm": 0.5465772151947021,
      "learning_rate": 0.00012400897531787586,
      "logits/chosen": -3.8221359252929688,
      "logits/rejected": -3.7892353534698486,
      "logps/chosen": -97.63330841064453,
      "logps/rejected": -116.29536437988281,
      "loss": 0.3859058141708374,
      "rewards/accuracies": 0.7867187261581421,
      "rewards/chosen": -1.561934232711792,
      "rewards/margins": 1.3092113733291626,
      "rewards/rejected": -2.871145248413086,
      "step": 1100
    },
    {
      "epoch": 1.2078911564625852,
      "grad_norm": 0.6201313138008118,
      "learning_rate": 0.00012326103216155572,
      "logits/chosen": -3.8689208030700684,
      "logits/rejected": -3.814204454421997,
      "logps/chosen": -94.79856872558594,
      "logps/rejected": -114.0688705444336,
      "loss": 0.38736424446105955,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -1.537530541419983,
      "rewards/margins": 1.3583099842071533,
      "rewards/rejected": -2.895840644836426,
      "step": 1110
    },
    {
      "epoch": 1.2187755102040816,
      "grad_norm": 0.6293015480041504,
      "learning_rate": 0.00012251308900523562,
      "logits/chosen": -3.8243889808654785,
      "logits/rejected": -3.680662155151367,
      "logps/chosen": -98.95996856689453,
      "logps/rejected": -118.1916732788086,
      "loss": 0.3828889846801758,
      "rewards/accuracies": 0.782031238079071,
      "rewards/chosen": -1.7024767398834229,
      "rewards/margins": 1.3618117570877075,
      "rewards/rejected": -3.06428861618042,
      "step": 1120
    },
    {
      "epoch": 1.2296598639455782,
      "grad_norm": 0.6542093753814697,
      "learning_rate": 0.0001217651458489155,
      "logits/chosen": -3.806445360183716,
      "logits/rejected": -3.8042194843292236,
      "logps/chosen": -95.04216003417969,
      "logps/rejected": -113.69169616699219,
      "loss": 0.38822317123413086,
      "rewards/accuracies": 0.7710937261581421,
      "rewards/chosen": -1.6071590185165405,
      "rewards/margins": 1.3052054643630981,
      "rewards/rejected": -2.9123644828796387,
      "step": 1130
    },
    {
      "epoch": 1.240544217687075,
      "grad_norm": 0.6824989914894104,
      "learning_rate": 0.00012101720269259537,
      "logits/chosen": -3.831465482711792,
      "logits/rejected": -3.731153964996338,
      "logps/chosen": -97.02737426757812,
      "logps/rejected": -115.9981918334961,
      "loss": 0.39344158172607424,
      "rewards/accuracies": 0.7796875238418579,
      "rewards/chosen": -1.642228364944458,
      "rewards/margins": 1.3141920566558838,
      "rewards/rejected": -2.956420421600342,
      "step": 1140
    },
    {
      "epoch": 1.2514285714285713,
      "grad_norm": 0.5602055788040161,
      "learning_rate": 0.00012026925953627525,
      "logits/chosen": -3.8642430305480957,
      "logits/rejected": -3.8064823150634766,
      "logps/chosen": -96.62242126464844,
      "logps/rejected": -116.4171142578125,
      "loss": 0.38083562850952146,
      "rewards/accuracies": 0.789843738079071,
      "rewards/chosen": -1.6083332300186157,
      "rewards/margins": 1.389948844909668,
      "rewards/rejected": -2.998281717300415,
      "step": 1150
    },
    {
      "epoch": 1.262312925170068,
      "grad_norm": 0.5861300826072693,
      "learning_rate": 0.00011952131637995514,
      "logits/chosen": -3.833646297454834,
      "logits/rejected": -3.7338168621063232,
      "logps/chosen": -95.55216979980469,
      "logps/rejected": -114.51301574707031,
      "loss": 0.3848297595977783,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6490459442138672,
      "rewards/margins": 1.3717120885849,
      "rewards/rejected": -3.0207581520080566,
      "step": 1160
    },
    {
      "epoch": 1.2731972789115646,
      "grad_norm": 0.5589214563369751,
      "learning_rate": 0.00011877337322363502,
      "logits/chosen": -3.80780291557312,
      "logits/rejected": -3.658127546310425,
      "logps/chosen": -98.92552185058594,
      "logps/rejected": -120.0743408203125,
      "loss": 0.3743234872817993,
      "rewards/accuracies": 0.788281261920929,
      "rewards/chosen": -1.8888273239135742,
      "rewards/margins": 1.4256237745285034,
      "rewards/rejected": -3.314451217651367,
      "step": 1170
    },
    {
      "epoch": 1.2840816326530613,
      "grad_norm": 0.773446798324585,
      "learning_rate": 0.00011802543006731489,
      "logits/chosen": -3.791332244873047,
      "logits/rejected": -3.7509467601776123,
      "logps/chosen": -100.43743133544922,
      "logps/rejected": -119.4636001586914,
      "loss": 0.3916522979736328,
      "rewards/accuracies": 0.772656261920929,
      "rewards/chosen": -2.132657766342163,
      "rewards/margins": 1.3497222661972046,
      "rewards/rejected": -3.482379913330078,
      "step": 1180
    },
    {
      "epoch": 1.294965986394558,
      "grad_norm": 0.6507579684257507,
      "learning_rate": 0.00011727748691099475,
      "logits/chosen": -3.631664752960205,
      "logits/rejected": -3.559692859649658,
      "logps/chosen": -102.70137786865234,
      "logps/rejected": -119.88555908203125,
      "loss": 0.38732936382293703,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -1.9864223003387451,
      "rewards/margins": 1.3899860382080078,
      "rewards/rejected": -3.376408338546753,
      "step": 1190
    },
    {
      "epoch": 1.3058503401360544,
      "grad_norm": 0.5761600136756897,
      "learning_rate": 0.00011652954375467464,
      "logits/chosen": -3.7357120513916016,
      "logits/rejected": -3.7192249298095703,
      "logps/chosen": -96.95130157470703,
      "logps/rejected": -118.63916015625,
      "loss": 0.386475658416748,
      "rewards/accuracies": 0.7789062261581421,
      "rewards/chosen": -1.7698255777359009,
      "rewards/margins": 1.3770931959152222,
      "rewards/rejected": -3.146918773651123,
      "step": 1200
    },
    {
      "epoch": 1.316734693877551,
      "grad_norm": 0.751353919506073,
      "learning_rate": 0.00011578160059835452,
      "logits/chosen": -3.761479139328003,
      "logits/rejected": -3.714494228363037,
      "logps/chosen": -99.30330657958984,
      "logps/rejected": -118.73576354980469,
      "loss": 0.3787353515625,
      "rewards/accuracies": 0.789843738079071,
      "rewards/chosen": -1.8722871541976929,
      "rewards/margins": 1.4351270198822021,
      "rewards/rejected": -3.3074138164520264,
      "step": 1210
    },
    {
      "epoch": 1.3276190476190477,
      "grad_norm": 0.6535031795501709,
      "learning_rate": 0.0001150336574420344,
      "logits/chosen": -3.6952037811279297,
      "logits/rejected": -3.615670680999756,
      "logps/chosen": -100.49695587158203,
      "logps/rejected": -121.04229736328125,
      "loss": 0.3696281433105469,
      "rewards/accuracies": 0.7945312261581421,
      "rewards/chosen": -1.8464534282684326,
      "rewards/margins": 1.4216703176498413,
      "rewards/rejected": -3.2681241035461426,
      "step": 1220
    },
    {
      "epoch": 1.3385034013605441,
      "grad_norm": 0.660273551940918,
      "learning_rate": 0.00011428571428571428,
      "logits/chosen": -3.7871832847595215,
      "logits/rejected": -3.731839418411255,
      "logps/chosen": -98.5318374633789,
      "logps/rejected": -115.12672424316406,
      "loss": 0.3887288808822632,
      "rewards/accuracies": 0.7828124761581421,
      "rewards/chosen": -1.8460729122161865,
      "rewards/margins": 1.3180320262908936,
      "rewards/rejected": -3.164105176925659,
      "step": 1230
    },
    {
      "epoch": 1.3493877551020408,
      "grad_norm": 0.7036870121955872,
      "learning_rate": 0.00011353777112939417,
      "logits/chosen": -3.7227275371551514,
      "logits/rejected": -3.703014373779297,
      "logps/chosen": -96.31185913085938,
      "logps/rejected": -115.09098815917969,
      "loss": 0.3810297966003418,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6640751361846924,
      "rewards/margins": 1.3640429973602295,
      "rewards/rejected": -3.028118133544922,
      "step": 1240
    },
    {
      "epoch": 1.3602721088435374,
      "grad_norm": 0.6380351185798645,
      "learning_rate": 0.00011278982797307405,
      "logits/chosen": -3.743776798248291,
      "logits/rejected": -3.590733051300049,
      "logps/chosen": -95.80562591552734,
      "logps/rejected": -114.26639556884766,
      "loss": 0.38879597187042236,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -1.5717710256576538,
      "rewards/margins": 1.3705636262893677,
      "rewards/rejected": -2.9423346519470215,
      "step": 1250
    },
    {
      "epoch": 1.3711564625850339,
      "grad_norm": 0.695436418056488,
      "learning_rate": 0.00011204188481675393,
      "logits/chosen": -3.7618496417999268,
      "logits/rejected": -3.742903470993042,
      "logps/chosen": -94.9074478149414,
      "logps/rejected": -116.57875061035156,
      "loss": 0.3822426080703735,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -1.6461149454116821,
      "rewards/margins": 1.4086042642593384,
      "rewards/rejected": -3.0547192096710205,
      "step": 1260
    },
    {
      "epoch": 1.3820408163265305,
      "grad_norm": 0.5786954760551453,
      "learning_rate": 0.00011129394166043381,
      "logits/chosen": -3.881277561187744,
      "logits/rejected": -3.810837984085083,
      "logps/chosen": -98.8532485961914,
      "logps/rejected": -118.10380554199219,
      "loss": 0.3603384256362915,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.8184385299682617,
      "rewards/margins": 1.4062011241912842,
      "rewards/rejected": -3.224639415740967,
      "step": 1270
    },
    {
      "epoch": 1.3929251700680272,
      "grad_norm": 0.6679598093032837,
      "learning_rate": 0.0001105459985041137,
      "logits/chosen": -3.8195717334747314,
      "logits/rejected": -3.790808916091919,
      "logps/chosen": -100.193603515625,
      "logps/rejected": -120.28971862792969,
      "loss": 0.366161584854126,
      "rewards/accuracies": 0.803906261920929,
      "rewards/chosen": -1.7882047891616821,
      "rewards/margins": 1.4713773727416992,
      "rewards/rejected": -3.259582042694092,
      "step": 1280
    },
    {
      "epoch": 1.4038095238095238,
      "grad_norm": 0.6280708909034729,
      "learning_rate": 0.00010979805534779358,
      "logits/chosen": -3.7300808429718018,
      "logits/rejected": -3.7090256214141846,
      "logps/chosen": -95.59440612792969,
      "logps/rejected": -115.55784606933594,
      "loss": 0.38781962394714353,
      "rewards/accuracies": 0.770312488079071,
      "rewards/chosen": -1.5936332941055298,
      "rewards/margins": 1.418022871017456,
      "rewards/rejected": -3.0116562843322754,
      "step": 1290
    },
    {
      "epoch": 1.4146938775510205,
      "grad_norm": 0.6393949389457703,
      "learning_rate": 0.00010905011219147346,
      "logits/chosen": -3.6777634620666504,
      "logits/rejected": -3.60345721244812,
      "logps/chosen": -97.09107971191406,
      "logps/rejected": -115.95845031738281,
      "loss": 0.37133514881134033,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -1.5525085926055908,
      "rewards/margins": 1.4391489028930664,
      "rewards/rejected": -2.9916577339172363,
      "step": 1300
    },
    {
      "epoch": 1.425578231292517,
      "grad_norm": 0.5887162685394287,
      "learning_rate": 0.00010830216903515333,
      "logits/chosen": -3.858353853225708,
      "logits/rejected": -3.729213237762451,
      "logps/chosen": -97.08758544921875,
      "logps/rejected": -115.01194763183594,
      "loss": 0.3837103366851807,
      "rewards/accuracies": 0.77734375,
      "rewards/chosen": -1.7705045938491821,
      "rewards/margins": 1.3879071474075317,
      "rewards/rejected": -3.158411741256714,
      "step": 1310
    },
    {
      "epoch": 1.4364625850340136,
      "grad_norm": 0.6474682092666626,
      "learning_rate": 0.00010755422587883321,
      "logits/chosen": -3.9399871826171875,
      "logits/rejected": -3.7817294597625732,
      "logps/chosen": -97.05113220214844,
      "logps/rejected": -119.55680847167969,
      "loss": 0.36365351676940916,
      "rewards/accuracies": 0.788281261920929,
      "rewards/chosen": -1.7540229558944702,
      "rewards/margins": 1.5094449520111084,
      "rewards/rejected": -3.263467788696289,
      "step": 1320
    },
    {
      "epoch": 1.4473469387755102,
      "grad_norm": 0.6645624041557312,
      "learning_rate": 0.00010680628272251309,
      "logits/chosen": -3.977292537689209,
      "logits/rejected": -3.8931617736816406,
      "logps/chosen": -96.40562438964844,
      "logps/rejected": -117.29931640625,
      "loss": 0.37579851150512694,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -1.6829277276992798,
      "rewards/margins": 1.5337533950805664,
      "rewards/rejected": -3.2166812419891357,
      "step": 1330
    },
    {
      "epoch": 1.4582312925170067,
      "grad_norm": 0.795894980430603,
      "learning_rate": 0.00010605833956619297,
      "logits/chosen": -4.0191144943237305,
      "logits/rejected": -3.899752140045166,
      "logps/chosen": -92.3825454711914,
      "logps/rejected": -112.3360595703125,
      "loss": 0.3910695552825928,
      "rewards/accuracies": 0.7671874761581421,
      "rewards/chosen": -1.5036238431930542,
      "rewards/margins": 1.455990195274353,
      "rewards/rejected": -2.9596142768859863,
      "step": 1340
    },
    {
      "epoch": 1.4691156462585033,
      "grad_norm": 0.6805703639984131,
      "learning_rate": 0.00010531039640987286,
      "logits/chosen": -3.859619140625,
      "logits/rejected": -3.820143938064575,
      "logps/chosen": -101.65718841552734,
      "logps/rejected": -121.03543853759766,
      "loss": 0.38420472145080564,
      "rewards/accuracies": 0.7757812738418579,
      "rewards/chosen": -1.6729732751846313,
      "rewards/margins": 1.4384580850601196,
      "rewards/rejected": -3.111431121826172,
      "step": 1350
    },
    {
      "epoch": 1.48,
      "grad_norm": 0.6531292200088501,
      "learning_rate": 0.00010456245325355274,
      "logits/chosen": -3.826378583908081,
      "logits/rejected": -3.772704601287842,
      "logps/chosen": -100.1290283203125,
      "logps/rejected": -119.37181091308594,
      "loss": 0.3683862447738647,
      "rewards/accuracies": 0.78515625,
      "rewards/chosen": -1.9099397659301758,
      "rewards/margins": 1.4697914123535156,
      "rewards/rejected": -3.3797316551208496,
      "step": 1360
    },
    {
      "epoch": 1.4908843537414966,
      "grad_norm": 0.6555562019348145,
      "learning_rate": 0.00010381451009723262,
      "logits/chosen": -3.857220411300659,
      "logits/rejected": -3.754575252532959,
      "logps/chosen": -101.08953857421875,
      "logps/rejected": -120.2709732055664,
      "loss": 0.3801293849945068,
      "rewards/accuracies": 0.764843761920929,
      "rewards/chosen": -1.9699108600616455,
      "rewards/margins": 1.38763427734375,
      "rewards/rejected": -3.3575451374053955,
      "step": 1370
    },
    {
      "epoch": 1.5017687074829933,
      "grad_norm": 0.5829383134841919,
      "learning_rate": 0.0001030665669409125,
      "logits/chosen": -3.9333407878875732,
      "logits/rejected": -3.8222217559814453,
      "logps/chosen": -99.89224243164062,
      "logps/rejected": -118.7002182006836,
      "loss": 0.3699865102767944,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -1.854455590248108,
      "rewards/margins": 1.431877851486206,
      "rewards/rejected": -3.2863335609436035,
      "step": 1380
    },
    {
      "epoch": 1.5126530612244897,
      "grad_norm": 0.7183417677879333,
      "learning_rate": 0.00010231862378459238,
      "logits/chosen": -4.135873317718506,
      "logits/rejected": -3.9824626445770264,
      "logps/chosen": -97.94303131103516,
      "logps/rejected": -117.84385681152344,
      "loss": 0.373444128036499,
      "rewards/accuracies": 0.7828124761581421,
      "rewards/chosen": -1.8929665088653564,
      "rewards/margins": 1.4810130596160889,
      "rewards/rejected": -3.373979091644287,
      "step": 1390
    },
    {
      "epoch": 1.5235374149659864,
      "grad_norm": 0.62285315990448,
      "learning_rate": 0.00010157068062827227,
      "logits/chosen": -3.943370819091797,
      "logits/rejected": -3.885420322418213,
      "logps/chosen": -101.32877349853516,
      "logps/rejected": -122.38797760009766,
      "loss": 0.36237530708312987,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6976028680801392,
      "rewards/margins": 1.5130655765533447,
      "rewards/rejected": -3.2106685638427734,
      "step": 1400
    },
    {
      "epoch": 1.534421768707483,
      "grad_norm": 0.7117959856987,
      "learning_rate": 0.00010082273747195215,
      "logits/chosen": -4.146048545837402,
      "logits/rejected": -4.0635809898376465,
      "logps/chosen": -96.91676330566406,
      "logps/rejected": -115.88706970214844,
      "loss": 0.3611806631088257,
      "rewards/accuracies": 0.796093761920929,
      "rewards/chosen": -1.7968066930770874,
      "rewards/margins": 1.4590308666229248,
      "rewards/rejected": -3.2558376789093018,
      "step": 1410
    },
    {
      "epoch": 1.5453061224489795,
      "grad_norm": 0.4945203363895416,
      "learning_rate": 0.00010007479431563203,
      "logits/chosen": -4.0039167404174805,
      "logits/rejected": -3.9553489685058594,
      "logps/chosen": -100.09791564941406,
      "logps/rejected": -120.74732971191406,
      "loss": 0.37725100517272947,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -1.9454562664031982,
      "rewards/margins": 1.4613468647003174,
      "rewards/rejected": -3.406803607940674,
      "step": 1420
    },
    {
      "epoch": 1.5561904761904763,
      "grad_norm": 0.5278945565223694,
      "learning_rate": 9.93268511593119e-05,
      "logits/chosen": -3.986499309539795,
      "logits/rejected": -3.932950496673584,
      "logps/chosen": -102.49642181396484,
      "logps/rejected": -122.0504150390625,
      "loss": 0.37192845344543457,
      "rewards/accuracies": 0.7796875238418579,
      "rewards/chosen": -1.9946882724761963,
      "rewards/margins": 1.5072121620178223,
      "rewards/rejected": -3.5019004344940186,
      "step": 1430
    },
    {
      "epoch": 1.5670748299319728,
      "grad_norm": 0.5255491733551025,
      "learning_rate": 9.857890800299178e-05,
      "logits/chosen": -3.868673801422119,
      "logits/rejected": -3.8117432594299316,
      "logps/chosen": -100.22129821777344,
      "logps/rejected": -119.8470687866211,
      "loss": 0.374340295791626,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -1.920660376548767,
      "rewards/margins": 1.4606765508651733,
      "rewards/rejected": -3.3813369274139404,
      "step": 1440
    },
    {
      "epoch": 1.5779591836734694,
      "grad_norm": 0.6175586581230164,
      "learning_rate": 9.783096484667166e-05,
      "logits/chosen": -3.869438886642456,
      "logits/rejected": -3.7423954010009766,
      "logps/chosen": -101.46055603027344,
      "logps/rejected": -121.7767562866211,
      "loss": 0.3565861940383911,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.973406195640564,
      "rewards/margins": 1.5132112503051758,
      "rewards/rejected": -3.48661732673645,
      "step": 1450
    },
    {
      "epoch": 1.588843537414966,
      "grad_norm": 0.6701238751411438,
      "learning_rate": 9.708302169035153e-05,
      "logits/chosen": -3.9192538261413574,
      "logits/rejected": -3.899573564529419,
      "logps/chosen": -97.5078353881836,
      "logps/rejected": -118.8314437866211,
      "loss": 0.3684267997741699,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -2.0173895359039307,
      "rewards/margins": 1.507660150527954,
      "rewards/rejected": -3.525049924850464,
      "step": 1460
    },
    {
      "epoch": 1.5997278911564625,
      "grad_norm": 0.8422195315361023,
      "learning_rate": 9.633507853403142e-05,
      "logits/chosen": -3.8154900074005127,
      "logits/rejected": -3.763354539871216,
      "logps/chosen": -103.52520751953125,
      "logps/rejected": -125.2503890991211,
      "loss": 0.36898369789123536,
      "rewards/accuracies": 0.7835937738418579,
      "rewards/chosen": -1.952511191368103,
      "rewards/margins": 1.5541565418243408,
      "rewards/rejected": -3.5066676139831543,
      "step": 1470
    },
    {
      "epoch": 1.6106122448979592,
      "grad_norm": 0.6260126233100891,
      "learning_rate": 9.55871353777113e-05,
      "logits/chosen": -3.938748836517334,
      "logits/rejected": -3.8893561363220215,
      "logps/chosen": -98.54754638671875,
      "logps/rejected": -118.55876159667969,
      "loss": 0.378691291809082,
      "rewards/accuracies": 0.788281261920929,
      "rewards/chosen": -1.7820552587509155,
      "rewards/margins": 1.4857076406478882,
      "rewards/rejected": -3.2677626609802246,
      "step": 1480
    },
    {
      "epoch": 1.6214965986394558,
      "grad_norm": 0.6023070216178894,
      "learning_rate": 9.483919222139118e-05,
      "logits/chosen": -4.026276111602783,
      "logits/rejected": -3.953202724456787,
      "logps/chosen": -96.39505004882812,
      "logps/rejected": -115.8733901977539,
      "loss": 0.36678736209869384,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.5815566778182983,
      "rewards/margins": 1.527267336845398,
      "rewards/rejected": -3.1088242530822754,
      "step": 1490
    },
    {
      "epoch": 1.6323809523809523,
      "grad_norm": 0.5291011929512024,
      "learning_rate": 9.409124906507106e-05,
      "logits/chosen": -3.9902701377868652,
      "logits/rejected": -3.968461513519287,
      "logps/chosen": -96.85718536376953,
      "logps/rejected": -118.74107360839844,
      "loss": 0.3673092365264893,
      "rewards/accuracies": 0.789843738079071,
      "rewards/chosen": -1.714505910873413,
      "rewards/margins": 1.5418702363967896,
      "rewards/rejected": -3.256376266479492,
      "step": 1500
    },
    {
      "epoch": 1.6323809523809523,
      "eval_logits/chosen": -4.040825366973877,
      "eval_logits/rejected": -3.988185167312622,
      "eval_logps/chosen": -98.07389068603516,
      "eval_logps/rejected": -115.78838348388672,
      "eval_loss": 0.47849762439727783,
      "eval_rewards/accuracies": 0.7079166769981384,
      "eval_rewards/chosen": -1.8468737602233887,
      "eval_rewards/margins": 1.1470292806625366,
      "eval_rewards/rejected": -2.993903160095215,
      "eval_runtime": 106.0205,
      "eval_samples_per_second": 22.637,
      "eval_steps_per_second": 2.83,
      "step": 1500
    },
    {
      "epoch": 1.6432653061224491,
      "grad_norm": 0.6328744292259216,
      "learning_rate": 9.334330590875094e-05,
      "logits/chosen": -4.012321949005127,
      "logits/rejected": -3.944819927215576,
      "logps/chosen": -97.4483642578125,
      "logps/rejected": -118.50297546386719,
      "loss": 0.3724888801574707,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.79526686668396,
      "rewards/margins": 1.4999581575393677,
      "rewards/rejected": -3.2952256202697754,
      "step": 1510
    },
    {
      "epoch": 1.6541496598639456,
      "grad_norm": 0.8266810178756714,
      "learning_rate": 9.259536275243081e-05,
      "logits/chosen": -4.003256797790527,
      "logits/rejected": -3.9533722400665283,
      "logps/chosen": -99.8465347290039,
      "logps/rejected": -118.92730712890625,
      "loss": 0.3884738922119141,
      "rewards/accuracies": 0.776562511920929,
      "rewards/chosen": -2.056798219680786,
      "rewards/margins": 1.452993392944336,
      "rewards/rejected": -3.509791612625122,
      "step": 1520
    },
    {
      "epoch": 1.6650340136054422,
      "grad_norm": 0.6178652048110962,
      "learning_rate": 9.18474195961107e-05,
      "logits/chosen": -3.8090343475341797,
      "logits/rejected": -3.734511613845825,
      "logps/chosen": -100.30364990234375,
      "logps/rejected": -121.00141906738281,
      "loss": 0.36191649436950685,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.9928449392318726,
      "rewards/margins": 1.523863673210144,
      "rewards/rejected": -3.5167088508605957,
      "step": 1530
    },
    {
      "epoch": 1.6759183673469389,
      "grad_norm": 0.6442583799362183,
      "learning_rate": 9.109947643979058e-05,
      "logits/chosen": -3.819338321685791,
      "logits/rejected": -3.7492592334747314,
      "logps/chosen": -103.84706115722656,
      "logps/rejected": -123.3454360961914,
      "loss": 0.3575705289840698,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.11049222946167,
      "rewards/margins": 1.5337401628494263,
      "rewards/rejected": -3.6442322731018066,
      "step": 1540
    },
    {
      "epoch": 1.6868027210884353,
      "grad_norm": 0.7383544445037842,
      "learning_rate": 9.035153328347046e-05,
      "logits/chosen": -3.8853554725646973,
      "logits/rejected": -3.7873711585998535,
      "logps/chosen": -103.81562805175781,
      "logps/rejected": -122.89927673339844,
      "loss": 0.37395687103271485,
      "rewards/accuracies": 0.7679687738418579,
      "rewards/chosen": -2.210036516189575,
      "rewards/margins": 1.5190045833587646,
      "rewards/rejected": -3.729041337966919,
      "step": 1550
    },
    {
      "epoch": 1.697687074829932,
      "grad_norm": 0.5653782486915588,
      "learning_rate": 8.960359012715034e-05,
      "logits/chosen": -3.935488224029541,
      "logits/rejected": -3.8435440063476562,
      "logps/chosen": -100.53340148925781,
      "logps/rejected": -120.791259765625,
      "loss": 0.3584398031234741,
      "rewards/accuracies": 0.78515625,
      "rewards/chosen": -2.0043153762817383,
      "rewards/margins": 1.5743240118026733,
      "rewards/rejected": -3.578639268875122,
      "step": 1560
    },
    {
      "epoch": 1.7085714285714286,
      "grad_norm": 0.8251403570175171,
      "learning_rate": 8.885564697083022e-05,
      "logits/chosen": -4.113719940185547,
      "logits/rejected": -4.015976905822754,
      "logps/chosen": -102.41825866699219,
      "logps/rejected": -124.1919937133789,
      "loss": 0.35360021591186525,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.1747488975524902,
      "rewards/margins": 1.5947285890579224,
      "rewards/rejected": -3.7694778442382812,
      "step": 1570
    },
    {
      "epoch": 1.719455782312925,
      "grad_norm": 0.7069426774978638,
      "learning_rate": 8.81077038145101e-05,
      "logits/chosen": -4.1289448738098145,
      "logits/rejected": -4.012777805328369,
      "logps/chosen": -100.58708953857422,
      "logps/rejected": -121.4145736694336,
      "loss": 0.3635096549987793,
      "rewards/accuracies": 0.7835937738418579,
      "rewards/chosen": -2.133143424987793,
      "rewards/margins": 1.5998557806015015,
      "rewards/rejected": -3.732999324798584,
      "step": 1580
    },
    {
      "epoch": 1.7303401360544217,
      "grad_norm": 0.8968989253044128,
      "learning_rate": 8.735976065818999e-05,
      "logits/chosen": -4.026947021484375,
      "logits/rejected": -4.019469738006592,
      "logps/chosen": -101.74604797363281,
      "logps/rejected": -124.91915130615234,
      "loss": 0.37527787685394287,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -2.15088152885437,
      "rewards/margins": 1.591313123703003,
      "rewards/rejected": -3.742194652557373,
      "step": 1590
    },
    {
      "epoch": 1.7412244897959184,
      "grad_norm": 0.7351986169815063,
      "learning_rate": 8.661181750186986e-05,
      "logits/chosen": -3.981262683868408,
      "logits/rejected": -3.917276382446289,
      "logps/chosen": -103.64302825927734,
      "logps/rejected": -124.09883880615234,
      "loss": 0.37657463550567627,
      "rewards/accuracies": 0.774218738079071,
      "rewards/chosen": -2.378594160079956,
      "rewards/margins": 1.506073236465454,
      "rewards/rejected": -3.8846676349639893,
      "step": 1600
    },
    {
      "epoch": 1.7521088435374148,
      "grad_norm": 0.5647110342979431,
      "learning_rate": 8.586387434554974e-05,
      "logits/chosen": -4.083320140838623,
      "logits/rejected": -4.027539253234863,
      "logps/chosen": -103.57328796386719,
      "logps/rejected": -125.90059661865234,
      "loss": 0.36530237197875975,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -2.352818250656128,
      "rewards/margins": 1.5950738191604614,
      "rewards/rejected": -3.9478919506073,
      "step": 1610
    },
    {
      "epoch": 1.7629931972789117,
      "grad_norm": 0.7188398838043213,
      "learning_rate": 8.511593118922962e-05,
      "logits/chosen": -3.8719964027404785,
      "logits/rejected": -3.833092212677002,
      "logps/chosen": -104.86869812011719,
      "logps/rejected": -124.95475006103516,
      "loss": 0.36907384395599363,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -2.335792064666748,
      "rewards/margins": 1.5145189762115479,
      "rewards/rejected": -3.850311279296875,
      "step": 1620
    },
    {
      "epoch": 1.7738775510204081,
      "grad_norm": 0.6679319739341736,
      "learning_rate": 8.43679880329095e-05,
      "logits/chosen": -4.030211448669434,
      "logits/rejected": -3.9728660583496094,
      "logps/chosen": -103.6771011352539,
      "logps/rejected": -124.9799575805664,
      "loss": 0.3572399377822876,
      "rewards/accuracies": 0.7945312261581421,
      "rewards/chosen": -2.2580087184906006,
      "rewards/margins": 1.571171522140503,
      "rewards/rejected": -3.829180955886841,
      "step": 1630
    },
    {
      "epoch": 1.7847619047619048,
      "grad_norm": 0.6013081073760986,
      "learning_rate": 8.362004487658939e-05,
      "logits/chosen": -3.995105028152466,
      "logits/rejected": -3.910709857940674,
      "logps/chosen": -100.53841400146484,
      "logps/rejected": -123.62651062011719,
      "loss": 0.3458747625350952,
      "rewards/accuracies": 0.8023437261581421,
      "rewards/chosen": -2.2358031272888184,
      "rewards/margins": 1.6609306335449219,
      "rewards/rejected": -3.8967342376708984,
      "step": 1640
    },
    {
      "epoch": 1.7956462585034014,
      "grad_norm": 0.6696369051933289,
      "learning_rate": 8.287210172026927e-05,
      "logits/chosen": -3.9745001792907715,
      "logits/rejected": -3.900158405303955,
      "logps/chosen": -100.8124771118164,
      "logps/rejected": -123.26814270019531,
      "loss": 0.3555989027023315,
      "rewards/accuracies": 0.79296875,
      "rewards/chosen": -2.186405658721924,
      "rewards/margins": 1.6339943408966064,
      "rewards/rejected": -3.8204002380371094,
      "step": 1650
    },
    {
      "epoch": 1.8065306122448979,
      "grad_norm": 0.5695591568946838,
      "learning_rate": 8.212415856394914e-05,
      "logits/chosen": -4.0766448974609375,
      "logits/rejected": -3.9804439544677734,
      "logps/chosen": -99.67530822753906,
      "logps/rejected": -120.61580657958984,
      "loss": 0.3668109893798828,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -2.070260524749756,
      "rewards/margins": 1.568867802619934,
      "rewards/rejected": -3.6391282081604004,
      "step": 1660
    },
    {
      "epoch": 1.8174149659863945,
      "grad_norm": 0.6718661189079285,
      "learning_rate": 8.137621540762902e-05,
      "logits/chosen": -4.0458664894104,
      "logits/rejected": -3.9342434406280518,
      "logps/chosen": -103.21232604980469,
      "logps/rejected": -124.14998626708984,
      "loss": 0.3770064115524292,
      "rewards/accuracies": 0.789843738079071,
      "rewards/chosen": -2.1097495555877686,
      "rewards/margins": 1.5439164638519287,
      "rewards/rejected": -3.6536660194396973,
      "step": 1670
    },
    {
      "epoch": 1.8282993197278912,
      "grad_norm": 0.6736993193626404,
      "learning_rate": 8.06282722513089e-05,
      "logits/chosen": -4.07711124420166,
      "logits/rejected": -4.0070013999938965,
      "logps/chosen": -99.71659088134766,
      "logps/rejected": -121.86555480957031,
      "loss": 0.3560856580734253,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -2.129600763320923,
      "rewards/margins": 1.6453748941421509,
      "rewards/rejected": -3.774975538253784,
      "step": 1680
    },
    {
      "epoch": 1.8391836734693876,
      "grad_norm": 0.7004987597465515,
      "learning_rate": 7.988032909498878e-05,
      "logits/chosen": -3.9158096313476562,
      "logits/rejected": -3.911224365234375,
      "logps/chosen": -105.80472564697266,
      "logps/rejected": -128.0396728515625,
      "loss": 0.35362706184387205,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -2.293348789215088,
      "rewards/margins": 1.6550325155258179,
      "rewards/rejected": -3.9483814239501953,
      "step": 1690
    },
    {
      "epoch": 1.8500680272108845,
      "grad_norm": 0.5832870006561279,
      "learning_rate": 7.913238593866867e-05,
      "logits/chosen": -4.208107948303223,
      "logits/rejected": -4.131274223327637,
      "logps/chosen": -98.68458557128906,
      "logps/rejected": -118.3831558227539,
      "loss": 0.3670835256576538,
      "rewards/accuracies": 0.760937511920929,
      "rewards/chosen": -2.237819194793701,
      "rewards/margins": 1.6039073467254639,
      "rewards/rejected": -3.841726779937744,
      "step": 1700
    },
    {
      "epoch": 1.860952380952381,
      "grad_norm": 0.5559285879135132,
      "learning_rate": 7.838444278234855e-05,
      "logits/chosen": -4.1811747550964355,
      "logits/rejected": -4.1075592041015625,
      "logps/chosen": -99.44391632080078,
      "logps/rejected": -122.75013732910156,
      "loss": 0.3504934787750244,
      "rewards/accuracies": 0.809374988079071,
      "rewards/chosen": -2.2351126670837402,
      "rewards/margins": 1.6141564846038818,
      "rewards/rejected": -3.849269151687622,
      "step": 1710
    },
    {
      "epoch": 1.8718367346938776,
      "grad_norm": 0.5495217442512512,
      "learning_rate": 7.763649962602843e-05,
      "logits/chosen": -4.15755558013916,
      "logits/rejected": -4.096221923828125,
      "logps/chosen": -104.17137145996094,
      "logps/rejected": -125.32877349853516,
      "loss": 0.3607916355133057,
      "rewards/accuracies": 0.7828124761581421,
      "rewards/chosen": -2.3636012077331543,
      "rewards/margins": 1.6084495782852173,
      "rewards/rejected": -3.972050428390503,
      "step": 1720
    },
    {
      "epoch": 1.8827210884353742,
      "grad_norm": 0.7562318444252014,
      "learning_rate": 7.688855646970831e-05,
      "logits/chosen": -4.053821563720703,
      "logits/rejected": -3.9341418743133545,
      "logps/chosen": -102.71726989746094,
      "logps/rejected": -123.45503234863281,
      "loss": 0.3550391674041748,
      "rewards/accuracies": 0.7906249761581421,
      "rewards/chosen": -2.264005422592163,
      "rewards/margins": 1.6550270318984985,
      "rewards/rejected": -3.919032573699951,
      "step": 1730
    },
    {
      "epoch": 1.8936054421768707,
      "grad_norm": 0.6610931158065796,
      "learning_rate": 7.61406133133882e-05,
      "logits/chosen": -3.961141586303711,
      "logits/rejected": -3.830481767654419,
      "logps/chosen": -104.4319839477539,
      "logps/rejected": -125.9029541015625,
      "loss": 0.3494965314865112,
      "rewards/accuracies": 0.7953125238418579,
      "rewards/chosen": -2.1706981658935547,
      "rewards/margins": 1.6067132949829102,
      "rewards/rejected": -3.777411699295044,
      "step": 1740
    },
    {
      "epoch": 1.9044897959183673,
      "grad_norm": 0.6473342776298523,
      "learning_rate": 7.539267015706806e-05,
      "logits/chosen": -3.913191556930542,
      "logits/rejected": -3.8687243461608887,
      "logps/chosen": -105.96980285644531,
      "logps/rejected": -125.6346206665039,
      "loss": 0.3611732482910156,
      "rewards/accuracies": 0.7835937738418579,
      "rewards/chosen": -2.0681533813476562,
      "rewards/margins": 1.6147505044937134,
      "rewards/rejected": -3.68290376663208,
      "step": 1750
    },
    {
      "epoch": 1.915374149659864,
      "grad_norm": 0.7833436727523804,
      "learning_rate": 7.464472700074794e-05,
      "logits/chosen": -3.938861846923828,
      "logits/rejected": -3.9061336517333984,
      "logps/chosen": -97.59529876708984,
      "logps/rejected": -119.07645416259766,
      "loss": 0.36034407615661623,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -1.8530477285385132,
      "rewards/margins": 1.5701138973236084,
      "rewards/rejected": -3.423161745071411,
      "step": 1760
    },
    {
      "epoch": 1.9262585034013604,
      "grad_norm": 0.7200015783309937,
      "learning_rate": 7.389678384442783e-05,
      "logits/chosen": -3.8537850379943848,
      "logits/rejected": -3.8197879791259766,
      "logps/chosen": -104.39387512207031,
      "logps/rejected": -122.6333236694336,
      "loss": 0.3665923118591309,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.3217196464538574,
      "rewards/margins": 1.548337697982788,
      "rewards/rejected": -3.8700573444366455,
      "step": 1770
    },
    {
      "epoch": 1.9371428571428573,
      "grad_norm": 0.6586691737174988,
      "learning_rate": 7.314884068810771e-05,
      "logits/chosen": -3.951854705810547,
      "logits/rejected": -4.002626419067383,
      "logps/chosen": -106.01944732666016,
      "logps/rejected": -127.8281021118164,
      "loss": 0.36367824077606203,
      "rewards/accuracies": 0.780468761920929,
      "rewards/chosen": -2.4460396766662598,
      "rewards/margins": 1.604691743850708,
      "rewards/rejected": -4.050731658935547,
      "step": 1780
    },
    {
      "epoch": 1.9480272108843537,
      "grad_norm": 0.5734717845916748,
      "learning_rate": 7.240089753178759e-05,
      "logits/chosen": -3.999734878540039,
      "logits/rejected": -3.9233245849609375,
      "logps/chosen": -100.71134948730469,
      "logps/rejected": -121.8692626953125,
      "loss": 0.3612796783447266,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.5755791664123535,
      "rewards/margins": 1.5662591457366943,
      "rewards/rejected": -4.141838550567627,
      "step": 1790
    },
    {
      "epoch": 1.9589115646258504,
      "grad_norm": 0.6537356376647949,
      "learning_rate": 7.165295437546746e-05,
      "logits/chosen": -3.9834160804748535,
      "logits/rejected": -3.872994899749756,
      "logps/chosen": -103.37413024902344,
      "logps/rejected": -125.7213134765625,
      "loss": 0.3468764781951904,
      "rewards/accuracies": 0.819531261920929,
      "rewards/chosen": -2.4130101203918457,
      "rewards/margins": 1.6939111948013306,
      "rewards/rejected": -4.106921195983887,
      "step": 1800
    },
    {
      "epoch": 1.969795918367347,
      "grad_norm": 0.6989219784736633,
      "learning_rate": 7.090501121914734e-05,
      "logits/chosen": -3.91550874710083,
      "logits/rejected": -3.8702545166015625,
      "logps/chosen": -106.2231674194336,
      "logps/rejected": -126.7488784790039,
      "loss": 0.36261188983917236,
      "rewards/accuracies": 0.7945312261581421,
      "rewards/chosen": -2.3560307025909424,
      "rewards/margins": 1.6367266178131104,
      "rewards/rejected": -3.9927573204040527,
      "step": 1810
    },
    {
      "epoch": 1.9806802721088435,
      "grad_norm": 0.7157489657402039,
      "learning_rate": 7.015706806282722e-05,
      "logits/chosen": -3.9314968585968018,
      "logits/rejected": -3.872746229171753,
      "logps/chosen": -100.2810287475586,
      "logps/rejected": -121.5164794921875,
      "loss": 0.35974826812744143,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -2.0922586917877197,
      "rewards/margins": 1.629852533340454,
      "rewards/rejected": -3.722111225128174,
      "step": 1820
    },
    {
      "epoch": 1.99156462585034,
      "grad_norm": 0.6526369452476501,
      "learning_rate": 6.94091249065071e-05,
      "logits/chosen": -3.94775652885437,
      "logits/rejected": -3.8687806129455566,
      "logps/chosen": -99.9976577758789,
      "logps/rejected": -121.957763671875,
      "loss": 0.3599375247955322,
      "rewards/accuracies": 0.7945312261581421,
      "rewards/chosen": -1.9912713766098022,
      "rewards/margins": 1.6352510452270508,
      "rewards/rejected": -3.6265225410461426,
      "step": 1830
    },
    {
      "epoch": 2.002176870748299,
      "grad_norm": 0.49071577191352844,
      "learning_rate": 6.866118175018699e-05,
      "logits/chosen": -3.9457576274871826,
      "logits/rejected": -3.855695962905884,
      "logps/chosen": -103.30020904541016,
      "logps/rejected": -127.69253540039062,
      "loss": 0.32846436500549314,
      "rewards/accuracies": 0.8221153616905212,
      "rewards/chosen": -2.0800564289093018,
      "rewards/margins": 1.746621012687683,
      "rewards/rejected": -3.8266773223876953,
      "step": 1840
    },
    {
      "epoch": 2.013061224489796,
      "grad_norm": 0.5088359713554382,
      "learning_rate": 6.791323859386687e-05,
      "logits/chosen": -3.9414725303649902,
      "logits/rejected": -3.9251086711883545,
      "logps/chosen": -101.10342407226562,
      "logps/rejected": -126.2572250366211,
      "loss": 0.279949951171875,
      "rewards/accuracies": 0.839062511920929,
      "rewards/chosen": -2.04860258102417,
      "rewards/margins": 1.9327900409698486,
      "rewards/rejected": -3.9813923835754395,
      "step": 1850
    },
    {
      "epoch": 2.0239455782312925,
      "grad_norm": 0.5052030682563782,
      "learning_rate": 6.716529543754675e-05,
      "logits/chosen": -4.173396110534668,
      "logits/rejected": -4.081063270568848,
      "logps/chosen": -104.18453216552734,
      "logps/rejected": -128.5143280029297,
      "loss": 0.26949272155761717,
      "rewards/accuracies": 0.858593761920929,
      "rewards/chosen": -2.204545259475708,
      "rewards/margins": 1.9530636072158813,
      "rewards/rejected": -4.157608985900879,
      "step": 1860
    },
    {
      "epoch": 2.034829931972789,
      "grad_norm": 0.4783055782318115,
      "learning_rate": 6.641735228122664e-05,
      "logits/chosen": -4.18683385848999,
      "logits/rejected": -4.042752265930176,
      "logps/chosen": -104.73225402832031,
      "logps/rejected": -129.3142852783203,
      "loss": 0.2710374116897583,
      "rewards/accuracies": 0.85546875,
      "rewards/chosen": -2.1699516773223877,
      "rewards/margins": 2.064490795135498,
      "rewards/rejected": -4.234442234039307,
      "step": 1870
    },
    {
      "epoch": 2.045714285714286,
      "grad_norm": 0.5429951548576355,
      "learning_rate": 6.566940912490652e-05,
      "logits/chosen": -4.225846290588379,
      "logits/rejected": -4.173259735107422,
      "logps/chosen": -102.9245834350586,
      "logps/rejected": -129.71678161621094,
      "loss": 0.2762880563735962,
      "rewards/accuracies": 0.844531238079071,
      "rewards/chosen": -2.279546022415161,
      "rewards/margins": 2.050327777862549,
      "rewards/rejected": -4.329874038696289,
      "step": 1880
    },
    {
      "epoch": 2.0565986394557823,
      "grad_norm": 0.5694318413734436,
      "learning_rate": 6.492146596858639e-05,
      "logits/chosen": -4.403803825378418,
      "logits/rejected": -4.27715539932251,
      "logps/chosen": -103.4933853149414,
      "logps/rejected": -131.231201171875,
      "loss": 0.2582036733627319,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.3441028594970703,
      "rewards/margins": 2.2010316848754883,
      "rewards/rejected": -4.545134544372559,
      "step": 1890
    },
    {
      "epoch": 2.067482993197279,
      "grad_norm": 0.488471657037735,
      "learning_rate": 6.417352281226627e-05,
      "logits/chosen": -4.374741554260254,
      "logits/rejected": -4.235744476318359,
      "logps/chosen": -108.23167419433594,
      "logps/rejected": -136.00350952148438,
      "loss": 0.2654814004898071,
      "rewards/accuracies": 0.844531238079071,
      "rewards/chosen": -2.7162108421325684,
      "rewards/margins": 2.194584846496582,
      "rewards/rejected": -4.910796165466309,
      "step": 1900
    },
    {
      "epoch": 2.0783673469387756,
      "grad_norm": 0.5405411720275879,
      "learning_rate": 6.342557965594615e-05,
      "logits/chosen": -4.263213157653809,
      "logits/rejected": -4.2035136222839355,
      "logps/chosen": -108.0687255859375,
      "logps/rejected": -135.1525115966797,
      "loss": 0.269286060333252,
      "rewards/accuracies": 0.8460937738418579,
      "rewards/chosen": -2.618056297302246,
      "rewards/margins": 2.1149375438690186,
      "rewards/rejected": -4.732994079589844,
      "step": 1910
    },
    {
      "epoch": 2.089251700680272,
      "grad_norm": 0.5153743624687195,
      "learning_rate": 6.267763649962603e-05,
      "logits/chosen": -4.374856472015381,
      "logits/rejected": -4.3279643058776855,
      "logps/chosen": -102.6282958984375,
      "logps/rejected": -129.84768676757812,
      "loss": 0.27457327842712403,
      "rewards/accuracies": 0.83984375,
      "rewards/chosen": -2.5648117065429688,
      "rewards/margins": 2.1569466590881348,
      "rewards/rejected": -4.7217583656311035,
      "step": 1920
    },
    {
      "epoch": 2.100136054421769,
      "grad_norm": 0.5591779947280884,
      "learning_rate": 6.192969334330591e-05,
      "logits/chosen": -4.222095489501953,
      "logits/rejected": -4.156058311462402,
      "logps/chosen": -107.1645736694336,
      "logps/rejected": -131.6280059814453,
      "loss": 0.2775254726409912,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -2.541388988494873,
      "rewards/margins": 2.1209020614624023,
      "rewards/rejected": -4.662291526794434,
      "step": 1930
    },
    {
      "epoch": 2.1110204081632653,
      "grad_norm": 0.47012007236480713,
      "learning_rate": 6.118175018698578e-05,
      "logits/chosen": -4.261131763458252,
      "logits/rejected": -4.192044258117676,
      "logps/chosen": -109.32353210449219,
      "logps/rejected": -134.66297912597656,
      "loss": 0.26659140586853025,
      "rewards/accuracies": 0.8531249761581421,
      "rewards/chosen": -2.73209810256958,
      "rewards/margins": 2.093428134918213,
      "rewards/rejected": -4.825526237487793,
      "step": 1940
    },
    {
      "epoch": 2.1219047619047617,
      "grad_norm": 0.5636943578720093,
      "learning_rate": 6.0433807030665666e-05,
      "logits/chosen": -4.38666296005249,
      "logits/rejected": -4.33133602142334,
      "logps/chosen": -107.5749282836914,
      "logps/rejected": -132.30654907226562,
      "loss": 0.27169456481933596,
      "rewards/accuracies": 0.839062511920929,
      "rewards/chosen": -2.6706109046936035,
      "rewards/margins": 2.161822557449341,
      "rewards/rejected": -4.832433223724365,
      "step": 1950
    },
    {
      "epoch": 2.1327891156462586,
      "grad_norm": 0.5739550590515137,
      "learning_rate": 5.968586387434555e-05,
      "logits/chosen": -4.306714057922363,
      "logits/rejected": -4.2323102951049805,
      "logps/chosen": -110.61419677734375,
      "logps/rejected": -136.73745727539062,
      "loss": 0.2676869869232178,
      "rewards/accuracies": 0.85546875,
      "rewards/chosen": -2.7505619525909424,
      "rewards/margins": 2.1400463581085205,
      "rewards/rejected": -4.890608310699463,
      "step": 1960
    },
    {
      "epoch": 2.143673469387755,
      "grad_norm": 0.5009965896606445,
      "learning_rate": 5.893792071802543e-05,
      "logits/chosen": -4.33192253112793,
      "logits/rejected": -4.311135768890381,
      "logps/chosen": -108.64591979980469,
      "logps/rejected": -132.796142578125,
      "loss": 0.2648327112197876,
      "rewards/accuracies": 0.8492187261581421,
      "rewards/chosen": -2.6455886363983154,
      "rewards/margins": 2.131373882293701,
      "rewards/rejected": -4.776961803436279,
      "step": 1970
    },
    {
      "epoch": 2.1545578231292515,
      "grad_norm": 0.6320508718490601,
      "learning_rate": 5.818997756170531e-05,
      "logits/chosen": -4.376672267913818,
      "logits/rejected": -4.221155643463135,
      "logps/chosen": -107.09896087646484,
      "logps/rejected": -135.03883361816406,
      "loss": 0.2541360855102539,
      "rewards/accuracies": 0.8570312261581421,
      "rewards/chosen": -2.8258004188537598,
      "rewards/margins": 2.2211556434631348,
      "rewards/rejected": -5.046955585479736,
      "step": 1980
    },
    {
      "epoch": 2.1654421768707484,
      "grad_norm": 0.5602008700370789,
      "learning_rate": 5.7442034405385194e-05,
      "logits/chosen": -4.365457057952881,
      "logits/rejected": -4.279487609863281,
      "logps/chosen": -110.8012924194336,
      "logps/rejected": -138.8103790283203,
      "loss": 0.26124260425567625,
      "rewards/accuracies": 0.8609374761581421,
      "rewards/chosen": -3.039825439453125,
      "rewards/margins": 2.225407838821411,
      "rewards/rejected": -5.265233039855957,
      "step": 1990
    },
    {
      "epoch": 2.176326530612245,
      "grad_norm": 0.5244850516319275,
      "learning_rate": 5.6694091249065076e-05,
      "logits/chosen": -4.355788707733154,
      "logits/rejected": -4.297573566436768,
      "logps/chosen": -113.49159240722656,
      "logps/rejected": -140.8461456298828,
      "loss": 0.2652238130569458,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -3.1340157985687256,
      "rewards/margins": 2.230769634246826,
      "rewards/rejected": -5.364785194396973,
      "step": 2000
    },
    {
      "epoch": 2.176326530612245,
      "eval_logits/chosen": -4.392631530761719,
      "eval_logits/rejected": -4.344852924346924,
      "eval_logps/chosen": -113.7379150390625,
      "eval_logps/rejected": -135.5451202392578,
      "eval_loss": 0.46534663438796997,
      "eval_rewards/accuracies": 0.7124999761581421,
      "eval_rewards/chosen": -3.413275957107544,
      "eval_rewards/margins": 1.556301474571228,
      "eval_rewards/rejected": -4.969577789306641,
      "eval_runtime": 104.78,
      "eval_samples_per_second": 22.905,
      "eval_steps_per_second": 2.863,
      "step": 2000
    },
    {
      "epoch": 2.1872108843537417,
      "grad_norm": 0.6399655342102051,
      "learning_rate": 5.594614809274495e-05,
      "logits/chosen": -4.25847053527832,
      "logits/rejected": -4.194684028625488,
      "logps/chosen": -115.2059555053711,
      "logps/rejected": -140.93521118164062,
      "loss": 0.2672285079956055,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.0837132930755615,
      "rewards/margins": 2.1926894187927246,
      "rewards/rejected": -5.276402950286865,
      "step": 2010
    },
    {
      "epoch": 2.198095238095238,
      "grad_norm": 0.5022415518760681,
      "learning_rate": 5.5198204936424834e-05,
      "logits/chosen": -4.377929210662842,
      "logits/rejected": -4.255377769470215,
      "logps/chosen": -108.50428771972656,
      "logps/rejected": -137.51651000976562,
      "loss": 0.2487508773803711,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.958159923553467,
      "rewards/margins": 2.2823288440704346,
      "rewards/rejected": -5.240488529205322,
      "step": 2020
    },
    {
      "epoch": 2.2089795918367345,
      "grad_norm": 0.4892471134662628,
      "learning_rate": 5.4450261780104716e-05,
      "logits/chosen": -4.4286956787109375,
      "logits/rejected": -4.3171610832214355,
      "logps/chosen": -108.22352600097656,
      "logps/rejected": -137.69073486328125,
      "loss": 0.25264739990234375,
      "rewards/accuracies": 0.850781261920929,
      "rewards/chosen": -2.9794039726257324,
      "rewards/margins": 2.2621421813964844,
      "rewards/rejected": -5.241546154022217,
      "step": 2030
    },
    {
      "epoch": 2.2198639455782314,
      "grad_norm": 0.5124601125717163,
      "learning_rate": 5.37023186237846e-05,
      "logits/chosen": -4.357671737670898,
      "logits/rejected": -4.335933685302734,
      "logps/chosen": -110.78375244140625,
      "logps/rejected": -140.46006774902344,
      "loss": 0.24789047241210938,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -2.977566957473755,
      "rewards/margins": 2.320561170578003,
      "rewards/rejected": -5.2981276512146,
      "step": 2040
    },
    {
      "epoch": 2.230748299319728,
      "grad_norm": 0.590368926525116,
      "learning_rate": 5.295437546746448e-05,
      "logits/chosen": -4.396256446838379,
      "logits/rejected": -4.374178409576416,
      "logps/chosen": -108.7447280883789,
      "logps/rejected": -136.3175506591797,
      "loss": 0.27484884262084963,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -2.8392646312713623,
      "rewards/margins": 2.158670663833618,
      "rewards/rejected": -4.9979352951049805,
      "step": 2050
    },
    {
      "epoch": 2.2416326530612247,
      "grad_norm": 0.5153351426124573,
      "learning_rate": 5.220643231114436e-05,
      "logits/chosen": -4.382789134979248,
      "logits/rejected": -4.334689140319824,
      "logps/chosen": -108.401611328125,
      "logps/rejected": -137.27871704101562,
      "loss": 0.2631440877914429,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -2.8784780502319336,
      "rewards/margins": 2.264145612716675,
      "rewards/rejected": -5.1426239013671875,
      "step": 2060
    },
    {
      "epoch": 2.252517006802721,
      "grad_norm": 0.5086898803710938,
      "learning_rate": 5.145848915482424e-05,
      "logits/chosen": -4.282557487487793,
      "logits/rejected": -4.198935508728027,
      "logps/chosen": -107.8048095703125,
      "logps/rejected": -135.6783905029297,
      "loss": 0.26320858001708985,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -2.7925009727478027,
      "rewards/margins": 2.1981430053710938,
      "rewards/rejected": -4.990644454956055,
      "step": 2070
    },
    {
      "epoch": 2.2634013605442176,
      "grad_norm": 0.5398783683776855,
      "learning_rate": 5.0710545998504114e-05,
      "logits/chosen": -4.345008850097656,
      "logits/rejected": -4.317435264587402,
      "logps/chosen": -106.77925872802734,
      "logps/rejected": -136.1848907470703,
      "loss": 0.2555952787399292,
      "rewards/accuracies": 0.8570312261581421,
      "rewards/chosen": -2.866353988647461,
      "rewards/margins": 2.281134843826294,
      "rewards/rejected": -5.147488594055176,
      "step": 2080
    },
    {
      "epoch": 2.2742857142857145,
      "grad_norm": 0.4644974172115326,
      "learning_rate": 4.9962602842183996e-05,
      "logits/chosen": -4.256831169128418,
      "logits/rejected": -4.2593584060668945,
      "logps/chosen": -106.45500183105469,
      "logps/rejected": -136.2468719482422,
      "loss": 0.26251227855682374,
      "rewards/accuracies": 0.84765625,
      "rewards/chosen": -2.910574436187744,
      "rewards/margins": 2.2922780513763428,
      "rewards/rejected": -5.202852725982666,
      "step": 2090
    },
    {
      "epoch": 2.285170068027211,
      "grad_norm": 0.38658151030540466,
      "learning_rate": 4.921465968586388e-05,
      "logits/chosen": -4.299098491668701,
      "logits/rejected": -4.222846984863281,
      "logps/chosen": -111.60084533691406,
      "logps/rejected": -139.23606872558594,
      "loss": 0.26093616485595705,
      "rewards/accuracies": 0.8460937738418579,
      "rewards/chosen": -2.9908206462860107,
      "rewards/margins": 2.2845864295959473,
      "rewards/rejected": -5.275406837463379,
      "step": 2100
    },
    {
      "epoch": 2.2960544217687073,
      "grad_norm": 0.5244455933570862,
      "learning_rate": 4.846671652954376e-05,
      "logits/chosen": -4.305111408233643,
      "logits/rejected": -4.2441325187683105,
      "logps/chosen": -105.7240982055664,
      "logps/rejected": -132.33541870117188,
      "loss": 0.26281836032867434,
      "rewards/accuracies": 0.8578125238418579,
      "rewards/chosen": -2.8944408893585205,
      "rewards/margins": 2.207293748855591,
      "rewards/rejected": -5.1017351150512695,
      "step": 2110
    },
    {
      "epoch": 2.306938775510204,
      "grad_norm": 0.5264010429382324,
      "learning_rate": 4.7718773373223636e-05,
      "logits/chosen": -4.359262943267822,
      "logits/rejected": -4.287570476531982,
      "logps/chosen": -107.4122085571289,
      "logps/rejected": -135.27484130859375,
      "loss": 0.279754638671875,
      "rewards/accuracies": 0.821093738079071,
      "rewards/chosen": -2.8419368267059326,
      "rewards/margins": 2.1469409465789795,
      "rewards/rejected": -4.98887825012207,
      "step": 2120
    },
    {
      "epoch": 2.3178231292517006,
      "grad_norm": 0.6190440058708191,
      "learning_rate": 4.697083021690352e-05,
      "logits/chosen": -4.3371100425720215,
      "logits/rejected": -4.2610368728637695,
      "logps/chosen": -109.62430572509766,
      "logps/rejected": -136.06649780273438,
      "loss": 0.27123911380767823,
      "rewards/accuracies": 0.8414062261581421,
      "rewards/chosen": -2.924973487854004,
      "rewards/margins": 2.1363399028778076,
      "rewards/rejected": -5.061313152313232,
      "step": 2130
    },
    {
      "epoch": 2.328707482993197,
      "grad_norm": 0.4766679108142853,
      "learning_rate": 4.62228870605834e-05,
      "logits/chosen": -4.276871681213379,
      "logits/rejected": -4.208295822143555,
      "logps/chosen": -110.64579010009766,
      "logps/rejected": -139.74888610839844,
      "loss": 0.24541153907775878,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.862879991531372,
      "rewards/margins": 2.3135013580322266,
      "rewards/rejected": -5.176381587982178,
      "step": 2140
    },
    {
      "epoch": 2.339591836734694,
      "grad_norm": 0.4791225492954254,
      "learning_rate": 4.5474943904263275e-05,
      "logits/chosen": -4.355441093444824,
      "logits/rejected": -4.288341522216797,
      "logps/chosen": -105.79945373535156,
      "logps/rejected": -131.28944396972656,
      "loss": 0.27122857570648196,
      "rewards/accuracies": 0.8414062261581421,
      "rewards/chosen": -2.718742609024048,
      "rewards/margins": 2.105710744857788,
      "rewards/rejected": -4.824452877044678,
      "step": 2150
    },
    {
      "epoch": 2.3504761904761904,
      "grad_norm": 0.6803158521652222,
      "learning_rate": 4.472700074794316e-05,
      "logits/chosen": -4.416818141937256,
      "logits/rejected": -4.399478435516357,
      "logps/chosen": -105.3951187133789,
      "logps/rejected": -134.6707763671875,
      "loss": 0.25935161113739014,
      "rewards/accuracies": 0.846875011920929,
      "rewards/chosen": -2.693054676055908,
      "rewards/margins": 2.30621337890625,
      "rewards/rejected": -4.999267578125,
      "step": 2160
    },
    {
      "epoch": 2.3613605442176873,
      "grad_norm": 0.5753185749053955,
      "learning_rate": 4.397905759162304e-05,
      "logits/chosen": -4.324499607086182,
      "logits/rejected": -4.302281856536865,
      "logps/chosen": -108.54312896728516,
      "logps/rejected": -138.02798461914062,
      "loss": 0.24565551280975342,
      "rewards/accuracies": 0.85546875,
      "rewards/chosen": -2.722982883453369,
      "rewards/margins": 2.3493704795837402,
      "rewards/rejected": -5.072353363037109,
      "step": 2170
    },
    {
      "epoch": 2.3722448979591837,
      "grad_norm": 0.5277524590492249,
      "learning_rate": 4.323111443530292e-05,
      "logits/chosen": -4.220860481262207,
      "logits/rejected": -4.199186325073242,
      "logps/chosen": -109.83711242675781,
      "logps/rejected": -138.1239013671875,
      "loss": 0.2549403429031372,
      "rewards/accuracies": 0.854687511920929,
      "rewards/chosen": -2.7489871978759766,
      "rewards/margins": 2.2629806995391846,
      "rewards/rejected": -5.011967658996582,
      "step": 2180
    },
    {
      "epoch": 2.38312925170068,
      "grad_norm": 0.6214152574539185,
      "learning_rate": 4.24831712789828e-05,
      "logits/chosen": -4.419411659240723,
      "logits/rejected": -4.302346706390381,
      "logps/chosen": -109.27337646484375,
      "logps/rejected": -137.83436584472656,
      "loss": 0.25436248779296877,
      "rewards/accuracies": 0.852343738079071,
      "rewards/chosen": -2.788382053375244,
      "rewards/margins": 2.3577487468719482,
      "rewards/rejected": -5.1461310386657715,
      "step": 2190
    },
    {
      "epoch": 2.394013605442177,
      "grad_norm": 0.6792303323745728,
      "learning_rate": 4.173522812266268e-05,
      "logits/chosen": -4.336219787597656,
      "logits/rejected": -4.238919258117676,
      "logps/chosen": -107.77840423583984,
      "logps/rejected": -135.3695831298828,
      "loss": 0.26418342590332033,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.842252492904663,
      "rewards/margins": 2.2352678775787354,
      "rewards/rejected": -5.077520847320557,
      "step": 2200
    },
    {
      "epoch": 2.4048979591836734,
      "grad_norm": 0.5071048736572266,
      "learning_rate": 4.098728496634256e-05,
      "logits/chosen": -4.275275230407715,
      "logits/rejected": -4.160519599914551,
      "logps/chosen": -108.19576263427734,
      "logps/rejected": -135.44322204589844,
      "loss": 0.25397777557373047,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.940242052078247,
      "rewards/margins": 2.3158531188964844,
      "rewards/rejected": -5.256094932556152,
      "step": 2210
    },
    {
      "epoch": 2.4157823129251703,
      "grad_norm": 0.5598253607749939,
      "learning_rate": 4.0239341810022444e-05,
      "logits/chosen": -4.34271240234375,
      "logits/rejected": -4.281783103942871,
      "logps/chosen": -111.84043884277344,
      "logps/rejected": -139.2178497314453,
      "loss": 0.25306150913238523,
      "rewards/accuracies": 0.8453124761581421,
      "rewards/chosen": -2.9429874420166016,
      "rewards/margins": 2.3303146362304688,
      "rewards/rejected": -5.27330207824707,
      "step": 2220
    },
    {
      "epoch": 2.4266666666666667,
      "grad_norm": 0.4812939763069153,
      "learning_rate": 3.949139865370232e-05,
      "logits/chosen": -4.2378034591674805,
      "logits/rejected": -4.214576244354248,
      "logps/chosen": -108.81040954589844,
      "logps/rejected": -136.83827209472656,
      "loss": 0.26318118572235105,
      "rewards/accuracies": 0.8421875238418579,
      "rewards/chosen": -2.9642369747161865,
      "rewards/margins": 2.3236169815063477,
      "rewards/rejected": -5.287853717803955,
      "step": 2230
    },
    {
      "epoch": 2.437551020408163,
      "grad_norm": 0.5383220314979553,
      "learning_rate": 3.87434554973822e-05,
      "logits/chosen": -4.168697357177734,
      "logits/rejected": -4.0632429122924805,
      "logps/chosen": -113.34503173828125,
      "logps/rejected": -140.87977600097656,
      "loss": 0.2669438362121582,
      "rewards/accuracies": 0.8453124761581421,
      "rewards/chosen": -3.066909074783325,
      "rewards/margins": 2.23907470703125,
      "rewards/rejected": -5.305983066558838,
      "step": 2240
    },
    {
      "epoch": 2.4484353741496596,
      "grad_norm": 0.4986330270767212,
      "learning_rate": 3.799551234106208e-05,
      "logits/chosen": -4.118273735046387,
      "logits/rejected": -4.0370354652404785,
      "logps/chosen": -110.60295104980469,
      "logps/rejected": -138.21926879882812,
      "loss": 0.26406409740448,
      "rewards/accuracies": 0.850781261920929,
      "rewards/chosen": -2.9518330097198486,
      "rewards/margins": 2.2067174911499023,
      "rewards/rejected": -5.158550262451172,
      "step": 2250
    },
    {
      "epoch": 2.4593197278911565,
      "grad_norm": 0.5483224391937256,
      "learning_rate": 3.724756918474196e-05,
      "logits/chosen": -4.164062023162842,
      "logits/rejected": -4.178997993469238,
      "logps/chosen": -112.80091857910156,
      "logps/rejected": -140.44825744628906,
      "loss": 0.25186111927032473,
      "rewards/accuracies": 0.8570312261581421,
      "rewards/chosen": -3.0120620727539062,
      "rewards/margins": 2.2925403118133545,
      "rewards/rejected": -5.30460262298584,
      "step": 2260
    },
    {
      "epoch": 2.470204081632653,
      "grad_norm": 0.49677279591560364,
      "learning_rate": 3.649962602842184e-05,
      "logits/chosen": -4.339353561401367,
      "logits/rejected": -4.249403953552246,
      "logps/chosen": -110.08421325683594,
      "logps/rejected": -137.5714569091797,
      "loss": 0.25184576511383056,
      "rewards/accuracies": 0.848437488079071,
      "rewards/chosen": -3.0821781158447266,
      "rewards/margins": 2.328827381134033,
      "rewards/rejected": -5.411005020141602,
      "step": 2270
    },
    {
      "epoch": 2.48108843537415,
      "grad_norm": 0.5725961923599243,
      "learning_rate": 3.5751682872101724e-05,
      "logits/chosen": -4.2684502601623535,
      "logits/rejected": -4.1795573234558105,
      "logps/chosen": -113.55293273925781,
      "logps/rejected": -142.6102752685547,
      "loss": 0.24979727268218993,
      "rewards/accuracies": 0.8492187261581421,
      "rewards/chosen": -3.1018178462982178,
      "rewards/margins": 2.409865617752075,
      "rewards/rejected": -5.511682987213135,
      "step": 2280
    },
    {
      "epoch": 2.4919727891156462,
      "grad_norm": 0.5567153096199036,
      "learning_rate": 3.5003739715781606e-05,
      "logits/chosen": -4.302936553955078,
      "logits/rejected": -4.220743656158447,
      "logps/chosen": -107.4947738647461,
      "logps/rejected": -135.80044555664062,
      "loss": 0.2599829435348511,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -2.999293804168701,
      "rewards/margins": 2.2999939918518066,
      "rewards/rejected": -5.299286842346191,
      "step": 2290
    },
    {
      "epoch": 2.5028571428571427,
      "grad_norm": 0.46114829182624817,
      "learning_rate": 3.425579655946148e-05,
      "logits/chosen": -4.257116794586182,
      "logits/rejected": -4.284517765045166,
      "logps/chosen": -109.46214294433594,
      "logps/rejected": -138.5498809814453,
      "loss": 0.262731671333313,
      "rewards/accuracies": 0.8414062261581421,
      "rewards/chosen": -3.0271120071411133,
      "rewards/margins": 2.3265421390533447,
      "rewards/rejected": -5.353653907775879,
      "step": 2300
    },
    {
      "epoch": 2.5137414965986395,
      "grad_norm": 0.5548444390296936,
      "learning_rate": 3.350785340314136e-05,
      "logits/chosen": -4.13088321685791,
      "logits/rejected": -4.090167045593262,
      "logps/chosen": -112.9273681640625,
      "logps/rejected": -140.58865356445312,
      "loss": 0.26635379791259767,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -3.1592650413513184,
      "rewards/margins": 2.2824602127075195,
      "rewards/rejected": -5.441725730895996,
      "step": 2310
    },
    {
      "epoch": 2.524625850340136,
      "grad_norm": 0.6786127686500549,
      "learning_rate": 3.275991024682124e-05,
      "logits/chosen": -4.259942531585693,
      "logits/rejected": -4.2501912117004395,
      "logps/chosen": -113.3272476196289,
      "logps/rejected": -140.26583862304688,
      "loss": 0.26966152191162107,
      "rewards/accuracies": 0.8382812738418579,
      "rewards/chosen": -3.1748275756835938,
      "rewards/margins": 2.2647013664245605,
      "rewards/rejected": -5.439528942108154,
      "step": 2320
    },
    {
      "epoch": 2.535510204081633,
      "grad_norm": 0.5421922206878662,
      "learning_rate": 3.201196709050112e-05,
      "logits/chosen": -4.2332868576049805,
      "logits/rejected": -4.159974098205566,
      "logps/chosen": -112.0591049194336,
      "logps/rejected": -142.06764221191406,
      "loss": 0.26064703464508054,
      "rewards/accuracies": 0.842968761920929,
      "rewards/chosen": -3.2271761894226074,
      "rewards/margins": 2.313660144805908,
      "rewards/rejected": -5.540836334228516,
      "step": 2330
    },
    {
      "epoch": 2.5463945578231293,
      "grad_norm": 0.6206730008125305,
      "learning_rate": 3.1264023934181e-05,
      "logits/chosen": -4.249005317687988,
      "logits/rejected": -4.177195072174072,
      "logps/chosen": -110.71000671386719,
      "logps/rejected": -138.38018798828125,
      "loss": 0.264833927154541,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -3.2622718811035156,
      "rewards/margins": 2.270108699798584,
      "rewards/rejected": -5.532380104064941,
      "step": 2340
    },
    {
      "epoch": 2.5572789115646257,
      "grad_norm": 0.6249208450317383,
      "learning_rate": 3.0516080777860885e-05,
      "logits/chosen": -4.308166980743408,
      "logits/rejected": -4.222630977630615,
      "logps/chosen": -115.17570495605469,
      "logps/rejected": -144.38851928710938,
      "loss": 0.2524010896682739,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.351501941680908,
      "rewards/margins": 2.3318376541137695,
      "rewards/rejected": -5.683339595794678,
      "step": 2350
    },
    {
      "epoch": 2.5681632653061226,
      "grad_norm": 0.6504024863243103,
      "learning_rate": 2.9768137621540764e-05,
      "logits/chosen": -4.296634674072266,
      "logits/rejected": -4.280292987823486,
      "logps/chosen": -111.20161437988281,
      "logps/rejected": -141.90963745117188,
      "loss": 0.26886260509490967,
      "rewards/accuracies": 0.83984375,
      "rewards/chosen": -3.2387752532958984,
      "rewards/margins": 2.3250107765197754,
      "rewards/rejected": -5.563786506652832,
      "step": 2360
    },
    {
      "epoch": 2.579047619047619,
      "grad_norm": 0.6845365762710571,
      "learning_rate": 2.9020194465220646e-05,
      "logits/chosen": -4.261755466461182,
      "logits/rejected": -4.167389392852783,
      "logps/chosen": -111.0313949584961,
      "logps/rejected": -138.18850708007812,
      "loss": 0.2643138885498047,
      "rewards/accuracies": 0.8335937261581421,
      "rewards/chosen": -3.0966439247131348,
      "rewards/margins": 2.2732677459716797,
      "rewards/rejected": -5.369911193847656,
      "step": 2370
    },
    {
      "epoch": 2.589931972789116,
      "grad_norm": 0.4879046380519867,
      "learning_rate": 2.827225130890053e-05,
      "logits/chosen": -4.409466743469238,
      "logits/rejected": -4.352431297302246,
      "logps/chosen": -106.5313949584961,
      "logps/rejected": -137.22837829589844,
      "loss": 0.25514419078826905,
      "rewards/accuracies": 0.848437488079071,
      "rewards/chosen": -2.9418864250183105,
      "rewards/margins": 2.351497173309326,
      "rewards/rejected": -5.293383598327637,
      "step": 2380
    },
    {
      "epoch": 2.6008163265306123,
      "grad_norm": 0.5651640892028809,
      "learning_rate": 2.7524308152580404e-05,
      "logits/chosen": -4.4179205894470215,
      "logits/rejected": -4.379713535308838,
      "logps/chosen": -108.92497253417969,
      "logps/rejected": -137.62344360351562,
      "loss": 0.24242501258850097,
      "rewards/accuracies": 0.8578125238418579,
      "rewards/chosen": -3.073535203933716,
      "rewards/margins": 2.418722152709961,
      "rewards/rejected": -5.492257118225098,
      "step": 2390
    },
    {
      "epoch": 2.6117006802721088,
      "grad_norm": 0.523777425289154,
      "learning_rate": 2.6776364996260283e-05,
      "logits/chosen": -4.350298881530762,
      "logits/rejected": -4.247467517852783,
      "logps/chosen": -112.7027359008789,
      "logps/rejected": -142.613037109375,
      "loss": 0.2534715890884399,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.111325979232788,
      "rewards/margins": 2.404831886291504,
      "rewards/rejected": -5.516158103942871,
      "step": 2400
    },
    {
      "epoch": 2.622585034013605,
      "grad_norm": 0.5190083384513855,
      "learning_rate": 2.6028421839940165e-05,
      "logits/chosen": -4.415008544921875,
      "logits/rejected": -4.2630157470703125,
      "logps/chosen": -109.89335632324219,
      "logps/rejected": -137.45790100097656,
      "loss": 0.26002261638641355,
      "rewards/accuracies": 0.8382812738418579,
      "rewards/chosen": -3.0627522468566895,
      "rewards/margins": 2.3217196464538574,
      "rewards/rejected": -5.384472370147705,
      "step": 2410
    },
    {
      "epoch": 2.633469387755102,
      "grad_norm": 0.5784986615180969,
      "learning_rate": 2.5280478683620047e-05,
      "logits/chosen": -4.250868797302246,
      "logits/rejected": -4.1973700523376465,
      "logps/chosen": -111.86470794677734,
      "logps/rejected": -139.78781127929688,
      "loss": 0.25365798473358153,
      "rewards/accuracies": 0.852343738079071,
      "rewards/chosen": -3.048133611679077,
      "rewards/margins": 2.3121330738067627,
      "rewards/rejected": -5.36026668548584,
      "step": 2420
    },
    {
      "epoch": 2.6443537414965985,
      "grad_norm": 0.5177122354507446,
      "learning_rate": 2.4532535527299926e-05,
      "logits/chosen": -4.4071784019470215,
      "logits/rejected": -4.31978702545166,
      "logps/chosen": -114.18182373046875,
      "logps/rejected": -142.67727661132812,
      "loss": 0.24829225540161132,
      "rewards/accuracies": 0.8421875238418579,
      "rewards/chosen": -3.116865873336792,
      "rewards/margins": 2.388279438018799,
      "rewards/rejected": -5.50514554977417,
      "step": 2430
    },
    {
      "epoch": 2.6552380952380954,
      "grad_norm": 0.6798398494720459,
      "learning_rate": 2.3784592370979808e-05,
      "logits/chosen": -4.312591075897217,
      "logits/rejected": -4.273346424102783,
      "logps/chosen": -109.682861328125,
      "logps/rejected": -138.36782836914062,
      "loss": 0.2676241397857666,
      "rewards/accuracies": 0.839062511920929,
      "rewards/chosen": -2.952719211578369,
      "rewards/margins": 2.2746853828430176,
      "rewards/rejected": -5.227404594421387,
      "step": 2440
    },
    {
      "epoch": 2.666122448979592,
      "grad_norm": 0.6012817025184631,
      "learning_rate": 2.3036649214659687e-05,
      "logits/chosen": -4.368619918823242,
      "logits/rejected": -4.296780586242676,
      "logps/chosen": -110.58982849121094,
      "logps/rejected": -139.80557250976562,
      "loss": 0.25081255435943606,
      "rewards/accuracies": 0.86328125,
      "rewards/chosen": -2.887507915496826,
      "rewards/margins": 2.3469045162200928,
      "rewards/rejected": -5.23441219329834,
      "step": 2450
    },
    {
      "epoch": 2.6770068027210883,
      "grad_norm": 0.5708255171775818,
      "learning_rate": 2.228870605833957e-05,
      "logits/chosen": -4.390729904174805,
      "logits/rejected": -4.355556964874268,
      "logps/chosen": -107.79595947265625,
      "logps/rejected": -134.82522583007812,
      "loss": 0.27308039665222167,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -2.8849995136260986,
      "rewards/margins": 2.2530252933502197,
      "rewards/rejected": -5.13802433013916,
      "step": 2460
    },
    {
      "epoch": 2.687891156462585,
      "grad_norm": 0.47491082549095154,
      "learning_rate": 2.1540762902019448e-05,
      "logits/chosen": -4.335263729095459,
      "logits/rejected": -4.283821105957031,
      "logps/chosen": -108.29530334472656,
      "logps/rejected": -135.4779815673828,
      "loss": 0.2607786417007446,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.8725428581237793,
      "rewards/margins": 2.3250625133514404,
      "rewards/rejected": -5.197605133056641,
      "step": 2470
    },
    {
      "epoch": 2.6987755102040816,
      "grad_norm": 0.5339019298553467,
      "learning_rate": 2.0792819745699327e-05,
      "logits/chosen": -4.364335060119629,
      "logits/rejected": -4.271590709686279,
      "logps/chosen": -109.25032043457031,
      "logps/rejected": -137.19566345214844,
      "loss": 0.25407183170318604,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.983041286468506,
      "rewards/margins": 2.2924866676330566,
      "rewards/rejected": -5.275527477264404,
      "step": 2480
    },
    {
      "epoch": 2.7096598639455785,
      "grad_norm": 0.5330259799957275,
      "learning_rate": 2.004487658937921e-05,
      "logits/chosen": -4.272704601287842,
      "logits/rejected": -4.25288724899292,
      "logps/chosen": -109.9069595336914,
      "logps/rejected": -138.80812072753906,
      "loss": 0.25796828269958494,
      "rewards/accuracies": 0.858593761920929,
      "rewards/chosen": -3.022810697555542,
      "rewards/margins": 2.3227627277374268,
      "rewards/rejected": -5.3455729484558105,
      "step": 2490
    },
    {
      "epoch": 2.720544217687075,
      "grad_norm": 0.48291197419166565,
      "learning_rate": 1.9296933433059088e-05,
      "logits/chosen": -4.345968723297119,
      "logits/rejected": -4.260931015014648,
      "logps/chosen": -114.93940734863281,
      "logps/rejected": -143.88821411132812,
      "loss": 0.24913032054901124,
      "rewards/accuracies": 0.844531238079071,
      "rewards/chosen": -3.0745816230773926,
      "rewards/margins": 2.3651044368743896,
      "rewards/rejected": -5.439685821533203,
      "step": 2500
    },
    {
      "epoch": 2.720544217687075,
      "eval_logits/chosen": -4.365705490112305,
      "eval_logits/rejected": -4.315591335296631,
      "eval_logps/chosen": -113.0962142944336,
      "eval_logps/rejected": -135.48565673828125,
      "eval_loss": 0.45733407139778137,
      "eval_rewards/accuracies": 0.721666693687439,
      "eval_rewards/chosen": -3.3491051197052,
      "eval_rewards/margins": 1.6145259141921997,
      "eval_rewards/rejected": -4.9636311531066895,
      "eval_runtime": 104.6509,
      "eval_samples_per_second": 22.933,
      "eval_steps_per_second": 2.867,
      "step": 2500
    },
    {
      "epoch": 2.7314285714285713,
      "grad_norm": 0.5165424346923828,
      "learning_rate": 1.854899027673897e-05,
      "logits/chosen": -4.321234226226807,
      "logits/rejected": -4.339522838592529,
      "logps/chosen": -113.94354248046875,
      "logps/rejected": -145.6949005126953,
      "loss": 0.23960201740264891,
      "rewards/accuracies": 0.8609374761581421,
      "rewards/chosen": -3.1459145545959473,
      "rewards/margins": 2.455543041229248,
      "rewards/rejected": -5.6014580726623535,
      "step": 2510
    },
    {
      "epoch": 2.7423129251700678,
      "grad_norm": 0.5312625765800476,
      "learning_rate": 1.780104712041885e-05,
      "logits/chosen": -4.337346076965332,
      "logits/rejected": -4.251561164855957,
      "logps/chosen": -111.99488830566406,
      "logps/rejected": -144.23214721679688,
      "loss": 0.25103614330291746,
      "rewards/accuracies": 0.854687511920929,
      "rewards/chosen": -3.094682455062866,
      "rewards/margins": 2.4001941680908203,
      "rewards/rejected": -5.494876384735107,
      "step": 2520
    },
    {
      "epoch": 2.7531972789115646,
      "grad_norm": 0.600907564163208,
      "learning_rate": 1.705310396409873e-05,
      "logits/chosen": -4.3451642990112305,
      "logits/rejected": -4.26493501663208,
      "logps/chosen": -112.1881103515625,
      "logps/rejected": -141.45523071289062,
      "loss": 0.2668695688247681,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -3.144423246383667,
      "rewards/margins": 2.3063411712646484,
      "rewards/rejected": -5.4507646560668945,
      "step": 2530
    },
    {
      "epoch": 2.764081632653061,
      "grad_norm": 0.5516788363456726,
      "learning_rate": 1.630516080777861e-05,
      "logits/chosen": -4.365313529968262,
      "logits/rejected": -4.301580905914307,
      "logps/chosen": -113.34366607666016,
      "logps/rejected": -140.64866638183594,
      "loss": 0.25537886619567873,
      "rewards/accuracies": 0.8421875238418579,
      "rewards/chosen": -3.1079978942871094,
      "rewards/margins": 2.3831117153167725,
      "rewards/rejected": -5.4911088943481445,
      "step": 2540
    },
    {
      "epoch": 2.774965986394558,
      "grad_norm": 0.6179223656654358,
      "learning_rate": 1.555721765145849e-05,
      "logits/chosen": -4.21829891204834,
      "logits/rejected": -4.297372817993164,
      "logps/chosen": -113.7059097290039,
      "logps/rejected": -143.8195037841797,
      "loss": 0.25823125839233396,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.127429246902466,
      "rewards/margins": 2.3170604705810547,
      "rewards/rejected": -5.444489479064941,
      "step": 2550
    },
    {
      "epoch": 2.7858503401360544,
      "grad_norm": 0.583663284778595,
      "learning_rate": 1.480927449513837e-05,
      "logits/chosen": -4.288941860198975,
      "logits/rejected": -4.223701477050781,
      "logps/chosen": -107.650634765625,
      "logps/rejected": -136.13128662109375,
      "loss": 0.2668041706085205,
      "rewards/accuracies": 0.8421875238418579,
      "rewards/chosen": -3.0965638160705566,
      "rewards/margins": 2.296903610229492,
      "rewards/rejected": -5.393467426300049,
      "step": 2560
    },
    {
      "epoch": 2.796734693877551,
      "grad_norm": 0.6680634021759033,
      "learning_rate": 1.4061331338818251e-05,
      "logits/chosen": -4.223349571228027,
      "logits/rejected": -4.183167457580566,
      "logps/chosen": -112.09420013427734,
      "logps/rejected": -140.25161743164062,
      "loss": 0.26484224796295164,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -3.1480445861816406,
      "rewards/margins": 2.2968597412109375,
      "rewards/rejected": -5.444904804229736,
      "step": 2570
    },
    {
      "epoch": 2.8076190476190477,
      "grad_norm": 0.5711221694946289,
      "learning_rate": 1.3313388182498132e-05,
      "logits/chosen": -4.209951877593994,
      "logits/rejected": -4.184714317321777,
      "logps/chosen": -113.71773529052734,
      "logps/rejected": -142.4655303955078,
      "loss": 0.246695613861084,
      "rewards/accuracies": 0.854687511920929,
      "rewards/chosen": -3.118393659591675,
      "rewards/margins": 2.321854829788208,
      "rewards/rejected": -5.440248489379883,
      "step": 2580
    },
    {
      "epoch": 2.818503401360544,
      "grad_norm": 0.5573529005050659,
      "learning_rate": 1.256544502617801e-05,
      "logits/chosen": -4.283592224121094,
      "logits/rejected": -4.182808876037598,
      "logps/chosen": -109.0276870727539,
      "logps/rejected": -136.32115173339844,
      "loss": 0.26199021339416506,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.077354907989502,
      "rewards/margins": 2.296853542327881,
      "rewards/rejected": -5.374207973480225,
      "step": 2590
    },
    {
      "epoch": 2.829387755102041,
      "grad_norm": 0.5489506721496582,
      "learning_rate": 1.1817501869857891e-05,
      "logits/chosen": -4.319849491119385,
      "logits/rejected": -4.266009330749512,
      "logps/chosen": -111.2982406616211,
      "logps/rejected": -140.7898712158203,
      "loss": 0.24689509868621826,
      "rewards/accuracies": 0.85546875,
      "rewards/chosen": -3.052299976348877,
      "rewards/margins": 2.4046759605407715,
      "rewards/rejected": -5.45697546005249,
      "step": 2600
    },
    {
      "epoch": 2.8402721088435374,
      "grad_norm": 0.4994034767150879,
      "learning_rate": 1.1069558713537771e-05,
      "logits/chosen": -4.284036159515381,
      "logits/rejected": -4.174200534820557,
      "logps/chosen": -108.4582290649414,
      "logps/rejected": -137.0861358642578,
      "loss": 0.24927878379821777,
      "rewards/accuracies": 0.850781261920929,
      "rewards/chosen": -3.06015944480896,
      "rewards/margins": 2.373507499694824,
      "rewards/rejected": -5.433667182922363,
      "step": 2610
    },
    {
      "epoch": 2.851156462585034,
      "grad_norm": 0.5960386991500854,
      "learning_rate": 1.0321615557217652e-05,
      "logits/chosen": -4.267377853393555,
      "logits/rejected": -4.11255407333374,
      "logps/chosen": -107.7799301147461,
      "logps/rejected": -135.4087371826172,
      "loss": 0.2596835374832153,
      "rewards/accuracies": 0.8578125238418579,
      "rewards/chosen": -3.0362658500671387,
      "rewards/margins": 2.3004674911499023,
      "rewards/rejected": -5.336733818054199,
      "step": 2620
    },
    {
      "epoch": 2.8620408163265307,
      "grad_norm": 0.5376743674278259,
      "learning_rate": 9.573672400897532e-06,
      "logits/chosen": -4.321850776672363,
      "logits/rejected": -4.283345699310303,
      "logps/chosen": -111.8201904296875,
      "logps/rejected": -141.06961059570312,
      "loss": 0.2590658664703369,
      "rewards/accuracies": 0.846875011920929,
      "rewards/chosen": -3.168375253677368,
      "rewards/margins": 2.404663562774658,
      "rewards/rejected": -5.5730390548706055,
      "step": 2630
    },
    {
      "epoch": 2.872925170068027,
      "grad_norm": 0.6059140563011169,
      "learning_rate": 8.825729244577413e-06,
      "logits/chosen": -4.2465620040893555,
      "logits/rejected": -4.216645240783691,
      "logps/chosen": -111.7029800415039,
      "logps/rejected": -141.03634643554688,
      "loss": 0.2559323310852051,
      "rewards/accuracies": 0.835156261920929,
      "rewards/chosen": -3.0912365913391113,
      "rewards/margins": 2.3511834144592285,
      "rewards/rejected": -5.44242000579834,
      "step": 2640
    },
    {
      "epoch": 2.883809523809524,
      "grad_norm": 0.6501792073249817,
      "learning_rate": 8.077786088257293e-06,
      "logits/chosen": -4.256227016448975,
      "logits/rejected": -4.207150936126709,
      "logps/chosen": -112.23890686035156,
      "logps/rejected": -142.5592803955078,
      "loss": 0.24930896759033203,
      "rewards/accuracies": 0.852343738079071,
      "rewards/chosen": -3.1018924713134766,
      "rewards/margins": 2.3848838806152344,
      "rewards/rejected": -5.486776351928711,
      "step": 2650
    },
    {
      "epoch": 2.8946938775510205,
      "grad_norm": 0.5240415334701538,
      "learning_rate": 7.329842931937172e-06,
      "logits/chosen": -4.290334701538086,
      "logits/rejected": -4.219546318054199,
      "logps/chosen": -113.01908111572266,
      "logps/rejected": -140.31143188476562,
      "loss": 0.26281979084014895,
      "rewards/accuracies": 0.8335937261581421,
      "rewards/chosen": -3.170388698577881,
      "rewards/margins": 2.3088202476501465,
      "rewards/rejected": -5.4792094230651855,
      "step": 2660
    },
    {
      "epoch": 2.905578231292517,
      "grad_norm": 0.6188440322875977,
      "learning_rate": 6.5818997756170535e-06,
      "logits/chosen": -4.308688163757324,
      "logits/rejected": -4.274056911468506,
      "logps/chosen": -111.7992935180664,
      "logps/rejected": -140.93899536132812,
      "loss": 0.26155283451080324,
      "rewards/accuracies": 0.848437488079071,
      "rewards/chosen": -3.144166946411133,
      "rewards/margins": 2.3630402088165283,
      "rewards/rejected": -5.507206916809082,
      "step": 2670
    },
    {
      "epoch": 2.9164625850340133,
      "grad_norm": 0.6599675416946411,
      "learning_rate": 5.833956619296934e-06,
      "logits/chosen": -4.264589309692383,
      "logits/rejected": -4.202670097351074,
      "logps/chosen": -112.2196044921875,
      "logps/rejected": -143.64590454101562,
      "loss": 0.2505872964859009,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.0915122032165527,
      "rewards/margins": 2.401637077331543,
      "rewards/rejected": -5.4931488037109375,
      "step": 2680
    },
    {
      "epoch": 2.92734693877551,
      "grad_norm": 0.43635502457618713,
      "learning_rate": 5.086013462976814e-06,
      "logits/chosen": -4.175580978393555,
      "logits/rejected": -4.107194900512695,
      "logps/chosen": -110.0128173828125,
      "logps/rejected": -139.2493438720703,
      "loss": 0.25678586959838867,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -3.1154348850250244,
      "rewards/margins": 2.3281521797180176,
      "rewards/rejected": -5.443586826324463,
      "step": 2690
    },
    {
      "epoch": 2.9382312925170067,
      "grad_norm": 0.5918434858322144,
      "learning_rate": 4.338070306656694e-06,
      "logits/chosen": -4.194636344909668,
      "logits/rejected": -4.221497535705566,
      "logps/chosen": -112.2238998413086,
      "logps/rejected": -139.56298828125,
      "loss": 0.25926196575164795,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.071394681930542,
      "rewards/margins": 2.3040318489074707,
      "rewards/rejected": -5.375426292419434,
      "step": 2700
    },
    {
      "epoch": 2.9491156462585035,
      "grad_norm": 0.48835864663124084,
      "learning_rate": 3.5901271503365746e-06,
      "logits/chosen": -4.236893653869629,
      "logits/rejected": -4.151623725891113,
      "logps/chosen": -113.73643493652344,
      "logps/rejected": -140.67477416992188,
      "loss": 0.2562965154647827,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.1573872566223145,
      "rewards/margins": 2.3107500076293945,
      "rewards/rejected": -5.468136787414551,
      "step": 2710
    },
    {
      "epoch": 2.96,
      "grad_norm": 0.6017318367958069,
      "learning_rate": 2.8421839940164547e-06,
      "logits/chosen": -4.273608207702637,
      "logits/rejected": -4.202578544616699,
      "logps/chosen": -112.49813079833984,
      "logps/rejected": -141.3792724609375,
      "loss": 0.2622231960296631,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -3.1241061687469482,
      "rewards/margins": 2.355341911315918,
      "rewards/rejected": -5.479447841644287,
      "step": 2720
    },
    {
      "epoch": 2.9708843537414964,
      "grad_norm": 0.6066134572029114,
      "learning_rate": 2.094240837696335e-06,
      "logits/chosen": -4.19366455078125,
      "logits/rejected": -4.144550800323486,
      "logps/chosen": -112.6771240234375,
      "logps/rejected": -142.29103088378906,
      "loss": 0.24986703395843507,
      "rewards/accuracies": 0.854687511920929,
      "rewards/chosen": -3.070312023162842,
      "rewards/margins": 2.411825656890869,
      "rewards/rejected": -5.482138156890869,
      "step": 2730
    },
    {
      "epoch": 2.9817687074829933,
      "grad_norm": 0.6237701177597046,
      "learning_rate": 1.3462976813762155e-06,
      "logits/chosen": -4.226711750030518,
      "logits/rejected": -4.1740617752075195,
      "logps/chosen": -111.61177825927734,
      "logps/rejected": -141.50173950195312,
      "loss": 0.253825306892395,
      "rewards/accuracies": 0.8492187261581421,
      "rewards/chosen": -3.0980031490325928,
      "rewards/margins": 2.3578884601593018,
      "rewards/rejected": -5.4558916091918945,
      "step": 2740
    },
    {
      "epoch": 2.9926530612244897,
      "grad_norm": 0.6461321711540222,
      "learning_rate": 5.983545250560958e-07,
      "logits/chosen": -4.288928031921387,
      "logits/rejected": -4.2158074378967285,
      "logps/chosen": -109.243896484375,
      "logps/rejected": -137.41378784179688,
      "loss": 0.26066405773162843,
      "rewards/accuracies": 0.8492187261581421,
      "rewards/chosen": -3.127552032470703,
      "rewards/margins": 2.241049289703369,
      "rewards/rejected": -5.368602275848389,
      "step": 2750
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -4.330223560333252,
      "eval_logits/rejected": -4.280575752258301,
      "eval_logps/chosen": -112.92227935791016,
      "eval_logps/rejected": -135.6313934326172,
      "eval_loss": 0.4575725197792053,
      "eval_rewards/accuracies": 0.7233333587646484,
      "eval_rewards/chosen": -3.3317134380340576,
      "eval_rewards/margins": 1.6464910507202148,
      "eval_rewards/rejected": -4.978204727172852,
      "eval_runtime": 104.9021,
      "eval_samples_per_second": 22.878,
      "eval_steps_per_second": 2.86,
      "step": 2757
    }
  ],
  "logging_steps": 10,
  "max_steps": 2757,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "EarlyStoppingCallback": {
      "args": {
        "early_stopping_patience": 3,
        "early_stopping_threshold": 0.0
      },
      "attributes": {
        "early_stopping_patience_counter": 1
      }
    },
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}