{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 681,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0014684287812041115,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06123492121696472,
      "fcm_dpo/q_t": 0.4984707534313202,
      "grad_norm": 38.81842041015625,
      "learning_rate": 0.0,
      "logits/chosen": 0.2677389979362488,
      "logits/rejected": 0.6191319227218628,
      "logps/chosen": -85.37664031982422,
      "logps/ref_chosen": -85.43083190917969,
      "logps/ref_rejected": -79.90458679199219,
      "logps/rejected": -79.91163635253906,
      "loss": 1.3807,
      "margin_dpo/margin_mean": 0.061235010623931885,
      "margin_dpo/margin_std": 0.4424324035644531,
      "step": 1
    },
    {
      "epoch": 0.002936857562408223,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.12442219257354736,
      "fcm_dpo/q_t": 0.5031089186668396,
      "grad_norm": 29.758039474487305,
      "learning_rate": 7.246376811594203e-09,
      "logits/chosen": 0.47771453857421875,
      "logits/rejected": 0.5908545851707458,
      "logps/chosen": -82.11383056640625,
      "logps/ref_chosen": -82.06893157958984,
      "logps/ref_rejected": -81.65457916259766,
      "logps/rejected": -81.57505798339844,
      "loss": 1.3992,
      "margin_dpo/margin_mean": -0.12442246079444885,
      "margin_dpo/margin_std": 0.41358208656311035,
      "step": 2
    },
    {
      "epoch": 0.004405286343612335,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.033603206276893616,
      "fcm_dpo/q_t": 0.49916017055511475,
      "grad_norm": 26.0107479095459,
      "learning_rate": 1.4492753623188406e-08,
      "logits/chosen": 0.4268984794616699,
      "logits/rejected": 0.6501251459121704,
      "logps/chosen": -93.78929901123047,
      "logps/ref_chosen": -93.81098937988281,
      "logps/ref_rejected": -74.22950744628906,
      "logps/rejected": -74.24141693115234,
      "loss": 1.3835,
      "margin_dpo/margin_mean": 0.033603474497795105,
      "margin_dpo/margin_std": 0.4623871445655823,
      "step": 3
    },
    {
      "epoch": 0.005873715124816446,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.027436256408691406,
      "fcm_dpo/q_t": 0.5006846189498901,
      "grad_norm": 32.81400680541992,
      "learning_rate": 2.1739130434782606e-08,
      "logits/chosen": 0.646797776222229,
      "logits/rejected": 0.778215765953064,
      "logps/chosen": -87.31211853027344,
      "logps/ref_chosen": -87.29246520996094,
      "logps/ref_rejected": -93.82425689697266,
      "logps/rejected": -93.81646728515625,
      "loss": 1.3895,
      "margin_dpo/margin_mean": -0.027436137199401855,
      "margin_dpo/margin_std": 0.4442657232284546,
      "step": 4
    },
    {
      "epoch": 0.007342143906020558,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08646883070468903,
      "fcm_dpo/q_t": 0.49783921241760254,
      "grad_norm": 35.074466705322266,
      "learning_rate": 2.898550724637681e-08,
      "logits/chosen": 0.44550371170043945,
      "logits/rejected": 0.6571714878082275,
      "logps/chosen": -89.2946548461914,
      "logps/ref_chosen": -89.33676147460938,
      "logps/ref_rejected": -88.74783325195312,
      "logps/rejected": -88.79219818115234,
      "loss": 1.3781,
      "margin_dpo/margin_mean": 0.08646932244300842,
      "margin_dpo/margin_std": 0.42956870794296265,
      "step": 5
    },
    {
      "epoch": 0.00881057268722467,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04864634573459625,
      "fcm_dpo/q_t": 0.49878376722335815,
      "grad_norm": 35.885719299316406,
      "learning_rate": 3.6231884057971014e-08,
      "logits/chosen": 0.4864919185638428,
      "logits/rejected": 0.8014158010482788,
      "logps/chosen": -97.29067993164062,
      "logps/ref_chosen": -97.32147216796875,
      "logps/ref_rejected": -97.88345336914062,
      "logps/rejected": -97.90130615234375,
      "loss": 1.3819,
      "margin_dpo/margin_mean": 0.048646748065948486,
      "margin_dpo/margin_std": 0.40285322070121765,
      "step": 6
    },
    {
      "epoch": 0.010279001468428781,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.021133705973625183,
      "fcm_dpo/q_t": 0.49947190284729004,
      "grad_norm": 38.00736999511719,
      "learning_rate": 4.347826086956521e-08,
      "logits/chosen": 0.49776560068130493,
      "logits/rejected": 0.5453675985336304,
      "logps/chosen": -86.66970825195312,
      "logps/ref_chosen": -86.64852905273438,
      "logps/ref_rejected": -109.61618041992188,
      "logps/rejected": -109.65848541259766,
      "loss": 1.3846,
      "margin_dpo/margin_mean": 0.021133065223693848,
      "margin_dpo/margin_std": 0.42155858874320984,
      "step": 7
    },
    {
      "epoch": 0.011747430249632892,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.11333945393562317,
      "fcm_dpo/q_t": 0.5028325915336609,
      "grad_norm": 32.925907135009766,
      "learning_rate": 5.0724637681159424e-08,
      "logits/chosen": 0.41005465388298035,
      "logits/rejected": 0.7211995124816895,
      "logps/chosen": -90.00587463378906,
      "logps/ref_chosen": -89.9236831665039,
      "logps/ref_rejected": -86.22804260253906,
      "logps/rejected": -86.19688415527344,
      "loss": 1.3982,
      "margin_dpo/margin_mean": -0.11333887279033661,
      "margin_dpo/margin_std": 0.47709834575653076,
      "step": 8
    },
    {
      "epoch": 0.013215859030837005,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05302123725414276,
      "fcm_dpo/q_t": 0.4986753463745117,
      "grad_norm": 35.99601364135742,
      "learning_rate": 5.797101449275362e-08,
      "logits/chosen": 0.6641107797622681,
      "logits/rejected": 0.8435112237930298,
      "logps/chosen": -103.84231567382812,
      "logps/ref_chosen": -103.85713195800781,
      "logps/ref_rejected": -104.31932067871094,
      "logps/rejected": -104.35752868652344,
      "loss": 1.3815,
      "margin_dpo/margin_mean": 0.05302073061466217,
      "margin_dpo/margin_std": 0.44739070534706116,
      "step": 9
    },
    {
      "epoch": 0.014684287812041116,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07061505317687988,
      "fcm_dpo/q_t": 0.5017644762992859,
      "grad_norm": 33.54641342163086,
      "learning_rate": 6.521739130434782e-08,
      "logits/chosen": 0.49548810720443726,
      "logits/rejected": 0.7073336839675903,
      "logps/chosen": -76.18025207519531,
      "logps/ref_chosen": -76.20436096191406,
      "logps/ref_rejected": -87.152099609375,
      "logps/rejected": -87.057373046875,
      "loss": 1.3938,
      "margin_dpo/margin_mean": -0.07061484456062317,
      "margin_dpo/margin_std": 0.4113759398460388,
      "step": 10
    },
    {
      "epoch": 0.016152716593245228,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.044263288378715515,
      "fcm_dpo/q_t": 0.4988948404788971,
      "grad_norm": 36.77696990966797,
      "learning_rate": 7.246376811594203e-08,
      "logits/chosen": 0.4357748329639435,
      "logits/rejected": 0.659180223941803,
      "logps/chosen": -82.36294555664062,
      "logps/ref_chosen": -82.36650085449219,
      "logps/ref_rejected": -94.26461791992188,
      "logps/rejected": -94.30534362792969,
      "loss": 1.3826,
      "margin_dpo/margin_mean": 0.044263795018196106,
      "margin_dpo/margin_std": 0.5451699495315552,
      "step": 11
    },
    {
      "epoch": 0.01762114537444934,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11056771874427795,
      "fcm_dpo/q_t": 0.4972396492958069,
      "grad_norm": 39.76823043823242,
      "learning_rate": 7.971014492753623e-08,
      "logits/chosen": 0.24172449111938477,
      "logits/rejected": 0.3647180199623108,
      "logps/chosen": -99.05966186523438,
      "logps/ref_chosen": -99.10549926757812,
      "logps/ref_rejected": -110.27140808105469,
      "logps/rejected": -110.33615112304688,
      "loss": 1.376,
      "margin_dpo/margin_mean": 0.11056801676750183,
      "margin_dpo/margin_std": 0.5118380784988403,
      "step": 12
    },
    {
      "epoch": 0.01908957415565345,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0572160929441452,
      "fcm_dpo/q_t": 0.4985709488391876,
      "grad_norm": 41.4147834777832,
      "learning_rate": 8.695652173913042e-08,
      "logits/chosen": 0.5275347232818604,
      "logits/rejected": 0.7880017757415771,
      "logps/chosen": -90.50440216064453,
      "logps/ref_chosen": -90.55972290039062,
      "logps/ref_rejected": -93.69110107421875,
      "logps/rejected": -93.69297790527344,
      "loss": 1.3813,
      "margin_dpo/margin_mean": 0.05721601843833923,
      "margin_dpo/margin_std": 0.5135507583618164,
      "step": 13
    },
    {
      "epoch": 0.020558002936857563,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.08244989812374115,
      "fcm_dpo/q_t": 0.502057671546936,
      "grad_norm": 35.919612884521484,
      "learning_rate": 9.420289855072464e-08,
      "logits/chosen": 1.0716733932495117,
      "logits/rejected": 1.167722225189209,
      "logps/chosen": -99.8815689086914,
      "logps/ref_chosen": -99.82717895507812,
      "logps/ref_rejected": -108.94200134277344,
      "logps/rejected": -108.91395568847656,
      "loss": 1.3952,
      "margin_dpo/margin_mean": -0.0824495255947113,
      "margin_dpo/margin_std": 0.5182109475135803,
      "step": 14
    },
    {
      "epoch": 0.022026431718061675,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04384300112724304,
      "fcm_dpo/q_t": 0.49890750646591187,
      "grad_norm": 31.643028259277344,
      "learning_rate": 1.0144927536231885e-07,
      "logits/chosen": 0.3740409314632416,
      "logits/rejected": 0.5398809313774109,
      "logps/chosen": -78.9114990234375,
      "logps/ref_chosen": -78.90997314453125,
      "logps/ref_rejected": -90.06234741210938,
      "logps/rejected": -90.10771179199219,
      "loss": 1.3825,
      "margin_dpo/margin_mean": 0.043842852115631104,
      "margin_dpo/margin_std": 0.49486204981803894,
      "step": 15
    },
    {
      "epoch": 0.023494860499265784,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.020282208919525146,
      "fcm_dpo/q_t": 0.500505805015564,
      "grad_norm": 34.51702117919922,
      "learning_rate": 1.0869565217391303e-07,
      "logits/chosen": 0.6703887581825256,
      "logits/rejected": 0.9389725923538208,
      "logps/chosen": -97.39533996582031,
      "logps/ref_chosen": -97.42327880859375,
      "logps/ref_rejected": -90.59945678710938,
      "logps/rejected": -90.5512466430664,
      "loss": 1.3889,
      "margin_dpo/margin_mean": -0.02028217911720276,
      "margin_dpo/margin_std": 0.45394831895828247,
      "step": 16
    },
    {
      "epoch": 0.024963289280469897,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03842547535896301,
      "fcm_dpo/q_t": 0.4990399181842804,
      "grad_norm": 35.85000228881836,
      "learning_rate": 1.1594202898550725e-07,
      "logits/chosen": 0.5046379566192627,
      "logits/rejected": 0.7362222671508789,
      "logps/chosen": -104.31991577148438,
      "logps/ref_chosen": -104.36430358886719,
      "logps/ref_rejected": -90.46772766113281,
      "logps/rejected": -90.46174621582031,
      "loss": 1.3829,
      "margin_dpo/margin_mean": 0.03842410445213318,
      "margin_dpo/margin_std": 0.4408981204032898,
      "step": 17
    },
    {
      "epoch": 0.02643171806167401,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.004340857267379761,
      "fcm_dpo/q_t": 0.4998915195465088,
      "grad_norm": 41.64680099487305,
      "learning_rate": 1.2318840579710146e-07,
      "logits/chosen": 0.2849810719490051,
      "logits/rejected": 0.6180058121681213,
      "logps/chosen": -87.09352111816406,
      "logps/ref_chosen": -87.09195709228516,
      "logps/ref_rejected": -81.85072326660156,
      "logps/rejected": -81.85662841796875,
      "loss": 1.3863,
      "margin_dpo/margin_mean": 0.004341289401054382,
      "margin_dpo/margin_std": 0.3960126042366028,
      "step": 18
    },
    {
      "epoch": 0.027900146842878122,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.00902073085308075,
      "fcm_dpo/q_t": 0.49977564811706543,
      "grad_norm": 31.44943618774414,
      "learning_rate": 1.3043478260869563e-07,
      "logits/chosen": 0.725235104560852,
      "logits/rejected": 1.009577989578247,
      "logps/chosen": -105.87781524658203,
      "logps/ref_chosen": -105.87354278564453,
      "logps/ref_rejected": -96.93023681640625,
      "logps/rejected": -96.94352722167969,
      "loss": 1.3859,
      "margin_dpo/margin_mean": 0.009020596742630005,
      "margin_dpo/margin_std": 0.4446555972099304,
      "step": 19
    },
    {
      "epoch": 0.02936857562408223,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.004244685173034668,
      "fcm_dpo/q_t": 0.49989384412765503,
      "grad_norm": 32.87926483154297,
      "learning_rate": 1.3768115942028986e-07,
      "logits/chosen": 0.3713058829307556,
      "logits/rejected": 0.6612204909324646,
      "logps/chosen": -90.76942443847656,
      "logps/ref_chosen": -90.75811767578125,
      "logps/ref_rejected": -85.91232299804688,
      "logps/rejected": -85.9278564453125,
      "loss": 1.3862,
      "margin_dpo/margin_mean": 0.0042449235916137695,
      "margin_dpo/margin_std": 0.38462841510772705,
      "step": 20
    },
    {
      "epoch": 0.030837004405286344,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12687450647354126,
      "fcm_dpo/q_t": 0.4968298673629761,
      "grad_norm": 31.369789123535156,
      "learning_rate": 1.4492753623188405e-07,
      "logits/chosen": 0.7661827802658081,
      "logits/rejected": 0.9624724984169006,
      "logps/chosen": -80.27702331542969,
      "logps/ref_chosen": -80.33345794677734,
      "logps/ref_rejected": -83.93373107910156,
      "logps/rejected": -84.00416564941406,
      "loss": 1.374,
      "margin_dpo/margin_mean": 0.12687447667121887,
      "margin_dpo/margin_std": 0.39177459478378296,
      "step": 21
    },
    {
      "epoch": 0.032305433186490456,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11361062526702881,
      "fcm_dpo/q_t": 0.4971611797809601,
      "grad_norm": 42.44084548950195,
      "learning_rate": 1.5217391304347825e-07,
      "logits/chosen": 0.5324074029922485,
      "logits/rejected": 0.6477072238922119,
      "logps/chosen": -95.31715393066406,
      "logps/ref_chosen": -95.39530181884766,
      "logps/ref_rejected": -103.47351837158203,
      "logps/rejected": -103.50899505615234,
      "loss": 1.3754,
      "margin_dpo/margin_mean": 0.11361068487167358,
      "margin_dpo/margin_std": 0.3894934058189392,
      "step": 22
    },
    {
      "epoch": 0.033773861967694566,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.024963736534118652,
      "fcm_dpo/q_t": 0.499376118183136,
      "grad_norm": 31.390506744384766,
      "learning_rate": 1.5942028985507245e-07,
      "logits/chosen": 0.7837526202201843,
      "logits/rejected": 0.9659498333930969,
      "logps/chosen": -90.59944152832031,
      "logps/ref_chosen": -90.63751220703125,
      "logps/ref_rejected": -86.59425354003906,
      "logps/rejected": -86.5811538696289,
      "loss": 1.3842,
      "margin_dpo/margin_mean": 0.02496352791786194,
      "margin_dpo/margin_std": 0.3982018232345581,
      "step": 23
    },
    {
      "epoch": 0.03524229074889868,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.012234330177307129,
      "fcm_dpo/q_t": 0.4996940493583679,
      "grad_norm": 44.296112060546875,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 0.667069137096405,
      "logits/rejected": 0.6968539953231812,
      "logps/chosen": -69.93962097167969,
      "logps/ref_chosen": -69.91728973388672,
      "logps/ref_rejected": -106.60990142822266,
      "logps/rejected": -106.64446258544922,
      "loss": 1.3855,
      "margin_dpo/margin_mean": 0.012234941124916077,
      "margin_dpo/margin_std": 0.43183019757270813,
      "step": 24
    },
    {
      "epoch": 0.03671071953010279,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06985443830490112,
      "fcm_dpo/q_t": 0.4982590675354004,
      "grad_norm": 36.90232849121094,
      "learning_rate": 1.7391304347826085e-07,
      "logits/chosen": 0.5097548365592957,
      "logits/rejected": 0.6774953007698059,
      "logps/chosen": -80.84941101074219,
      "logps/ref_chosen": -80.82548522949219,
      "logps/ref_rejected": -95.95710754394531,
      "logps/rejected": -96.05089569091797,
      "loss": 1.38,
      "margin_dpo/margin_mean": 0.06985485553741455,
      "margin_dpo/margin_std": 0.5077825784683228,
      "step": 25
    },
    {
      "epoch": 0.0381791483113069,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05025067925453186,
      "fcm_dpo/q_t": 0.49874305725097656,
      "grad_norm": 41.10307693481445,
      "learning_rate": 1.8115942028985507e-07,
      "logits/chosen": 0.1814301759004593,
      "logits/rejected": 0.38568517565727234,
      "logps/chosen": -88.87095642089844,
      "logps/ref_chosen": -88.90116882324219,
      "logps/ref_rejected": -109.82819366455078,
      "logps/rejected": -109.84822845458984,
      "loss": 1.3817,
      "margin_dpo/margin_mean": 0.05025133490562439,
      "margin_dpo/margin_std": 0.412225604057312,
      "step": 26
    },
    {
      "epoch": 0.039647577092511016,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07869970798492432,
      "fcm_dpo/q_t": 0.4980340600013733,
      "grad_norm": 44.32426071166992,
      "learning_rate": 1.8840579710144927e-07,
      "logits/chosen": 0.33394181728363037,
      "logits/rejected": 0.5241584777832031,
      "logps/chosen": -77.59822082519531,
      "logps/ref_chosen": -77.59600830078125,
      "logps/ref_rejected": -103.93850708007812,
      "logps/rejected": -104.01941680908203,
      "loss": 1.379,
      "margin_dpo/margin_mean": 0.07869991660118103,
      "margin_dpo/margin_std": 0.4667752981185913,
      "step": 27
    },
    {
      "epoch": 0.041116005873715125,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06924325227737427,
      "fcm_dpo/q_t": 0.4982706308364868,
      "grad_norm": 36.54985809326172,
      "learning_rate": 1.9565217391304347e-07,
      "logits/chosen": 0.5311095714569092,
      "logits/rejected": 0.7446248531341553,
      "logps/chosen": -102.18016815185547,
      "logps/ref_chosen": -102.22856903076172,
      "logps/ref_rejected": -96.9594955444336,
      "logps/rejected": -96.98033142089844,
      "loss": 1.38,
      "margin_dpo/margin_mean": 0.0692436695098877,
      "margin_dpo/margin_std": 0.47658565640449524,
      "step": 28
    },
    {
      "epoch": 0.042584434654919234,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16378992795944214,
      "fcm_dpo/q_t": 0.49590858817100525,
      "grad_norm": 42.102882385253906,
      "learning_rate": 2.028985507246377e-07,
      "logits/chosen": 0.20089560747146606,
      "logits/rejected": 0.450194776058197,
      "logps/chosen": -88.62367248535156,
      "logps/ref_chosen": -88.64704895019531,
      "logps/ref_rejected": -102.96011352539062,
      "logps/rejected": -103.10052490234375,
      "loss": 1.3705,
      "margin_dpo/margin_mean": 0.1637905091047287,
      "margin_dpo/margin_std": 0.46207696199417114,
      "step": 29
    },
    {
      "epoch": 0.04405286343612335,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13101083040237427,
      "fcm_dpo/q_t": 0.49672505259513855,
      "grad_norm": 39.2225341796875,
      "learning_rate": 2.1014492753623187e-07,
      "logits/chosen": 0.5814670324325562,
      "logits/rejected": 0.6965478658676147,
      "logps/chosen": -88.37109375,
      "logps/ref_chosen": -88.38838195800781,
      "logps/ref_rejected": -102.31889343261719,
      "logps/rejected": -102.4326171875,
      "loss": 1.3736,
      "margin_dpo/margin_mean": 0.13101181387901306,
      "margin_dpo/margin_std": 0.39710086584091187,
      "step": 30
    },
    {
      "epoch": 0.04552129221732746,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11164575815200806,
      "fcm_dpo/q_t": 0.4972113370895386,
      "grad_norm": 31.29022216796875,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": 0.6445361971855164,
      "logits/rejected": 0.8826954960823059,
      "logps/chosen": -101.128662109375,
      "logps/ref_chosen": -101.12565612792969,
      "logps/ref_rejected": -79.8584213256836,
      "logps/rejected": -79.97307586669922,
      "loss": 1.3755,
      "margin_dpo/margin_mean": 0.11164584755897522,
      "margin_dpo/margin_std": 0.38801681995391846,
      "step": 31
    },
    {
      "epoch": 0.04698972099853157,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08825841546058655,
      "fcm_dpo/q_t": 0.4977962076663971,
      "grad_norm": 35.21088790893555,
      "learning_rate": 2.2463768115942027e-07,
      "logits/chosen": 0.5731779932975769,
      "logits/rejected": 0.7231494188308716,
      "logps/chosen": -96.67442321777344,
      "logps/ref_chosen": -96.61702728271484,
      "logps/ref_rejected": -96.224365234375,
      "logps/rejected": -96.3700180053711,
      "loss": 1.3781,
      "margin_dpo/margin_mean": 0.0882592499256134,
      "margin_dpo/margin_std": 0.4722122251987457,
      "step": 32
    },
    {
      "epoch": 0.048458149779735685,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1351313441991806,
      "fcm_dpo/q_t": 0.49662476778030396,
      "grad_norm": 33.66917037963867,
      "learning_rate": 2.318840579710145e-07,
      "logits/chosen": 0.6991294026374817,
      "logits/rejected": 0.8825780749320984,
      "logps/chosen": -81.51029968261719,
      "logps/ref_chosen": -81.52101135253906,
      "logps/ref_rejected": -93.80595397949219,
      "logps/rejected": -93.93037414550781,
      "loss": 1.3735,
      "margin_dpo/margin_mean": 0.13513219356536865,
      "margin_dpo/margin_std": 0.4900803565979004,
      "step": 33
    },
    {
      "epoch": 0.049926578560939794,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1415727734565735,
      "fcm_dpo/q_t": 0.4964648485183716,
      "grad_norm": 41.019248962402344,
      "learning_rate": 2.391304347826087e-07,
      "logits/chosen": 0.5307381749153137,
      "logits/rejected": 0.7028604745864868,
      "logps/chosen": -77.18989562988281,
      "logps/ref_chosen": -77.20204162597656,
      "logps/ref_rejected": -106.71876525878906,
      "logps/rejected": -106.84819030761719,
      "loss": 1.3728,
      "margin_dpo/margin_mean": 0.14157328009605408,
      "margin_dpo/margin_std": 0.4783652722835541,
      "step": 34
    },
    {
      "epoch": 0.0513950073421439,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16865116357803345,
      "fcm_dpo/q_t": 0.4957859516143799,
      "grad_norm": 42.815826416015625,
      "learning_rate": 2.463768115942029e-07,
      "logits/chosen": 0.34624242782592773,
      "logits/rejected": 0.5021581053733826,
      "logps/chosen": -77.58589935302734,
      "logps/ref_chosen": -77.57035064697266,
      "logps/ref_rejected": -112.18855285644531,
      "logps/rejected": -112.37275695800781,
      "loss": 1.37,
      "margin_dpo/margin_mean": 0.1686515212059021,
      "margin_dpo/margin_std": 0.45406052470207214,
      "step": 35
    },
    {
      "epoch": 0.05286343612334802,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12370005249977112,
      "fcm_dpo/q_t": 0.4969105124473572,
      "grad_norm": 32.699363708496094,
      "learning_rate": 2.536231884057971e-07,
      "logits/chosen": 0.5115445852279663,
      "logits/rejected": 0.7059195041656494,
      "logps/chosen": -83.30363464355469,
      "logps/ref_chosen": -83.28823852539062,
      "logps/ref_rejected": -91.75741577148438,
      "logps/rejected": -91.89651489257812,
      "loss": 1.3746,
      "margin_dpo/margin_mean": 0.12370041012763977,
      "margin_dpo/margin_std": 0.4851023554801941,
      "step": 36
    },
    {
      "epoch": 0.05433186490455213,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2637890577316284,
      "fcm_dpo/q_t": 0.4934196472167969,
      "grad_norm": 35.98224639892578,
      "learning_rate": 2.6086956521739126e-07,
      "logits/chosen": 0.6918505430221558,
      "logits/rejected": 0.8543441295623779,
      "logps/chosen": -94.75133514404297,
      "logps/ref_chosen": -94.77108764648438,
      "logps/ref_rejected": -85.71726989746094,
      "logps/rejected": -85.9613037109375,
      "loss": 1.3611,
      "margin_dpo/margin_mean": 0.2637891173362732,
      "margin_dpo/margin_std": 0.6317564845085144,
      "step": 37
    },
    {
      "epoch": 0.055800293685756244,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.30049073696136475,
      "fcm_dpo/q_t": 0.49250102043151855,
      "grad_norm": 36.850074768066406,
      "learning_rate": 2.681159420289855e-07,
      "logits/chosen": 0.4852568805217743,
      "logits/rejected": 0.624308168888092,
      "logps/chosen": -75.86715698242188,
      "logps/ref_chosen": -75.97850036621094,
      "logps/ref_rejected": -104.10401916503906,
      "logps/rejected": -104.29315948486328,
      "loss": 1.3573,
      "margin_dpo/margin_mean": 0.3004909157752991,
      "margin_dpo/margin_std": 0.5791944861412048,
      "step": 38
    },
    {
      "epoch": 0.05726872246696035,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2556076645851135,
      "fcm_dpo/q_t": 0.4936175048351288,
      "grad_norm": 36.4305534362793,
      "learning_rate": 2.753623188405797e-07,
      "logits/chosen": 0.5048327445983887,
      "logits/rejected": 0.6937540769577026,
      "logps/chosen": -81.26030731201172,
      "logps/ref_chosen": -81.18577575683594,
      "logps/ref_rejected": -84.1959228515625,
      "logps/rejected": -84.52605438232422,
      "loss": 1.3617,
      "margin_dpo/margin_mean": 0.25560712814331055,
      "margin_dpo/margin_std": 0.5651835203170776,
      "step": 39
    },
    {
      "epoch": 0.05873715124816446,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2306583821773529,
      "fcm_dpo/q_t": 0.49423885345458984,
      "grad_norm": 29.03916358947754,
      "learning_rate": 2.8260869565217386e-07,
      "logits/chosen": 0.6887877583503723,
      "logits/rejected": 0.8908544182777405,
      "logps/chosen": -83.27583312988281,
      "logps/ref_chosen": -83.33257293701172,
      "logps/ref_rejected": -80.25591278076172,
      "logps/rejected": -80.42982482910156,
      "loss": 1.3639,
      "margin_dpo/margin_mean": 0.23065805435180664,
      "margin_dpo/margin_std": 0.4786723256111145,
      "step": 40
    },
    {
      "epoch": 0.06020558002936858,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.25650691986083984,
      "fcm_dpo/q_t": 0.4935963749885559,
      "grad_norm": 33.375675201416016,
      "learning_rate": 2.898550724637681e-07,
      "logits/chosen": 0.6165636777877808,
      "logits/rejected": 0.8028247356414795,
      "logps/chosen": -93.20872497558594,
      "logps/ref_chosen": -93.14866638183594,
      "logps/ref_rejected": -102.07920837402344,
      "logps/rejected": -102.39578247070312,
      "loss": 1.3619,
      "margin_dpo/margin_mean": 0.25650691986083984,
      "margin_dpo/margin_std": 0.6334136128425598,
      "step": 41
    },
    {
      "epoch": 0.06167400881057269,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.41214534640312195,
      "fcm_dpo/q_t": 0.48970967531204224,
      "grad_norm": 40.96079635620117,
      "learning_rate": 2.971014492753623e-07,
      "logits/chosen": 0.2511819899082184,
      "logits/rejected": 0.39002329111099243,
      "logps/chosen": -90.65541076660156,
      "logps/ref_chosen": -90.70162200927734,
      "logps/ref_rejected": -113.8646469116211,
      "logps/rejected": -114.23058319091797,
      "loss": 1.3463,
      "margin_dpo/margin_mean": 0.4121454060077667,
      "margin_dpo/margin_std": 0.5678550601005554,
      "step": 42
    },
    {
      "epoch": 0.0631424375917768,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.35536667704582214,
      "fcm_dpo/q_t": 0.4911324381828308,
      "grad_norm": 38.216575622558594,
      "learning_rate": 3.043478260869565e-07,
      "logits/chosen": 0.7120174765586853,
      "logits/rejected": 0.9082381725311279,
      "logps/chosen": -89.6702880859375,
      "logps/ref_chosen": -89.64402770996094,
      "logps/ref_rejected": -103.95185852050781,
      "logps/rejected": -104.33348083496094,
      "loss": 1.3522,
      "margin_dpo/margin_mean": 0.3553672134876251,
      "margin_dpo/margin_std": 0.6629294157028198,
      "step": 43
    },
    {
      "epoch": 0.06461086637298091,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.344689279794693,
      "fcm_dpo/q_t": 0.49139606952667236,
      "grad_norm": 34.39558792114258,
      "learning_rate": 3.115942028985507e-07,
      "logits/chosen": 0.9785443544387817,
      "logits/rejected": 1.0956333875656128,
      "logps/chosen": -81.92916870117188,
      "logps/ref_chosen": -81.8783187866211,
      "logps/ref_rejected": -113.421630859375,
      "logps/rejected": -113.81716918945312,
      "loss": 1.3531,
      "margin_dpo/margin_mean": 0.34468963742256165,
      "margin_dpo/margin_std": 0.6063359975814819,
      "step": 44
    },
    {
      "epoch": 0.06607929515418502,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.20663021504878998,
      "fcm_dpo/q_t": 0.49483978748321533,
      "grad_norm": 24.50741195678711,
      "learning_rate": 3.188405797101449e-07,
      "logits/chosen": 0.5508902072906494,
      "logits/rejected": 0.6601569056510925,
      "logps/chosen": -77.46229553222656,
      "logps/ref_chosen": -77.34459686279297,
      "logps/ref_rejected": -84.24774169921875,
      "logps/rejected": -84.57206726074219,
      "loss": 1.3666,
      "margin_dpo/margin_mean": 0.20663078129291534,
      "margin_dpo/margin_std": 0.5758439302444458,
      "step": 45
    },
    {
      "epoch": 0.06754772393538913,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5048208236694336,
      "fcm_dpo/q_t": 0.48739558458328247,
      "grad_norm": 33.225318908691406,
      "learning_rate": 3.260869565217391e-07,
      "logits/chosen": 0.35200345516204834,
      "logits/rejected": 0.5865339040756226,
      "logps/chosen": -90.34577941894531,
      "logps/ref_chosen": -90.3408203125,
      "logps/ref_rejected": -100.676513671875,
      "logps/rejected": -101.18629455566406,
      "loss": 1.3374,
      "margin_dpo/margin_mean": 0.5048205852508545,
      "margin_dpo/margin_std": 0.6096693873405457,
      "step": 46
    },
    {
      "epoch": 0.06901615271659324,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.35532331466674805,
      "fcm_dpo/q_t": 0.4911317825317383,
      "grad_norm": 34.18780517578125,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 0.5468966960906982,
      "logits/rejected": 0.7348242402076721,
      "logps/chosen": -104.42884826660156,
      "logps/ref_chosen": -104.41130065917969,
      "logps/ref_rejected": -101.25489807128906,
      "logps/rejected": -101.62776947021484,
      "loss": 1.3522,
      "margin_dpo/margin_mean": 0.3553228974342346,
      "margin_dpo/margin_std": 0.6635472178459167,
      "step": 47
    },
    {
      "epoch": 0.07048458149779736,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5949963927268982,
      "fcm_dpo/q_t": 0.485162615776062,
      "grad_norm": 38.76776123046875,
      "learning_rate": 3.4057971014492755e-07,
      "logits/chosen": 0.31125134229660034,
      "logits/rejected": 0.5184098482131958,
      "logps/chosen": -91.11347961425781,
      "logps/ref_chosen": -91.10027313232422,
      "logps/ref_rejected": -95.08058166503906,
      "logps/rejected": -95.68878173828125,
      "loss": 1.3294,
      "margin_dpo/margin_mean": 0.5949971079826355,
      "margin_dpo/margin_std": 0.8412452936172485,
      "step": 48
    },
    {
      "epoch": 0.07195301027900147,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8068197965621948,
      "fcm_dpo/q_t": 0.4799154996871948,
      "grad_norm": 40.11665344238281,
      "learning_rate": 3.478260869565217e-07,
      "logits/chosen": 0.042669445276260376,
      "logits/rejected": 0.5065343379974365,
      "logps/chosen": -92.9081802368164,
      "logps/ref_chosen": -93.00367736816406,
      "logps/ref_rejected": -91.74899291992188,
      "logps/rejected": -92.46031188964844,
      "loss": 1.3099,
      "margin_dpo/margin_mean": 0.8068201541900635,
      "margin_dpo/margin_std": 1.0258755683898926,
      "step": 49
    },
    {
      "epoch": 0.07342143906020558,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.646321177482605,
      "fcm_dpo/q_t": 0.4838915765285492,
      "grad_norm": 33.23332595825195,
      "learning_rate": 3.5507246376811595e-07,
      "logits/chosen": 0.6108515858650208,
      "logits/rejected": 0.7840231657028198,
      "logps/chosen": -94.68785095214844,
      "logps/ref_chosen": -94.62681579589844,
      "logps/ref_rejected": -103.57435607910156,
      "logps/rejected": -104.28172302246094,
      "loss": 1.3248,
      "margin_dpo/margin_mean": 0.6463215947151184,
      "margin_dpo/margin_std": 0.9034823179244995,
      "step": 50
    },
    {
      "epoch": 0.07488986784140969,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6294625997543335,
      "fcm_dpo/q_t": 0.48433637619018555,
      "grad_norm": 30.03205680847168,
      "learning_rate": 3.6231884057971015e-07,
      "logits/chosen": 0.351556658744812,
      "logits/rejected": 0.47928231954574585,
      "logps/chosen": -87.57754516601562,
      "logps/ref_chosen": -87.50727844238281,
      "logps/ref_rejected": -83.47235107421875,
      "logps/rejected": -84.17208862304688,
      "loss": 1.3273,
      "margin_dpo/margin_mean": 0.6294622421264648,
      "margin_dpo/margin_std": 1.0819957256317139,
      "step": 51
    },
    {
      "epoch": 0.0763582966226138,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7756294012069702,
      "fcm_dpo/q_t": 0.4806681275367737,
      "grad_norm": 32.24178695678711,
      "learning_rate": 3.695652173913043e-07,
      "logits/chosen": 0.47563832998275757,
      "logits/rejected": 0.6913837194442749,
      "logps/chosen": -90.67118835449219,
      "logps/ref_chosen": -90.63026428222656,
      "logps/ref_rejected": -87.0390625,
      "logps/rejected": -87.85562133789062,
      "loss": 1.3121,
      "margin_dpo/margin_mean": 0.7756293416023254,
      "margin_dpo/margin_std": 0.8545148968696594,
      "step": 52
    },
    {
      "epoch": 0.07782672540381791,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0988894701004028,
      "fcm_dpo/q_t": 0.47282397747039795,
      "grad_norm": 39.16925048828125,
      "learning_rate": 3.7681159420289855e-07,
      "logits/chosen": 0.2351905107498169,
      "logits/rejected": 0.35199305415153503,
      "logps/chosen": -81.59117126464844,
      "logps/ref_chosen": -81.58307647705078,
      "logps/ref_rejected": -95.66152954101562,
      "logps/rejected": -96.76851654052734,
      "loss": 1.2849,
      "margin_dpo/margin_mean": 1.0988894701004028,
      "margin_dpo/margin_std": 1.4093308448791504,
      "step": 53
    },
    {
      "epoch": 0.07929515418502203,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.159604787826538,
      "fcm_dpo/q_t": 0.4712783694267273,
      "grad_norm": 39.55070877075195,
      "learning_rate": 3.8405797101449274e-07,
      "logits/chosen": 0.44965824484825134,
      "logits/rejected": 0.6323707699775696,
      "logps/chosen": -88.9405288696289,
      "logps/ref_chosen": -88.91017150878906,
      "logps/ref_rejected": -99.1175537109375,
      "logps/rejected": -100.30752563476562,
      "loss": 1.2788,
      "margin_dpo/margin_mean": 1.159604787826538,
      "margin_dpo/margin_std": 1.439812421798706,
      "step": 54
    },
    {
      "epoch": 0.08076358296622614,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.234811782836914,
      "fcm_dpo/q_t": 0.46946150064468384,
      "grad_norm": 35.18501663208008,
      "learning_rate": 3.9130434782608694e-07,
      "logits/chosen": 0.10041927546262741,
      "logits/rejected": 0.40900927782058716,
      "logps/chosen": -92.3023910522461,
      "logps/ref_chosen": -92.45591735839844,
      "logps/ref_rejected": -92.97093200683594,
      "logps/rejected": -94.05221557617188,
      "loss": 1.273,
      "margin_dpo/margin_mean": 1.2348113059997559,
      "margin_dpo/margin_std": 1.5631911754608154,
      "step": 55
    },
    {
      "epoch": 0.08223201174743025,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0258830785751343,
      "fcm_dpo/q_t": 0.47462257742881775,
      "grad_norm": 33.30818176269531,
      "learning_rate": 3.9855072463768114e-07,
      "logits/chosen": 0.2586688697338104,
      "logits/rejected": 0.6011684536933899,
      "logps/chosen": -87.36018371582031,
      "logps/ref_chosen": -87.23665618896484,
      "logps/ref_rejected": -100.64553833007812,
      "logps/rejected": -101.79496002197266,
      "loss": 1.2928,
      "margin_dpo/margin_mean": 1.0258824825286865,
      "margin_dpo/margin_std": 1.6051685810089111,
      "step": 56
    },
    {
      "epoch": 0.08370044052863436,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1884700059890747,
      "fcm_dpo/q_t": 0.47055989503860474,
      "grad_norm": 33.485557556152344,
      "learning_rate": 4.057971014492754e-07,
      "logits/chosen": 0.29828542470932007,
      "logits/rejected": 0.5136665105819702,
      "logps/chosen": -98.16508483886719,
      "logps/ref_chosen": -98.15074157714844,
      "logps/ref_rejected": -101.05284118652344,
      "logps/rejected": -102.25564575195312,
      "loss": 1.2775,
      "margin_dpo/margin_mean": 1.1884701251983643,
      "margin_dpo/margin_std": 1.6172009706497192,
      "step": 57
    },
    {
      "epoch": 0.08516886930983847,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5477666854858398,
      "fcm_dpo/q_t": 0.46193379163742065,
      "grad_norm": 37.840396881103516,
      "learning_rate": 4.1304347826086954e-07,
      "logits/chosen": 0.1909644901752472,
      "logits/rejected": 0.517054557800293,
      "logps/chosen": -99.33880615234375,
      "logps/ref_chosen": -99.58097076416016,
      "logps/ref_rejected": -91.22227478027344,
      "logps/rejected": -92.52787780761719,
      "loss": 1.2468,
      "margin_dpo/margin_mean": 1.547766923904419,
      "margin_dpo/margin_std": 1.863322138786316,
      "step": 58
    },
    {
      "epoch": 0.08663729809104258,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1774803400039673,
      "fcm_dpo/q_t": 0.47080308198928833,
      "grad_norm": 34.35539245605469,
      "learning_rate": 4.2028985507246374e-07,
      "logits/chosen": 0.21893709897994995,
      "logits/rejected": 0.41581693291664124,
      "logps/chosen": -89.77354431152344,
      "logps/ref_chosen": -89.80232238769531,
      "logps/ref_rejected": -94.27667236328125,
      "logps/rejected": -95.42537689208984,
      "loss": 1.2771,
      "margin_dpo/margin_mean": 1.1774799823760986,
      "margin_dpo/margin_std": 1.4213714599609375,
      "step": 59
    },
    {
      "epoch": 0.0881057268722467,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2050294876098633,
      "fcm_dpo/q_t": 0.47011345624923706,
      "grad_norm": 28.579572677612305,
      "learning_rate": 4.2753623188405794e-07,
      "logits/chosen": 0.21510683000087738,
      "logits/rejected": 0.47372835874557495,
      "logps/chosen": -95.11346435546875,
      "logps/ref_chosen": -95.15571594238281,
      "logps/ref_rejected": -91.3724365234375,
      "logps/rejected": -92.53521728515625,
      "loss": 1.2744,
      "margin_dpo/margin_mean": 1.2050297260284424,
      "margin_dpo/margin_std": 1.4122042655944824,
      "step": 60
    },
    {
      "epoch": 0.08957415565345081,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0777596235275269,
      "fcm_dpo/q_t": 0.47358256578445435,
      "grad_norm": 28.4328670501709,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": 0.15031884610652924,
      "logits/rejected": 0.28975626826286316,
      "logps/chosen": -85.95419311523438,
      "logps/ref_chosen": -85.73230743408203,
      "logps/ref_rejected": -97.96575927734375,
      "logps/rejected": -99.26539611816406,
      "loss": 1.2898,
      "margin_dpo/margin_mean": 1.0777602195739746,
      "margin_dpo/margin_std": 1.7849442958831787,
      "step": 61
    },
    {
      "epoch": 0.09104258443465492,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.185563325881958,
      "fcm_dpo/q_t": 0.4706765115261078,
      "grad_norm": 28.67509651184082,
      "learning_rate": 4.420289855072464e-07,
      "logits/chosen": 0.3115650415420532,
      "logits/rejected": 0.5142878890037537,
      "logps/chosen": -81.77017211914062,
      "logps/ref_chosen": -81.63538360595703,
      "logps/ref_rejected": -84.03831481933594,
      "logps/rejected": -85.3586654663086,
      "loss": 1.2803,
      "margin_dpo/margin_mean": 1.1855626106262207,
      "margin_dpo/margin_std": 1.801519751548767,
      "step": 62
    },
    {
      "epoch": 0.09251101321585903,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5173454284667969,
      "fcm_dpo/q_t": 0.4624067544937134,
      "grad_norm": 35.08026123046875,
      "learning_rate": 4.4927536231884053e-07,
      "logits/chosen": 0.1734933853149414,
      "logits/rejected": 0.3734012246131897,
      "logps/chosen": -103.64640808105469,
      "logps/ref_chosen": -103.62405395507812,
      "logps/ref_rejected": -103.40303039550781,
      "logps/rejected": -104.94271850585938,
      "loss": 1.2463,
      "margin_dpo/margin_mean": 1.5173449516296387,
      "margin_dpo/margin_std": 1.5217807292938232,
      "step": 63
    },
    {
      "epoch": 0.09397944199706314,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6387507915496826,
      "fcm_dpo/q_t": 0.4597811698913574,
      "grad_norm": 35.231178283691406,
      "learning_rate": 4.5652173913043473e-07,
      "logits/chosen": 0.6302533149719238,
      "logits/rejected": 0.8535206913948059,
      "logps/chosen": -87.16665649414062,
      "logps/ref_chosen": -87.0015869140625,
      "logps/ref_rejected": -100.5854721069336,
      "logps/rejected": -102.38929748535156,
      "loss": 1.2402,
      "margin_dpo/margin_mean": 1.638750672340393,
      "margin_dpo/margin_std": 2.034325122833252,
      "step": 64
    },
    {
      "epoch": 0.09544787077826726,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7660255432128906,
      "fcm_dpo/q_t": 0.45665818452835083,
      "grad_norm": 38.46485137939453,
      "learning_rate": 4.63768115942029e-07,
      "logits/chosen": 0.02439071610569954,
      "logits/rejected": 0.16916462779045105,
      "logps/chosen": -91.31500244140625,
      "logps/ref_chosen": -91.22191619873047,
      "logps/ref_rejected": -115.33553314208984,
      "logps/rejected": -117.19464874267578,
      "loss": 1.2282,
      "margin_dpo/margin_mean": 1.7660255432128906,
      "margin_dpo/margin_std": 2.0794177055358887,
      "step": 65
    },
    {
      "epoch": 0.09691629955947137,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1536918878555298,
      "fcm_dpo/q_t": 0.4715958833694458,
      "grad_norm": 25.735258102416992,
      "learning_rate": 4.7101449275362313e-07,
      "logits/chosen": 0.2745053470134735,
      "logits/rejected": 0.48479628562927246,
      "logps/chosen": -84.15156555175781,
      "logps/ref_chosen": -83.78422546386719,
      "logps/ref_rejected": -82.7520980834961,
      "logps/rejected": -84.27313232421875,
      "loss": 1.2838,
      "margin_dpo/margin_mean": 1.1536920070648193,
      "margin_dpo/margin_std": 1.8838276863098145,
      "step": 66
    },
    {
      "epoch": 0.09838472834067548,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0333654880523682,
      "fcm_dpo/q_t": 0.47503939270973206,
      "grad_norm": 24.520505905151367,
      "learning_rate": 4.782608695652174e-07,
      "logits/chosen": 0.3828052282333374,
      "logits/rejected": 0.6662817001342773,
      "logps/chosen": -88.6138916015625,
      "logps/ref_chosen": -87.67295837402344,
      "logps/ref_rejected": -79.08674621582031,
      "logps/rejected": -81.06103515625,
      "loss": 1.3044,
      "margin_dpo/margin_mean": 1.0333654880523682,
      "margin_dpo/margin_std": 2.741905689239502,
      "step": 67
    },
    {
      "epoch": 0.09985315712187959,
      "fcm_dpo/beta": 0.1100722998380661,
      "fcm_dpo/delta": 0.6364251971244812,
      "fcm_dpo/margin": 2.0699269771575928,
      "fcm_dpo/q_t": 0.4486227035522461,
      "grad_norm": 30.40932273864746,
      "learning_rate": 4.855072463768116e-07,
      "logits/chosen": -0.0022230129688978195,
      "logits/rejected": 0.29651695489883423,
      "logps/chosen": -97.31098937988281,
      "logps/ref_chosen": -97.07884216308594,
      "logps/ref_rejected": -84.11872863769531,
      "logps/rejected": -86.4207992553711,
      "loss": 1.1941,
      "margin_dpo/margin_mean": 2.0699267387390137,
      "margin_dpo/margin_std": 2.751864433288574,
      "step": 68
    },
    {
      "epoch": 0.1013215859030837,
      "fcm_dpo/beta": 0.12420888990163803,
      "fcm_dpo/delta": 0.5869454145431519,
      "fcm_dpo/margin": 2.242255210876465,
      "fcm_dpo/q_t": 0.43765681982040405,
      "grad_norm": 35.120948791503906,
      "learning_rate": 4.927536231884058e-07,
      "logits/chosen": 0.10133904963731766,
      "logits/rejected": 0.21260981261730194,
      "logps/chosen": -85.975341796875,
      "logps/ref_chosen": -85.71971130371094,
      "logps/ref_rejected": -109.48020935058594,
      "logps/rejected": -111.97808837890625,
      "loss": 1.1604,
      "margin_dpo/margin_mean": 2.242255210876465,
      "margin_dpo/margin_std": 3.020266056060791,
      "step": 69
    },
    {
      "epoch": 0.1027900146842878,
      "fcm_dpo/beta": 0.13789792358875275,
      "fcm_dpo/delta": 0.5037232041358948,
      "fcm_dpo/margin": 2.6392881870269775,
      "fcm_dpo/q_t": 0.42064177989959717,
      "grad_norm": 35.125850677490234,
      "learning_rate": 5e-07,
      "logits/chosen": 0.376822829246521,
      "logits/rejected": 0.5594728589057922,
      "logps/chosen": -95.60337829589844,
      "logps/ref_chosen": -95.00994873046875,
      "logps/ref_rejected": -96.21272277832031,
      "logps/rejected": -99.44544219970703,
      "loss": 1.1087,
      "margin_dpo/margin_mean": 2.6392879486083984,
      "margin_dpo/margin_std": 3.4235267639160156,
      "step": 70
    },
    {
      "epoch": 0.10425844346549193,
      "fcm_dpo/beta": 0.14798226952552795,
      "fcm_dpo/delta": 0.3334314823150635,
      "fcm_dpo/margin": 3.616886854171753,
      "fcm_dpo/q_t": 0.38527965545654297,
      "grad_norm": 42.508155822753906,
      "learning_rate": 4.999967061337492e-07,
      "logits/chosen": -0.3210897445678711,
      "logits/rejected": 0.03221406042575836,
      "logps/chosen": -91.05834197998047,
      "logps/ref_chosen": -90.97735595703125,
      "logps/ref_rejected": -102.59103393554688,
      "logps/rejected": -106.28890991210938,
      "loss": 0.9942,
      "margin_dpo/margin_mean": 3.616886615753174,
      "margin_dpo/margin_std": 3.979628801345825,
      "step": 71
    },
    {
      "epoch": 0.10572687224669604,
      "fcm_dpo/beta": 0.15995782613754272,
      "fcm_dpo/delta": 0.3947334587574005,
      "fcm_dpo/margin": 2.966139316558838,
      "fcm_dpo/q_t": 0.39579302072525024,
      "grad_norm": 38.812652587890625,
      "learning_rate": 4.999868246217933e-07,
      "logits/chosen": -0.07235760986804962,
      "logits/rejected": 0.2604253590106964,
      "logps/chosen": -98.39273071289062,
      "logps/ref_chosen": -97.89152526855469,
      "logps/ref_rejected": -100.19171142578125,
      "logps/rejected": -103.6590576171875,
      "loss": 1.0302,
      "margin_dpo/margin_mean": 2.966139316558838,
      "margin_dpo/margin_std": 3.2803831100463867,
      "step": 72
    },
    {
      "epoch": 0.10719530102790015,
      "fcm_dpo/beta": 0.17192596197128296,
      "fcm_dpo/delta": 0.3318787217140198,
      "fcm_dpo/margin": 3.1142077445983887,
      "fcm_dpo/q_t": 0.3958582282066345,
      "grad_norm": 38.59789276123047,
      "learning_rate": 4.999703557245192e-07,
      "logits/chosen": -0.3595370054244995,
      "logits/rejected": -0.05686055123806,
      "logps/chosen": -96.527099609375,
      "logps/ref_chosen": -95.7690200805664,
      "logps/ref_rejected": -95.93243408203125,
      "logps/rejected": -99.80471801757812,
      "loss": 1.0633,
      "margin_dpo/margin_mean": 3.1142072677612305,
      "margin_dpo/margin_std": 4.913205146789551,
      "step": 73
    },
    {
      "epoch": 0.10866372980910426,
      "fcm_dpo/beta": 0.18055209517478943,
      "fcm_dpo/delta": 0.23689761757850647,
      "fcm_dpo/margin": 3.480034589767456,
      "fcm_dpo/q_t": 0.37341517210006714,
      "grad_norm": 39.81556701660156,
      "learning_rate": 4.999472998758977e-07,
      "logits/chosen": -0.11234185099601746,
      "logits/rejected": 0.03054591454565525,
      "logps/chosen": -79.81861877441406,
      "logps/ref_chosen": -78.80839538574219,
      "logps/ref_rejected": -101.64677429199219,
      "logps/rejected": -106.13703155517578,
      "loss": 0.9882,
      "margin_dpo/margin_mean": 3.480034589767456,
      "margin_dpo/margin_std": 4.480711936950684,
      "step": 74
    },
    {
      "epoch": 0.11013215859030837,
      "fcm_dpo/beta": 0.1853078305721283,
      "fcm_dpo/delta": 0.030487246811389923,
      "fcm_dpo/margin": 4.428960800170898,
      "fcm_dpo/q_t": 0.3425433039665222,
      "grad_norm": 40.42939758300781,
      "learning_rate": 4.999176576834721e-07,
      "logits/chosen": -0.18196149170398712,
      "logits/rejected": 0.018665947020053864,
      "logps/chosen": -79.34150695800781,
      "logps/ref_chosen": -78.28185272216797,
      "logps/ref_rejected": -115.40311431884766,
      "logps/rejected": -120.8917236328125,
      "loss": 0.8974,
      "margin_dpo/margin_mean": 4.428961277008057,
      "margin_dpo/margin_std": 5.278062343597412,
      "step": 75
    },
    {
      "epoch": 0.11160058737151249,
      "fcm_dpo/beta": 0.19199031591415405,
      "fcm_dpo/delta": 0.2775818109512329,
      "fcm_dpo/margin": 3.0665392875671387,
      "fcm_dpo/q_t": 0.38420790433883667,
      "grad_norm": 39.92314529418945,
      "learning_rate": 4.998814299283415e-07,
      "logits/chosen": -0.6445671319961548,
      "logits/rejected": -0.2710894048213959,
      "logps/chosen": -89.29500579833984,
      "logps/ref_chosen": -87.87713623046875,
      "logps/ref_rejected": -85.71968078613281,
      "logps/rejected": -90.20408630371094,
      "loss": 1.0398,
      "margin_dpo/margin_mean": 3.066539764404297,
      "margin_dpo/margin_std": 4.716013431549072,
      "step": 76
    },
    {
      "epoch": 0.1130690161527166,
      "fcm_dpo/beta": 0.19982174038887024,
      "fcm_dpo/delta": 0.19942709803581238,
      "fcm_dpo/margin": 3.324207305908203,
      "fcm_dpo/q_t": 0.37066739797592163,
      "grad_norm": 50.32765197753906,
      "learning_rate": 4.998386175651409e-07,
      "logits/chosen": -0.15631820261478424,
      "logits/rejected": 0.21387773752212524,
      "logps/chosen": -101.25022888183594,
      "logps/ref_chosen": -99.70034790039062,
      "logps/ref_rejected": -98.20576477050781,
      "logps/rejected": -103.07984924316406,
      "loss": 1.0254,
      "margin_dpo/margin_mean": 3.324207305908203,
      "margin_dpo/margin_std": 4.918254852294922,
      "step": 77
    },
    {
      "epoch": 0.1145374449339207,
      "fcm_dpo/beta": 0.20663225650787354,
      "fcm_dpo/delta": 0.1321578472852707,
      "fcm_dpo/margin": 3.520613193511963,
      "fcm_dpo/q_t": 0.35340678691864014,
      "grad_norm": 36.7442512512207,
      "learning_rate": 4.997892217220159e-07,
      "logits/chosen": 0.2885063588619232,
      "logits/rejected": 0.5307227373123169,
      "logps/chosen": -90.95689392089844,
      "logps/ref_chosen": -90.29670715332031,
      "logps/ref_rejected": -91.13772583007812,
      "logps/rejected": -95.31852722167969,
      "loss": 0.9308,
      "margin_dpo/margin_mean": 3.5206122398376465,
      "margin_dpo/margin_std": 4.204378604888916,
      "step": 78
    },
    {
      "epoch": 0.11600587371512482,
      "fcm_dpo/beta": 0.20485563576221466,
      "fcm_dpo/delta": -0.12860579788684845,
      "fcm_dpo/margin": 4.717245101928711,
      "fcm_dpo/q_t": 0.3210354447364807,
      "grad_norm": 39.420284271240234,
      "learning_rate": 4.997332437005931e-07,
      "logits/chosen": -0.5903613567352295,
      "logits/rejected": -0.21010416746139526,
      "logps/chosen": -87.16404724121094,
      "logps/ref_chosen": -86.37832641601562,
      "logps/ref_rejected": -94.10777282714844,
      "logps/rejected": -99.61073303222656,
      "loss": 0.8702,
      "margin_dpo/margin_mean": 4.717245578765869,
      "margin_dpo/margin_std": 5.7245635986328125,
      "step": 79
    },
    {
      "epoch": 0.11747430249632893,
      "fcm_dpo/beta": 0.20580410957336426,
      "fcm_dpo/delta": 0.09618805348873138,
      "fcm_dpo/margin": 3.69810152053833,
      "fcm_dpo/q_t": 0.3560129404067993,
      "grad_norm": 43.681617736816406,
      "learning_rate": 4.996706849759452e-07,
      "logits/chosen": -0.5645250678062439,
      "logits/rejected": -0.20316016674041748,
      "logps/chosen": -95.42378997802734,
      "logps/ref_chosen": -93.97032165527344,
      "logps/ref_rejected": -92.57441711425781,
      "logps/rejected": -97.72599792480469,
      "loss": 0.9497,
      "margin_dpo/margin_mean": 3.698101043701172,
      "margin_dpo/margin_std": 4.941034317016602,
      "step": 80
    },
    {
      "epoch": 0.11894273127753303,
      "fcm_dpo/beta": 0.19814518094062805,
      "fcm_dpo/delta": -0.21343490481376648,
      "fcm_dpo/margin": 5.233097076416016,
      "fcm_dpo/q_t": 0.3187583088874817,
      "grad_norm": 39.48776626586914,
      "learning_rate": 4.996015471965529e-07,
      "logits/chosen": -0.4514112174510956,
      "logits/rejected": -0.026859581470489502,
      "logps/chosen": -100.7833480834961,
      "logps/ref_chosen": -99.83012390136719,
      "logps/ref_rejected": -133.67247009277344,
      "logps/rejected": -139.85879516601562,
      "loss": 0.8648,
      "margin_dpo/margin_mean": 5.233097076416016,
      "margin_dpo/margin_std": 6.736208438873291,
      "step": 81
    },
    {
      "epoch": 0.12041116005873716,
      "fcm_dpo/beta": 0.2045668065547943,
      "fcm_dpo/delta": 0.13720954954624176,
      "fcm_dpo/margin": 3.512045383453369,
      "fcm_dpo/q_t": 0.35955876111984253,
      "grad_norm": 42.71525192260742,
      "learning_rate": 4.995258321842611e-07,
      "logits/chosen": -0.2967575490474701,
      "logits/rejected": -0.08636781573295593,
      "logps/chosen": -84.83177185058594,
      "logps/ref_chosen": -83.04598236083984,
      "logps/ref_rejected": -94.52595520019531,
      "logps/rejected": -99.82378387451172,
      "loss": 0.9743,
      "margin_dpo/margin_mean": 3.512044906616211,
      "margin_dpo/margin_std": 4.646213054656982,
      "step": 82
    },
    {
      "epoch": 0.12187958883994127,
      "fcm_dpo/beta": 0.20421761274337769,
      "fcm_dpo/delta": 0.02744770422577858,
      "fcm_dpo/margin": 4.038933753967285,
      "fcm_dpo/q_t": 0.3518849313259125,
      "grad_norm": 58.90052032470703,
      "learning_rate": 4.994435419342304e-07,
      "logits/chosen": -0.4892539381980896,
      "logits/rejected": -0.07342211157083511,
      "logps/chosen": -94.18898010253906,
      "logps/ref_chosen": -92.17621612548828,
      "logps/ref_rejected": -107.74464416503906,
      "logps/rejected": -113.79634094238281,
      "loss": 0.9931,
      "margin_dpo/margin_mean": 4.038934230804443,
      "margin_dpo/margin_std": 5.922978401184082,
      "step": 83
    },
    {
      "epoch": 0.12334801762114538,
      "fcm_dpo/beta": 0.20572717487812042,
      "fcm_dpo/delta": 0.02440953627228737,
      "fcm_dpo/margin": 2.4480397701263428,
      "fcm_dpo/q_t": 0.40574872493743896,
      "grad_norm": 62.624961853027344,
      "learning_rate": 4.993546786148857e-07,
      "logits/chosen": -0.3661155104637146,
      "logits/rejected": 0.011862829327583313,
      "logps/chosen": -104.15892028808594,
      "logps/ref_chosen": -101.5264892578125,
      "logps/ref_rejected": -92.42608642578125,
      "logps/rejected": -97.50656127929688,
      "loss": 1.148,
      "margin_dpo/margin_mean": 2.4480395317077637,
      "margin_dpo/margin_std": 4.715095043182373,
      "step": 84
    },
    {
      "epoch": 0.12481644640234948,
      "fcm_dpo/beta": 0.21252544224262238,
      "fcm_dpo/delta": 0.15723800659179688,
      "fcm_dpo/margin": 3.297471523284912,
      "fcm_dpo/q_t": 0.37289494276046753,
      "grad_norm": 64.13458251953125,
      "learning_rate": 4.992592445678582e-07,
      "logits/chosen": -0.2507883310317993,
      "logits/rejected": 0.07221034169197083,
      "logps/chosen": -98.31668853759766,
      "logps/ref_chosen": -96.12738037109375,
      "logps/ref_rejected": -85.05519104003906,
      "logps/rejected": -90.54196166992188,
      "loss": 1.0876,
      "margin_dpo/margin_mean": 3.2974720001220703,
      "margin_dpo/margin_std": 5.628973484039307,
      "step": 85
    },
    {
      "epoch": 0.1262848751835536,
      "fcm_dpo/beta": 0.2112560123205185,
      "fcm_dpo/delta": -0.015112241730093956,
      "fcm_dpo/margin": 4.088076591491699,
      "fcm_dpo/q_t": 0.35769122838974,
      "grad_norm": 42.73858642578125,
      "learning_rate": 4.991572423079235e-07,
      "logits/chosen": -0.5674518346786499,
      "logits/rejected": -0.3341968059539795,
      "logps/chosen": -83.38761138916016,
      "logps/ref_chosen": -81.70426177978516,
      "logps/ref_rejected": -93.65550231933594,
      "logps/rejected": -99.42691802978516,
      "loss": 0.9898,
      "margin_dpo/margin_mean": 4.088076114654541,
      "margin_dpo/margin_std": 6.537631988525391,
      "step": 86
    },
    {
      "epoch": 0.1277533039647577,
      "fcm_dpo/beta": 0.2079107165336609,
      "fcm_dpo/delta": -0.08900696039199829,
      "fcm_dpo/margin": 4.47269344329834,
      "fcm_dpo/q_t": 0.33168458938598633,
      "grad_norm": 54.897640228271484,
      "learning_rate": 4.990486745229364e-07,
      "logits/chosen": -0.6444218158721924,
      "logits/rejected": -0.2799781858921051,
      "logps/chosen": -94.55389404296875,
      "logps/ref_chosen": -92.68596649169922,
      "logps/ref_rejected": -102.91818237304688,
      "logps/rejected": -109.2587890625,
      "loss": 0.9951,
      "margin_dpo/margin_mean": 4.472694396972656,
      "margin_dpo/margin_std": 6.770974159240723,
      "step": 87
    },
    {
      "epoch": 0.12922173274596183,
      "fcm_dpo/beta": 0.20963647961616516,
      "fcm_dpo/delta": 0.13043585419654846,
      "fcm_dpo/margin": 3.4725332260131836,
      "fcm_dpo/q_t": 0.3684794306755066,
      "grad_norm": 53.998870849609375,
      "learning_rate": 4.989335440737586e-07,
      "logits/chosen": -0.18257489800453186,
      "logits/rejected": -0.004581686109304428,
      "logps/chosen": -103.09647369384766,
      "logps/ref_chosen": -100.76298522949219,
      "logps/ref_rejected": -113.15037536621094,
      "logps/rejected": -118.95639038085938,
      "loss": 1.0549,
      "margin_dpo/margin_mean": 3.4725332260131836,
      "margin_dpo/margin_std": 5.660479545593262,
      "step": 88
    },
    {
      "epoch": 0.13069016152716592,
      "fcm_dpo/beta": 0.21444444358348846,
      "fcm_dpo/delta": 0.058221083134412766,
      "fcm_dpo/margin": 3.713489055633545,
      "fcm_dpo/q_t": 0.3624182641506195,
      "grad_norm": 41.0621223449707,
      "learning_rate": 4.988118539941847e-07,
      "logits/chosen": -0.7678704857826233,
      "logits/rejected": -0.4581834077835083,
      "logps/chosen": -91.50296020507812,
      "logps/ref_chosen": -89.69108581542969,
      "logps/ref_rejected": -88.56832885742188,
      "logps/rejected": -94.09370422363281,
      "loss": 1.004,
      "margin_dpo/margin_mean": 3.7134885787963867,
      "margin_dpo/margin_std": 6.147014617919922,
      "step": 89
    },
    {
      "epoch": 0.13215859030837004,
      "fcm_dpo/beta": 0.21235866844654083,
      "fcm_dpo/delta": -0.1558917909860611,
      "fcm_dpo/margin": 4.664684295654297,
      "fcm_dpo/q_t": 0.3254511058330536,
      "grad_norm": 44.808536529541016,
      "learning_rate": 4.986836074908615e-07,
      "logits/chosen": -0.29161787033081055,
      "logits/rejected": -0.17332673072814941,
      "logps/chosen": -82.94302368164062,
      "logps/ref_chosen": -81.38255310058594,
      "logps/ref_rejected": -117.77714538574219,
      "logps/rejected": -124.00228881835938,
      "loss": 0.9002,
      "margin_dpo/margin_mean": 4.664685249328613,
      "margin_dpo/margin_std": 6.0813398361206055,
      "step": 90
    },
    {
      "epoch": 0.13362701908957417,
      "fcm_dpo/beta": 0.2086181640625,
      "fcm_dpo/delta": -0.03319477662444115,
      "fcm_dpo/margin": 4.219526290893555,
      "fcm_dpo/q_t": 0.34347307682037354,
      "grad_norm": 49.89957046508789,
      "learning_rate": 4.985488079432037e-07,
      "logits/chosen": -0.19626545906066895,
      "logits/rejected": 0.20309990644454956,
      "logps/chosen": -98.95803833007812,
      "logps/ref_chosen": -97.22187805175781,
      "logps/ref_rejected": -92.97674560546875,
      "logps/rejected": -98.93242645263672,
      "loss": 0.9327,
      "margin_dpo/margin_mean": 4.2195258140563965,
      "margin_dpo/margin_std": 5.781785011291504,
      "step": 91
    },
    {
      "epoch": 0.13509544787077826,
      "fcm_dpo/beta": 0.21198439598083496,
      "fcm_dpo/delta": 0.11860194057226181,
      "fcm_dpo/margin": 3.4903693199157715,
      "fcm_dpo/q_t": 0.3639296889305115,
      "grad_norm": 42.03072738647461,
      "learning_rate": 4.984074589033043e-07,
      "logits/chosen": -0.795829176902771,
      "logits/rejected": -0.45754796266555786,
      "logps/chosen": -86.59548950195312,
      "logps/ref_chosen": -84.5302734375,
      "logps/ref_rejected": -84.5013198852539,
      "logps/rejected": -90.05690002441406,
      "loss": 1.0081,
      "margin_dpo/margin_mean": 3.4903693199157715,
      "margin_dpo/margin_std": 5.338055610656738,
      "step": 92
    },
    {
      "epoch": 0.13656387665198239,
      "fcm_dpo/beta": 0.21652595698833466,
      "fcm_dpo/delta": 0.10127636045217514,
      "fcm_dpo/margin": 3.492115020751953,
      "fcm_dpo/q_t": 0.3656906485557556,
      "grad_norm": 58.36591720581055,
      "learning_rate": 4.982595640958425e-07,
      "logits/chosen": -0.7161461114883423,
      "logits/rejected": -0.16940158605575562,
      "logps/chosen": -92.29219055175781,
      "logps/ref_chosen": -90.25043487548828,
      "logps/ref_rejected": -84.0942153930664,
      "logps/rejected": -89.62808990478516,
      "loss": 1.068,
      "margin_dpo/margin_mean": 3.492115020751953,
      "margin_dpo/margin_std": 5.786944389343262,
      "step": 93
    },
    {
      "epoch": 0.13803230543318648,
      "fcm_dpo/beta": 0.20668484270572662,
      "fcm_dpo/delta": -0.22309033572673798,
      "fcm_dpo/margin": 5.036521911621094,
      "fcm_dpo/q_t": 0.3098403215408325,
      "grad_norm": 48.04051971435547,
      "learning_rate": 4.98105127417984e-07,
      "logits/chosen": -0.43175843358039856,
      "logits/rejected": -0.14784827828407288,
      "logps/chosen": -94.38967895507812,
      "logps/ref_chosen": -92.4542236328125,
      "logps/ref_rejected": -105.2472915649414,
      "logps/rejected": -112.21926879882812,
      "loss": 0.884,
      "margin_dpo/margin_mean": 5.036520957946777,
      "margin_dpo/margin_std": 6.06168794631958,
      "step": 94
    },
    {
      "epoch": 0.1395007342143906,
      "fcm_dpo/beta": 0.209566131234169,
      "fcm_dpo/delta": 0.002998150885105133,
      "fcm_dpo/margin": 4.036772727966309,
      "fcm_dpo/q_t": 0.34862929582595825,
      "grad_norm": 42.049312591552734,
      "learning_rate": 4.979441529392784e-07,
      "logits/chosen": -0.8037420511245728,
      "logits/rejected": -0.5004515647888184,
      "logps/chosen": -80.51316833496094,
      "logps/ref_chosen": -78.87370300292969,
      "logps/ref_rejected": -83.59121704101562,
      "logps/rejected": -89.26744079589844,
      "loss": 0.9502,
      "margin_dpo/margin_mean": 4.036772727966309,
      "margin_dpo/margin_std": 5.532905101776123,
      "step": 95
    },
    {
      "epoch": 0.14096916299559473,
      "fcm_dpo/beta": 0.20307976007461548,
      "fcm_dpo/delta": -0.13602276146411896,
      "fcm_dpo/margin": 4.787096977233887,
      "fcm_dpo/q_t": 0.3208298683166504,
      "grad_norm": 39.64480209350586,
      "learning_rate": 4.977766449015534e-07,
      "logits/chosen": -0.48742806911468506,
      "logits/rejected": -0.07584237307310104,
      "logps/chosen": -107.8035888671875,
      "logps/ref_chosen": -106.5921630859375,
      "logps/ref_rejected": -101.76802825927734,
      "logps/rejected": -107.76654052734375,
      "loss": 0.8767,
      "margin_dpo/margin_mean": 4.78709602355957,
      "margin_dpo/margin_std": 5.776007652282715,
      "step": 96
    },
    {
      "epoch": 0.14243759177679882,
      "fcm_dpo/beta": 0.20178145170211792,
      "fcm_dpo/delta": -0.09740416705608368,
      "fcm_dpo/margin": 4.642391681671143,
      "fcm_dpo/q_t": 0.3197094202041626,
      "grad_norm": 41.0906867980957,
      "learning_rate": 4.976026077188012e-07,
      "logits/chosen": -0.3646326959133148,
      "logits/rejected": 0.015910834074020386,
      "logps/chosen": -90.6146469116211,
      "logps/ref_chosen": -88.67988586425781,
      "logps/ref_rejected": -84.81229400634766,
      "logps/rejected": -91.38944244384766,
      "loss": 0.8343,
      "margin_dpo/margin_mean": 4.642391204833984,
      "margin_dpo/margin_std": 4.824398040771484,
      "step": 97
    },
    {
      "epoch": 0.14390602055800295,
      "fcm_dpo/beta": 0.19345466792583466,
      "fcm_dpo/delta": -0.10774275660514832,
      "fcm_dpo/margin": 4.8845977783203125,
      "fcm_dpo/q_t": 0.3147512674331665,
      "grad_norm": 38.62285614013672,
      "learning_rate": 4.974220459770639e-07,
      "logits/chosen": -0.33925938606262207,
      "logits/rejected": -0.09292425215244293,
      "logps/chosen": -93.5835189819336,
      "logps/ref_chosen": -92.24249267578125,
      "logps/ref_rejected": -101.51948547363281,
      "logps/rejected": -107.74510192871094,
      "loss": 0.8221,
      "margin_dpo/margin_mean": 4.884597301483154,
      "margin_dpo/margin_std": 5.033493518829346,
      "step": 98
    },
    {
      "epoch": 0.14537444933920704,
      "fcm_dpo/beta": 0.18968458473682404,
      "fcm_dpo/delta": -0.15581762790679932,
      "fcm_dpo/margin": 5.220826148986816,
      "fcm_dpo/q_t": 0.3240944743156433,
      "grad_norm": 34.411415100097656,
      "learning_rate": 4.972349644343108e-07,
      "logits/chosen": -0.3686472177505493,
      "logits/rejected": -0.16389545798301697,
      "logps/chosen": -73.22004699707031,
      "logps/ref_chosen": -72.18464660644531,
      "logps/ref_rejected": -91.88131713867188,
      "logps/rejected": -98.13754272460938,
      "loss": 0.8523,
      "margin_dpo/margin_mean": 5.220826148986816,
      "margin_dpo/margin_std": 6.51201057434082,
      "step": 99
    },
    {
      "epoch": 0.14684287812041116,
      "fcm_dpo/beta": 0.18906737864017487,
      "fcm_dpo/delta": 0.07484784722328186,
      "fcm_dpo/margin": 4.128679275512695,
      "fcm_dpo/q_t": 0.34153318405151367,
      "grad_norm": 33.857669830322266,
      "learning_rate": 4.970413680203148e-07,
      "logits/chosen": -0.4443850517272949,
      "logits/rejected": 0.00018494389951229095,
      "logps/chosen": -90.57693481445312,
      "logps/ref_chosen": -89.51382446289062,
      "logps/ref_rejected": -81.21713256835938,
      "logps/rejected": -86.40892028808594,
      "loss": 0.8851,
      "margin_dpo/margin_mean": 4.128678798675537,
      "margin_dpo/margin_std": 4.408847808837891,
      "step": 100
    },
    {
      "epoch": 0.14684287812041116,
      "eval_fcm_dpo/beta": 0.19106584787368774,
      "eval_logits/chosen": -0.04890349507331848,
      "eval_logits/rejected": 0.16974791884422302,
      "eval_logps/chosen": -102.81055450439453,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -99.17350006103516,
      "eval_loss": 0.5619437098503113,
      "eval_margin_dpo/margin_mean": 2.788757562637329,
      "eval_margin_dpo/margin_std": 5.18633508682251,
      "eval_runtime": 43.7613,
      "eval_samples_per_second": 53.449,
      "eval_steps_per_second": 1.691,
      "step": 100
    },
    {
      "epoch": 0.14831130690161526,
      "fcm_dpo/beta": 0.19471797347068787,
      "fcm_dpo/delta": 0.16918794810771942,
      "fcm_dpo/margin": 3.557769536972046,
      "fcm_dpo/q_t": 0.3666532635688782,
      "grad_norm": 48.270713806152344,
      "learning_rate": 4.968412618365215e-07,
      "logits/chosen": -0.5501360297203064,
      "logits/rejected": -0.311795175075531,
      "logps/chosen": -96.4827880859375,
      "logps/ref_chosen": -94.4031982421875,
      "logps/ref_rejected": -89.91001892089844,
      "logps/rejected": -95.54737854003906,
      "loss": 1.0253,
      "margin_dpo/margin_mean": 3.557770013809204,
      "margin_dpo/margin_std": 5.32769775390625,
      "step": 101
    },
    {
      "epoch": 0.14977973568281938,
      "fcm_dpo/beta": 0.20780380070209503,
      "fcm_dpo/delta": 0.24079912900924683,
      "fcm_dpo/margin": 2.968078136444092,
      "fcm_dpo/q_t": 0.3742523789405823,
      "grad_norm": 63.03786087036133,
      "learning_rate": 4.966346511559149e-07,
      "logits/chosen": -0.6122028827667236,
      "logits/rejected": -0.29000306129455566,
      "logps/chosen": -94.04932403564453,
      "logps/ref_chosen": -91.60942077636719,
      "logps/ref_rejected": -75.7689208984375,
      "logps/rejected": -81.17689514160156,
      "loss": 1.0796,
      "margin_dpo/margin_mean": 2.9680778980255127,
      "margin_dpo/margin_std": 4.8967742919921875,
      "step": 102
    },
    {
      "epoch": 0.1512481644640235,
      "fcm_dpo/beta": 0.20262879133224487,
      "fcm_dpo/delta": -0.1501111090183258,
      "fcm_dpo/margin": 4.861822605133057,
      "fcm_dpo/q_t": 0.3110318183898926,
      "grad_norm": 44.94874954223633,
      "learning_rate": 4.964215414228785e-07,
      "logits/chosen": -0.5446847677230835,
      "logits/rejected": -0.20542305707931519,
      "logps/chosen": -98.96780395507812,
      "logps/ref_chosen": -97.40238952636719,
      "logps/ref_rejected": -98.981689453125,
      "logps/rejected": -105.40892028808594,
      "loss": 0.8525,
      "margin_dpo/margin_mean": 4.861822605133057,
      "margin_dpo/margin_std": 5.37109899520874,
      "step": 103
    },
    {
      "epoch": 0.1527165932452276,
      "fcm_dpo/beta": 0.19483795762062073,
      "fcm_dpo/delta": -0.2128961682319641,
      "fcm_dpo/margin": 5.340367317199707,
      "fcm_dpo/q_t": 0.30966752767562866,
      "grad_norm": 33.6173210144043,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": -0.584915816783905,
      "logits/rejected": -0.27762916684150696,
      "logps/chosen": -95.84004211425781,
      "logps/ref_chosen": -94.85636138916016,
      "logps/ref_rejected": -97.02669525146484,
      "logps/rejected": -103.35074615478516,
      "loss": 0.8139,
      "margin_dpo/margin_mean": 5.340367317199707,
      "margin_dpo/margin_std": 6.170401573181152,
      "step": 104
    },
    {
      "epoch": 0.15418502202643172,
      "fcm_dpo/beta": 0.19031333923339844,
      "fcm_dpo/delta": -0.13976818323135376,
      "fcm_dpo/margin": 5.128673553466797,
      "fcm_dpo/q_t": 0.31237882375717163,
      "grad_norm": 40.28583908081055,
      "learning_rate": 4.959758474331832e-07,
      "logits/chosen": -0.6987419724464417,
      "logits/rejected": -0.28522351384162903,
      "logps/chosen": -98.16111755371094,
      "logps/ref_chosen": -96.75790405273438,
      "logps/ref_rejected": -101.5774917602539,
      "logps/rejected": -108.10939025878906,
      "loss": 0.8491,
      "margin_dpo/margin_mean": 5.128673076629639,
      "margin_dpo/margin_std": 5.720231056213379,
      "step": 105
    },
    {
      "epoch": 0.15565345080763582,
      "fcm_dpo/beta": 0.18653175234794617,
      "fcm_dpo/delta": -0.09469683468341827,
      "fcm_dpo/margin": 5.013686656951904,
      "fcm_dpo/q_t": 0.32377827167510986,
      "grad_norm": 35.92363739013672,
      "learning_rate": 4.957432749209755e-07,
      "logits/chosen": -0.38015517592430115,
      "logits/rejected": 0.013428784906864166,
      "logps/chosen": -93.89437866210938,
      "logps/ref_chosen": -92.43791198730469,
      "logps/ref_rejected": -91.00970458984375,
      "logps/rejected": -97.47986602783203,
      "loss": 0.8544,
      "margin_dpo/margin_mean": 5.0136871337890625,
      "margin_dpo/margin_std": 5.765174865722656,
      "step": 106
    },
    {
      "epoch": 0.15712187958883994,
      "fcm_dpo/beta": 0.18005526065826416,
      "fcm_dpo/delta": -0.16589003801345825,
      "fcm_dpo/margin": 5.5514326095581055,
      "fcm_dpo/q_t": 0.32457587122917175,
      "grad_norm": 35.89270782470703,
      "learning_rate": 4.955042268449307e-07,
      "logits/chosen": -0.4841831922531128,
      "logits/rejected": -0.03617634251713753,
      "logps/chosen": -105.86752319335938,
      "logps/ref_chosen": -105.12500762939453,
      "logps/ref_rejected": -101.1824951171875,
      "logps/rejected": -107.47644805908203,
      "loss": 0.8619,
      "margin_dpo/margin_mean": 5.5514326095581055,
      "margin_dpo/margin_std": 7.6866655349731445,
      "step": 107
    },
    {
      "epoch": 0.15859030837004406,
      "fcm_dpo/beta": 0.1751759946346283,
      "fcm_dpo/delta": -0.052587710320949554,
      "fcm_dpo/margin": 5.115565776824951,
      "fcm_dpo/q_t": 0.330201119184494,
      "grad_norm": 32.2303581237793,
      "learning_rate": 4.952587095041881e-07,
      "logits/chosen": -0.5506917238235474,
      "logits/rejected": -0.06726711988449097,
      "logps/chosen": -90.73790740966797,
      "logps/ref_chosen": -89.20936584472656,
      "logps/ref_rejected": -101.83784484863281,
      "logps/rejected": -108.48196411132812,
      "loss": 0.8834,
      "margin_dpo/margin_mean": 5.115566253662109,
      "margin_dpo/margin_std": 6.003669738769531,
      "step": 108
    },
    {
      "epoch": 0.16005873715124816,
      "fcm_dpo/beta": 0.16900481283664703,
      "fcm_dpo/delta": -0.2684016525745392,
      "fcm_dpo/margin": 6.4412641525268555,
      "fcm_dpo/q_t": 0.2995629906654358,
      "grad_norm": 33.71637725830078,
      "learning_rate": 4.95006729368358e-07,
      "logits/chosen": 0.004945341497659683,
      "logits/rejected": 0.20756444334983826,
      "logps/chosen": -101.10147094726562,
      "logps/ref_chosen": -100.18708801269531,
      "logps/ref_rejected": -103.53739929199219,
      "logps/rejected": -110.8930435180664,
      "loss": 0.8032,
      "margin_dpo/margin_mean": 6.441263675689697,
      "margin_dpo/margin_std": 6.986750602722168,
      "step": 109
    },
    {
      "epoch": 0.16152716593245228,
      "fcm_dpo/beta": 0.17018066346645355,
      "fcm_dpo/delta": 0.16250738501548767,
      "fcm_dpo/margin": 4.108128547668457,
      "fcm_dpo/q_t": 0.35990265011787415,
      "grad_norm": 35.48023986816406,
      "learning_rate": 4.947482930773511e-07,
      "logits/chosen": 0.06814084947109222,
      "logits/rejected": 0.411674439907074,
      "logps/chosen": -108.29478454589844,
      "logps/ref_chosen": -106.89572143554688,
      "logps/ref_rejected": -85.81120300292969,
      "logps/rejected": -91.31838989257812,
      "loss": 0.9661,
      "margin_dpo/margin_mean": 4.108128547668457,
      "margin_dpo/margin_std": 5.228711128234863,
      "step": 110
    },
    {
      "epoch": 0.16299559471365638,
      "fcm_dpo/beta": 0.16213089227676392,
      "fcm_dpo/delta": -0.34137529134750366,
      "fcm_dpo/margin": 7.0692596435546875,
      "fcm_dpo/q_t": 0.28307241201400757,
      "grad_norm": 29.604639053344727,
      "learning_rate": 4.944834074412042e-07,
      "logits/chosen": -0.5990385413169861,
      "logits/rejected": -0.3386729955673218,
      "logps/chosen": -94.55763244628906,
      "logps/ref_chosen": -93.3293685913086,
      "logps/ref_rejected": -104.177490234375,
      "logps/rejected": -112.47500610351562,
      "loss": 0.7428,
      "margin_dpo/margin_mean": 7.069259166717529,
      "margin_dpo/margin_std": 6.622108459472656,
      "step": 111
    },
    {
      "epoch": 0.1644640234948605,
      "fcm_dpo/beta": 0.1657009720802307,
      "fcm_dpo/delta": 0.1916673481464386,
      "fcm_dpo/margin": 4.050925254821777,
      "fcm_dpo/q_t": 0.3612492084503174,
      "grad_norm": 32.211814880371094,
      "learning_rate": 4.942120794399002e-07,
      "logits/chosen": -0.8714514374732971,
      "logits/rejected": -0.47875988483428955,
      "logps/chosen": -84.03836059570312,
      "logps/ref_chosen": -82.74703979492188,
      "logps/ref_rejected": -71.5584487915039,
      "logps/rejected": -76.90069580078125,
      "loss": 0.9483,
      "margin_dpo/margin_mean": 4.050925254821777,
      "margin_dpo/margin_std": 4.675178527832031,
      "step": 112
    },
    {
      "epoch": 0.16593245227606462,
      "fcm_dpo/beta": 0.17069506645202637,
      "fcm_dpo/delta": 0.12468759715557098,
      "fcm_dpo/margin": 4.301934242248535,
      "fcm_dpo/q_t": 0.34889668226242065,
      "grad_norm": 37.83422088623047,
      "learning_rate": 4.939343162231841e-07,
      "logits/chosen": -0.5917797088623047,
      "logits/rejected": -0.1455421894788742,
      "logps/chosen": -99.65104675292969,
      "logps/ref_chosen": -97.89949035644531,
      "logps/ref_rejected": -84.41960144042969,
      "logps/rejected": -90.47306823730469,
      "loss": 0.9257,
      "margin_dpo/margin_mean": 4.301935195922852,
      "margin_dpo/margin_std": 4.792448997497559,
      "step": 113
    },
    {
      "epoch": 0.16740088105726872,
      "fcm_dpo/beta": 0.16812041401863098,
      "fcm_dpo/delta": -0.09158610552549362,
      "fcm_dpo/margin": 5.540801048278809,
      "fcm_dpo/q_t": 0.3299529254436493,
      "grad_norm": 38.36504364013672,
      "learning_rate": 4.936501251103751e-07,
      "logits/chosen": -0.43144339323043823,
      "logits/rejected": 0.10210932791233063,
      "logps/chosen": -98.132080078125,
      "logps/ref_chosen": -96.4951171875,
      "logps/ref_rejected": -92.01536560058594,
      "logps/rejected": -99.19312286376953,
      "loss": 0.8925,
      "margin_dpo/margin_mean": 5.540801048278809,
      "margin_dpo/margin_std": 6.796547889709473,
      "step": 114
    },
    {
      "epoch": 0.16886930983847284,
      "fcm_dpo/beta": 0.1658993363380432,
      "fcm_dpo/delta": -0.11033232510089874,
      "fcm_dpo/margin": 5.7258405685424805,
      "fcm_dpo/q_t": 0.31527501344680786,
      "grad_norm": 31.082290649414062,
      "learning_rate": 4.933595135901732e-07,
      "logits/chosen": -0.16044090688228607,
      "logits/rejected": 0.21673715114593506,
      "logps/chosen": -104.51092529296875,
      "logps/ref_chosen": -103.18013000488281,
      "logps/ref_rejected": -105.71595764160156,
      "logps/rejected": -112.77259826660156,
      "loss": 0.8321,
      "margin_dpo/margin_mean": 5.725839614868164,
      "margin_dpo/margin_std": 6.0801544189453125,
      "step": 115
    },
    {
      "epoch": 0.17033773861967694,
      "fcm_dpo/beta": 0.16711236536502838,
      "fcm_dpo/delta": 0.06599107384681702,
      "fcm_dpo/margin": 4.719298839569092,
      "fcm_dpo/q_t": 0.34467944502830505,
      "grad_norm": 30.001972198486328,
      "learning_rate": 4.930624893204624e-07,
      "logits/chosen": -0.4106103777885437,
      "logits/rejected": -0.010030269622802734,
      "logps/chosen": -82.48646545410156,
      "logps/ref_chosen": -81.15837097167969,
      "logps/ref_rejected": -86.30586242675781,
      "logps/rejected": -92.3532485961914,
      "loss": 0.9057,
      "margin_dpo/margin_mean": 4.71929931640625,
      "margin_dpo/margin_std": 5.378247261047363,
      "step": 116
    },
    {
      "epoch": 0.17180616740088106,
      "fcm_dpo/beta": 0.16843508183956146,
      "fcm_dpo/delta": 0.12693826854228973,
      "fcm_dpo/margin": 4.342744827270508,
      "fcm_dpo/q_t": 0.3495209217071533,
      "grad_norm": 40.84837341308594,
      "learning_rate": 4.927590601281083e-07,
      "logits/chosen": -0.2890988290309906,
      "logits/rejected": 0.05249544978141785,
      "logps/chosen": -98.79878234863281,
      "logps/ref_chosen": -97.0809555053711,
      "logps/ref_rejected": -75.1399154663086,
      "logps/rejected": -81.20048522949219,
      "loss": 0.9336,
      "margin_dpo/margin_mean": 4.342745304107666,
      "margin_dpo/margin_std": 4.965022563934326,
      "step": 117
    },
    {
      "epoch": 0.17327459618208516,
      "fcm_dpo/beta": 0.16939415037631989,
      "fcm_dpo/delta": -0.06589382886886597,
      "fcm_dpo/margin": 5.3707122802734375,
      "fcm_dpo/q_t": 0.31360048055648804,
      "grad_norm": 30.420207977294922,
      "learning_rate": 4.924492340087524e-07,
      "logits/chosen": -0.663594663143158,
      "logits/rejected": -0.3713434636592865,
      "logps/chosen": -87.47075653076172,
      "logps/ref_chosen": -86.53623962402344,
      "logps/ref_rejected": -82.93563842773438,
      "logps/rejected": -89.24087524414062,
      "loss": 0.8266,
      "margin_dpo/margin_mean": 5.370712757110596,
      "margin_dpo/margin_std": 5.203623294830322,
      "step": 118
    },
    {
      "epoch": 0.17474302496328928,
      "fcm_dpo/beta": 0.16728976368904114,
      "fcm_dpo/delta": -0.050966691225767136,
      "fcm_dpo/margin": 5.35649299621582,
      "fcm_dpo/q_t": 0.32624873518943787,
      "grad_norm": 36.59634780883789,
      "learning_rate": 4.92133019126601e-07,
      "logits/chosen": -0.38907530903816223,
      "logits/rejected": -0.10628137737512589,
      "logps/chosen": -106.18470764160156,
      "logps/ref_chosen": -104.60908508300781,
      "logps/ref_rejected": -109.15780639648438,
      "logps/rejected": -116.08993530273438,
      "loss": 0.907,
      "margin_dpo/margin_mean": 5.35649299621582,
      "margin_dpo/margin_std": 6.589056015014648,
      "step": 119
    },
    {
      "epoch": 0.1762114537444934,
      "fcm_dpo/beta": 0.15864476561546326,
      "fcm_dpo/delta": -0.3532995283603668,
      "fcm_dpo/margin": 7.322626113891602,
      "fcm_dpo/q_t": 0.29675668478012085,
      "grad_norm": 42.30807876586914,
      "learning_rate": 4.918104238142103e-07,
      "logits/chosen": -0.6697523593902588,
      "logits/rejected": -0.18009905517101288,
      "logps/chosen": -115.46632385253906,
      "logps/ref_chosen": -114.68742370605469,
      "logps/ref_rejected": -111.18586730957031,
      "logps/rejected": -119.28739929199219,
      "loss": 0.823,
      "margin_dpo/margin_mean": 7.322626113891602,
      "margin_dpo/margin_std": 8.437666893005371,
      "step": 120
    },
    {
      "epoch": 0.1776798825256975,
      "fcm_dpo/beta": 0.15266025066375732,
      "fcm_dpo/delta": -0.10870355367660522,
      "fcm_dpo/margin": 6.205869674682617,
      "fcm_dpo/q_t": 0.31219685077667236,
      "grad_norm": 35.275596618652344,
      "learning_rate": 4.91481456572267e-07,
      "logits/chosen": -0.2544522285461426,
      "logits/rejected": 0.05134699493646622,
      "logps/chosen": -92.9080810546875,
      "logps/ref_chosen": -92.11666107177734,
      "logps/ref_rejected": -114.972900390625,
      "logps/rejected": -121.97018432617188,
      "loss": 0.8079,
      "margin_dpo/margin_mean": 6.205869674682617,
      "margin_dpo/margin_std": 6.08989143371582,
      "step": 121
    },
    {
      "epoch": 0.17914831130690162,
      "fcm_dpo/beta": 0.14998239278793335,
      "fcm_dpo/delta": -0.14806339144706726,
      "fcm_dpo/margin": 6.558841705322266,
      "fcm_dpo/q_t": 0.3043653964996338,
      "grad_norm": 35.02898025512695,
      "learning_rate": 4.911461260693638e-07,
      "logits/chosen": -0.5000349283218384,
      "logits/rejected": -0.36555373668670654,
      "logps/chosen": -86.06637573242188,
      "logps/ref_chosen": -84.87757110595703,
      "logps/ref_rejected": -111.0829849243164,
      "logps/rejected": -118.83062744140625,
      "loss": 0.8072,
      "margin_dpo/margin_mean": 6.558840751647949,
      "margin_dpo/margin_std": 6.412026405334473,
      "step": 122
    },
    {
      "epoch": 0.18061674008810572,
      "fcm_dpo/beta": 0.1485665887594223,
      "fcm_dpo/delta": 0.08498059958219528,
      "fcm_dpo/margin": 5.186343669891357,
      "fcm_dpo/q_t": 0.344007283449173,
      "grad_norm": 31.594209671020508,
      "learning_rate": 4.908044411417711e-07,
      "logits/chosen": -0.68192058801651,
      "logits/rejected": -0.4137950539588928,
      "logps/chosen": -99.76564025878906,
      "logps/ref_chosen": -98.47874450683594,
      "logps/ref_rejected": -93.85417175292969,
      "logps/rejected": -100.32740783691406,
      "loss": 0.907,
      "margin_dpo/margin_mean": 5.186343193054199,
      "margin_dpo/margin_std": 5.748686790466309,
      "step": 123
    },
    {
      "epoch": 0.18208516886930984,
      "fcm_dpo/beta": 0.14938881993293762,
      "fcm_dpo/delta": -0.0725385993719101,
      "fcm_dpo/margin": 6.131298065185547,
      "fcm_dpo/q_t": 0.33808284997940063,
      "grad_norm": 33.024810791015625,
      "learning_rate": 4.904564107932048e-07,
      "logits/chosen": -0.5029033422470093,
      "logits/rejected": -0.2667350769042969,
      "logps/chosen": -103.74954223632812,
      "logps/ref_chosen": -102.76290130615234,
      "logps/ref_rejected": -123.005615234375,
      "logps/rejected": -130.12356567382812,
      "loss": 0.9159,
      "margin_dpo/margin_mean": 6.1312971115112305,
      "margin_dpo/margin_std": 8.150118827819824,
      "step": 124
    },
    {
      "epoch": 0.18355359765051396,
      "fcm_dpo/beta": 0.15123271942138672,
      "fcm_dpo/delta": 0.11664462089538574,
      "fcm_dpo/margin": 4.905575752258301,
      "fcm_dpo/q_t": 0.353229820728302,
      "grad_norm": 33.92650604248047,
      "learning_rate": 4.90102044194588e-07,
      "logits/chosen": -0.37007012963294983,
      "logits/rejected": -0.20071515440940857,
      "logps/chosen": -92.7006607055664,
      "logps/ref_chosen": -90.42627716064453,
      "logps/ref_rejected": -89.24901580810547,
      "logps/rejected": -96.42897033691406,
      "loss": 0.9455,
      "margin_dpo/margin_mean": 4.905576705932617,
      "margin_dpo/margin_std": 6.141561031341553,
      "step": 125
    },
    {
      "epoch": 0.18502202643171806,
      "fcm_dpo/beta": 0.14798884093761444,
      "fcm_dpo/delta": -0.19207918643951416,
      "fcm_dpo/margin": 6.910327434539795,
      "fcm_dpo/q_t": 0.2891598343849182,
      "grad_norm": 30.2923583984375,
      "learning_rate": 4.897413506838102e-07,
      "logits/chosen": -0.3179720640182495,
      "logits/rejected": -0.07812147587537766,
      "logps/chosen": -95.60994720458984,
      "logps/ref_chosen": -94.42288208007812,
      "logps/ref_rejected": -103.88156127929688,
      "logps/rejected": -111.97894287109375,
      "loss": 0.7351,
      "margin_dpo/margin_mean": 6.910326957702637,
      "margin_dpo/margin_std": 5.4942731857299805,
      "step": 126
    },
    {
      "epoch": 0.18649045521292218,
      "fcm_dpo/beta": 0.147809237241745,
      "fcm_dpo/delta": 0.008888594806194305,
      "fcm_dpo/margin": 5.684311866760254,
      "fcm_dpo/q_t": 0.3361806869506836,
      "grad_norm": 35.29644775390625,
      "learning_rate": 4.89374339765481e-07,
      "logits/chosen": -0.5459603071212769,
      "logits/rejected": -0.2612873911857605,
      "logps/chosen": -86.01332092285156,
      "logps/ref_chosen": -84.75105285644531,
      "logps/ref_rejected": -83.54523468017578,
      "logps/rejected": -90.49181365966797,
      "loss": 0.9171,
      "margin_dpo/margin_mean": 5.684311866760254,
      "margin_dpo/margin_std": 6.909788608551025,
      "step": 127
    },
    {
      "epoch": 0.18795888399412627,
      "fcm_dpo/beta": 0.14771845936775208,
      "fcm_dpo/delta": 0.032183483242988586,
      "fcm_dpo/margin": 5.5521039962768555,
      "fcm_dpo/q_t": 0.33484911918640137,
      "grad_norm": 35.253360748291016,
      "learning_rate": 4.890010211106795e-07,
      "logits/chosen": -0.11950550228357315,
      "logits/rejected": 0.2775838077068329,
      "logps/chosen": -91.87654113769531,
      "logps/ref_chosen": -90.61964416503906,
      "logps/ref_rejected": -83.10347747802734,
      "logps/rejected": -89.9124755859375,
      "loss": 0.9291,
      "margin_dpo/margin_mean": 5.552103042602539,
      "margin_dpo/margin_std": 6.96898078918457,
      "step": 128
    },
    {
      "epoch": 0.1894273127753304,
      "fcm_dpo/beta": 0.14885255694389343,
      "fcm_dpo/delta": 0.012820325791835785,
      "fcm_dpo/margin": 5.623868942260742,
      "fcm_dpo/q_t": 0.33776140213012695,
      "grad_norm": 29.690258026123047,
      "learning_rate": 4.88621404556699e-07,
      "logits/chosen": -0.12152446806430817,
      "logits/rejected": 0.14454413950443268,
      "logps/chosen": -100.38304138183594,
      "logps/ref_chosen": -98.91870880126953,
      "logps/ref_rejected": -103.28419494628906,
      "logps/rejected": -110.37239837646484,
      "loss": 0.8899,
      "margin_dpo/margin_mean": 5.623868942260742,
      "margin_dpo/margin_std": 6.273656845092773,
      "step": 129
    },
    {
      "epoch": 0.19089574155653452,
      "fcm_dpo/beta": 0.14546489715576172,
      "fcm_dpo/delta": -0.19717153906822205,
      "fcm_dpo/margin": 7.054594993591309,
      "fcm_dpo/q_t": 0.3090992569923401,
      "grad_norm": 29.339309692382812,
      "learning_rate": 4.882355001067891e-07,
      "logits/chosen": -0.4405710697174072,
      "logits/rejected": -0.1895657330751419,
      "logps/chosen": -74.39476013183594,
      "logps/ref_chosen": -73.18368530273438,
      "logps/ref_rejected": -88.02154541015625,
      "logps/rejected": -96.28720092773438,
      "loss": 0.8141,
      "margin_dpo/margin_mean": 7.054594993591309,
      "margin_dpo/margin_std": 7.372723579406738,
      "step": 130
    },
    {
      "epoch": 0.19236417033773862,
      "fcm_dpo/beta": 0.14382612705230713,
      "fcm_dpo/delta": 0.0152159184217453,
      "fcm_dpo/margin": 5.799963474273682,
      "fcm_dpo/q_t": 0.33881255984306335,
      "grad_norm": 29.034713745117188,
      "learning_rate": 4.878433179298909e-07,
      "logits/chosen": -0.49546778202056885,
      "logits/rejected": -0.2850472331047058,
      "logps/chosen": -74.73524475097656,
      "logps/ref_chosen": -73.9564437866211,
      "logps/ref_rejected": -94.45582580566406,
      "logps/rejected": -101.03459167480469,
      "loss": 0.8991,
      "margin_dpo/margin_mean": 5.799962997436523,
      "margin_dpo/margin_std": 6.754213333129883,
      "step": 131
    },
    {
      "epoch": 0.19383259911894274,
      "fcm_dpo/beta": 0.14022360742092133,
      "fcm_dpo/delta": -0.07685554027557373,
      "fcm_dpo/margin": 6.553079605102539,
      "fcm_dpo/q_t": 0.31034043431282043,
      "grad_norm": 31.93946075439453,
      "learning_rate": 4.874448683603694e-07,
      "logits/chosen": -0.5433443188667297,
      "logits/rejected": -0.17930719256401062,
      "logps/chosen": -93.70853424072266,
      "logps/ref_chosen": -92.43948364257812,
      "logps/ref_rejected": -94.13252258300781,
      "logps/rejected": -101.95465087890625,
      "loss": 0.8211,
      "margin_dpo/margin_mean": 6.553079605102539,
      "margin_dpo/margin_std": 6.320675849914551,
      "step": 132
    },
    {
      "epoch": 0.19530102790014683,
      "fcm_dpo/beta": 0.13906362652778625,
      "fcm_dpo/delta": -0.09275015443563461,
      "fcm_dpo/margin": 6.717145919799805,
      "fcm_dpo/q_t": 0.31701478362083435,
      "grad_norm": 29.01922607421875,
      "learning_rate": 4.870401618977415e-07,
      "logits/chosen": -0.8589807748794556,
      "logits/rejected": -0.5169089436531067,
      "logps/chosen": -93.42756652832031,
      "logps/ref_chosen": -93.11666107177734,
      "logps/ref_rejected": -102.09490966796875,
      "logps/rejected": -109.12296295166016,
      "loss": 0.8198,
      "margin_dpo/margin_mean": 6.717145919799805,
      "margin_dpo/margin_std": 6.780609130859375,
      "step": 133
    },
    {
      "epoch": 0.19676945668135096,
      "fcm_dpo/beta": 0.13517756760120392,
      "fcm_dpo/delta": -0.17839491367340088,
      "fcm_dpo/margin": 7.475937366485596,
      "fcm_dpo/q_t": 0.2990626096725464,
      "grad_norm": 26.34817886352539,
      "learning_rate": 4.866292092063986e-07,
      "logits/chosen": -1.06900954246521,
      "logits/rejected": -0.7887668609619141,
      "logps/chosen": -84.31695556640625,
      "logps/ref_chosen": -83.55012512207031,
      "logps/ref_rejected": -93.8815689086914,
      "logps/rejected": -102.12433624267578,
      "loss": 0.777,
      "margin_dpo/margin_mean": 7.475937366485596,
      "margin_dpo/margin_std": 6.868247985839844,
      "step": 134
    },
    {
      "epoch": 0.19823788546255505,
      "fcm_dpo/beta": 0.13384297490119934,
      "fcm_dpo/delta": 0.0007069706916809082,
      "fcm_dpo/margin": 6.340190887451172,
      "fcm_dpo/q_t": 0.3378380239009857,
      "grad_norm": 36.35404586791992,
      "learning_rate": 4.862120211153265e-07,
      "logits/chosen": -0.6139770746231079,
      "logits/rejected": -0.4515921473503113,
      "logps/chosen": -78.59407043457031,
      "logps/ref_chosen": -77.58262634277344,
      "logps/ref_rejected": -121.05648040771484,
      "logps/rejected": -128.4081268310547,
      "loss": 0.8986,
      "margin_dpo/margin_mean": 6.34019136428833,
      "margin_dpo/margin_std": 7.819980144500732,
      "step": 135
    },
    {
      "epoch": 0.19970631424375918,
      "fcm_dpo/beta": 0.1300199031829834,
      "fcm_dpo/delta": -0.14350877702236176,
      "fcm_dpo/margin": 7.531214714050293,
      "fcm_dpo/q_t": 0.29828688502311707,
      "grad_norm": 25.63352394104004,
      "learning_rate": 4.857886086178193e-07,
      "logits/chosen": -0.6523430347442627,
      "logits/rejected": -0.24616163969039917,
      "logps/chosen": -94.6618423461914,
      "logps/ref_chosen": -94.1727294921875,
      "logps/ref_rejected": -103.26428985595703,
      "logps/rejected": -111.28461456298828,
      "loss": 0.7668,
      "margin_dpo/margin_mean": 7.531215190887451,
      "margin_dpo/margin_std": 6.398444175720215,
      "step": 136
    },
    {
      "epoch": 0.2011747430249633,
      "fcm_dpo/beta": 0.11987289786338806,
      "fcm_dpo/delta": -0.43336063623428345,
      "fcm_dpo/margin": 10.205141067504883,
      "fcm_dpo/q_t": 0.2635684013366699,
      "grad_norm": 27.441190719604492,
      "learning_rate": 4.853589828711902e-07,
      "logits/chosen": -0.8289228677749634,
      "logits/rejected": -0.5975626707077026,
      "logps/chosen": -78.52019500732422,
      "logps/ref_chosen": -78.11874389648438,
      "logps/ref_rejected": -117.92237091064453,
      "logps/rejected": -128.52896118164062,
      "loss": 0.6749,
      "margin_dpo/margin_mean": 10.205141067504883,
      "margin_dpo/margin_std": 8.54926872253418,
      "step": 137
    },
    {
      "epoch": 0.2026431718061674,
      "fcm_dpo/beta": 0.11853648722171783,
      "fcm_dpo/delta": 0.01834501326084137,
      "fcm_dpo/margin": 7.02825403213501,
      "fcm_dpo/q_t": 0.3304741680622101,
      "grad_norm": 24.61989974975586,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": -0.22558237612247467,
      "logits/rejected": -0.01865313947200775,
      "logps/chosen": -94.71937561035156,
      "logps/ref_chosen": -92.82994079589844,
      "logps/ref_rejected": -98.64840698242188,
      "logps/rejected": -107.56608581542969,
      "loss": 0.8528,
      "margin_dpo/margin_mean": 7.02825403213501,
      "margin_dpo/margin_std": 6.998868942260742,
      "step": 138
    },
    {
      "epoch": 0.20411160058737152,
      "fcm_dpo/beta": 0.11699052155017853,
      "fcm_dpo/delta": -0.09517881274223328,
      "fcm_dpo/margin": 8.000020980834961,
      "fcm_dpo/q_t": 0.30926352739334106,
      "grad_norm": 25.54629898071289,
      "learning_rate": 4.844811370781446e-07,
      "logits/chosen": -0.6505274772644043,
      "logits/rejected": -0.3552773594856262,
      "logps/chosen": -79.84962463378906,
      "logps/ref_chosen": -79.00301361083984,
      "logps/ref_rejected": -85.82733154296875,
      "logps/rejected": -94.67395782470703,
      "loss": 0.8347,
      "margin_dpo/margin_mean": 8.000020980834961,
      "margin_dpo/margin_std": 8.218055725097656,
      "step": 139
    },
    {
      "epoch": 0.2055800293685756,
      "fcm_dpo/beta": 0.11667490005493164,
      "fcm_dpo/delta": -0.006098256446421146,
      "fcm_dpo/margin": 7.332958221435547,
      "fcm_dpo/q_t": 0.3275277018547058,
      "grad_norm": 24.43271827697754,
      "learning_rate": 4.840329401637809e-07,
      "logits/chosen": -0.656264066696167,
      "logits/rejected": -0.28980007767677307,
      "logps/chosen": -89.24510192871094,
      "logps/ref_chosen": -88.35244750976562,
      "logps/ref_rejected": -90.57476806640625,
      "logps/rejected": -98.80037689208984,
      "loss": 0.8603,
      "margin_dpo/margin_mean": 7.332958221435547,
      "margin_dpo/margin_std": 7.586727619171143,
      "step": 140
    },
    {
      "epoch": 0.20704845814977973,
      "fcm_dpo/beta": 0.1137249618768692,
      "fcm_dpo/delta": -0.17630261182785034,
      "fcm_dpo/margin": 8.869564056396484,
      "fcm_dpo/q_t": 0.29895448684692383,
      "grad_norm": 23.59160804748535,
      "learning_rate": 4.83578576263792e-07,
      "logits/chosen": -0.40679359436035156,
      "logits/rejected": -0.06718328595161438,
      "logps/chosen": -108.21107482910156,
      "logps/ref_chosen": -107.22016143798828,
      "logps/ref_rejected": -104.71223449707031,
      "logps/rejected": -114.57270812988281,
      "loss": 0.7693,
      "margin_dpo/margin_mean": 8.8695650100708,
      "margin_dpo/margin_std": 8.169231414794922,
      "step": 141
    },
    {
      "epoch": 0.20851688693098386,
      "fcm_dpo/beta": 0.11140327155590057,
      "fcm_dpo/delta": -0.03288538381457329,
      "fcm_dpo/margin": 7.893212795257568,
      "fcm_dpo/q_t": 0.3255746364593506,
      "grad_norm": 25.964059829711914,
      "learning_rate": 4.83118057351089e-07,
      "logits/chosen": -0.4473114609718323,
      "logits/rejected": -0.21175801753997803,
      "logps/chosen": -90.84941101074219,
      "logps/ref_chosen": -88.72894287109375,
      "logps/ref_rejected": -99.73038482666016,
      "logps/rejected": -109.74406433105469,
      "loss": 0.8421,
      "margin_dpo/margin_mean": 7.893212795257568,
      "margin_dpo/margin_std": 8.307409286499023,
      "step": 142
    },
    {
      "epoch": 0.20998531571218795,
      "fcm_dpo/beta": 0.11170312762260437,
      "fcm_dpo/delta": 0.037874262779951096,
      "fcm_dpo/margin": 7.293005466461182,
      "fcm_dpo/q_t": 0.33844050765037537,
      "grad_norm": 23.84907341003418,
      "learning_rate": 4.826513955607734e-07,
      "logits/chosen": -0.87675940990448,
      "logits/rejected": -0.46402356028556824,
      "logps/chosen": -85.97489929199219,
      "logps/ref_chosen": -85.50457763671875,
      "logps/ref_rejected": -85.8056640625,
      "logps/rejected": -93.56900024414062,
      "loss": 0.9032,
      "margin_dpo/margin_mean": 7.293004989624023,
      "margin_dpo/margin_std": 8.4471435546875,
      "step": 143
    },
    {
      "epoch": 0.21145374449339208,
      "fcm_dpo/beta": 0.11300264298915863,
      "fcm_dpo/delta": -0.06249871850013733,
      "fcm_dpo/margin": 8.005437850952148,
      "fcm_dpo/q_t": 0.3288303315639496,
      "grad_norm": 29.966264724731445,
      "learning_rate": 4.821786031898176e-07,
      "logits/chosen": -0.6753568053245544,
      "logits/rejected": -0.2179480940103531,
      "logps/chosen": -105.02165222167969,
      "logps/ref_chosen": -104.68708801269531,
      "logps/ref_rejected": -87.71908569335938,
      "logps/rejected": -96.05908203125,
      "loss": 0.8923,
      "margin_dpo/margin_mean": 8.005437850952148,
      "margin_dpo/margin_std": 9.631026268005371,
      "step": 144
    },
    {
      "epoch": 0.21292217327459617,
      "fcm_dpo/beta": 0.10966965556144714,
      "fcm_dpo/delta": -0.08353471755981445,
      "fcm_dpo/margin": 8.439840316772461,
      "fcm_dpo/q_t": 0.3153953552246094,
      "grad_norm": 25.04039764404297,
      "learning_rate": 4.816996926967401e-07,
      "logits/chosen": -0.5359320044517517,
      "logits/rejected": 0.0038733258843421936,
      "logps/chosen": -89.20046997070312,
      "logps/ref_chosen": -88.51988983154297,
      "logps/ref_rejected": -83.76513671875,
      "logps/rejected": -92.88556671142578,
      "loss": 0.8383,
      "margin_dpo/margin_mean": 8.439842224121094,
      "margin_dpo/margin_std": 8.859256744384766,
      "step": 145
    },
    {
      "epoch": 0.2143906020558003,
      "fcm_dpo/beta": 0.10515066981315613,
      "fcm_dpo/delta": -0.19317114353179932,
      "fcm_dpo/margin": 9.692916870117188,
      "fcm_dpo/q_t": 0.2934582829475403,
      "grad_norm": 22.523944854736328,
      "learning_rate": 4.812146767012779e-07,
      "logits/chosen": -0.8558279275894165,
      "logits/rejected": -0.34566935896873474,
      "logps/chosen": -103.20064544677734,
      "logps/ref_chosen": -102.90586853027344,
      "logps/ref_rejected": -88.9493637084961,
      "logps/rejected": -98.93706512451172,
      "loss": 0.7536,
      "margin_dpo/margin_mean": 9.692916870117188,
      "margin_dpo/margin_std": 8.176064491271973,
      "step": 146
    },
    {
      "epoch": 0.21585903083700442,
      "fcm_dpo/beta": 0.10192033648490906,
      "fcm_dpo/delta": -0.21661145985126495,
      "fcm_dpo/margin": 10.242698669433594,
      "fcm_dpo/q_t": 0.2886643707752228,
      "grad_norm": 21.00252342224121,
      "learning_rate": 4.807235679840536e-07,
      "logits/chosen": -0.9141993522644043,
      "logits/rejected": -0.40335512161254883,
      "logps/chosen": -90.03712463378906,
      "logps/ref_chosen": -89.26219177246094,
      "logps/ref_rejected": -76.50011444091797,
      "logps/rejected": -87.51775360107422,
      "loss": 0.754,
      "margin_dpo/margin_mean": 10.242698669433594,
      "margin_dpo/margin_std": 8.800421714782715,
      "step": 147
    },
    {
      "epoch": 0.2173274596182085,
      "fcm_dpo/beta": 0.10235866904258728,
      "fcm_dpo/delta": 0.02111022174358368,
      "fcm_dpo/margin": 8.084315299987793,
      "fcm_dpo/q_t": 0.338812917470932,
      "grad_norm": 25.80347442626953,
      "learning_rate": 4.802263794862384e-07,
      "logits/chosen": -0.9320132732391357,
      "logits/rejected": -0.5644128322601318,
      "logps/chosen": -91.48237609863281,
      "logps/ref_chosen": -91.76287841796875,
      "logps/ref_rejected": -110.38552856445312,
      "logps/rejected": -118.18933868408203,
      "loss": 0.8942,
      "margin_dpo/margin_mean": 8.08431625366211,
      "margin_dpo/margin_std": 9.086051940917969,
      "step": 148
    },
    {
      "epoch": 0.21879588839941264,
      "fcm_dpo/beta": 0.10116075724363327,
      "fcm_dpo/delta": 0.01745833456516266,
      "fcm_dpo/margin": 8.242944717407227,
      "fcm_dpo/q_t": 0.3417103588581085,
      "grad_norm": 25.629846572875977,
      "learning_rate": 4.797231243092118e-07,
      "logits/chosen": -0.5707268714904785,
      "logits/rejected": -0.29623883962631226,
      "logps/chosen": -94.82536315917969,
      "logps/ref_chosen": -93.48401641845703,
      "logps/ref_rejected": -105.6436996459961,
      "logps/rejected": -115.22799682617188,
      "loss": 0.9025,
      "margin_dpo/margin_mean": 8.24294662475586,
      "margin_dpo/margin_std": 9.996318817138672,
      "step": 149
    },
    {
      "epoch": 0.22026431718061673,
      "fcm_dpo/beta": 0.10018637031316757,
      "fcm_dpo/delta": -0.07022020220756531,
      "fcm_dpo/margin": 9.116851806640625,
      "fcm_dpo/q_t": 0.3224765360355377,
      "grad_norm": 21.188783645629883,
      "learning_rate": 4.792138157142157e-07,
      "logits/chosen": -0.36832523345947266,
      "logits/rejected": -0.0916675478219986,
      "logps/chosen": -76.63916015625,
      "logps/ref_chosen": -75.7786865234375,
      "logps/ref_rejected": -90.02676391601562,
      "logps/rejected": -100.00409698486328,
      "loss": 0.8448,
      "margin_dpo/margin_mean": 9.116851806640625,
      "margin_dpo/margin_std": 10.08909797668457,
      "step": 150
    },
    {
      "epoch": 0.22173274596182085,
      "fcm_dpo/beta": 0.10023315250873566,
      "fcm_dpo/delta": -0.0017028320580720901,
      "fcm_dpo/margin": 8.495565414428711,
      "fcm_dpo/q_t": 0.3360986113548279,
      "grad_norm": 28.195302963256836,
      "learning_rate": 4.786984671220053e-07,
      "logits/chosen": -1.2456610202789307,
      "logits/rejected": -0.7084729671478271,
      "logps/chosen": -104.43856811523438,
      "logps/ref_chosen": -102.73422241210938,
      "logps/ref_rejected": -106.03781127929688,
      "logps/rejected": -116.23772430419922,
      "loss": 0.8976,
      "margin_dpo/margin_mean": 8.495565414428711,
      "margin_dpo/margin_std": 10.185565948486328,
      "step": 151
    },
    {
      "epoch": 0.22320117474302498,
      "fcm_dpo/beta": 0.0979384109377861,
      "fcm_dpo/delta": -0.1397112011909485,
      "fcm_dpo/margin": 9.964031219482422,
      "fcm_dpo/q_t": 0.30377432703971863,
      "grad_norm": 23.11962127685547,
      "learning_rate": 4.78177092112495e-07,
      "logits/chosen": -0.8004452586174011,
      "logits/rejected": -0.4601547420024872,
      "logps/chosen": -97.12671661376953,
      "logps/ref_chosen": -97.59074401855469,
      "logps/ref_rejected": -110.49913024902344,
      "logps/rejected": -119.9991455078125,
      "loss": 0.7848,
      "margin_dpo/margin_mean": 9.964032173156738,
      "margin_dpo/margin_std": 9.50239372253418,
      "step": 152
    },
    {
      "epoch": 0.22466960352422907,
      "fcm_dpo/beta": 0.0991874635219574,
      "fcm_dpo/delta": 0.12609340250492096,
      "fcm_dpo/margin": 7.391656875610352,
      "fcm_dpo/q_t": 0.3547060787677765,
      "grad_norm": 22.816524505615234,
      "learning_rate": 4.776497044244016e-07,
      "logits/chosen": -1.251391887664795,
      "logits/rejected": -0.9469621181488037,
      "logps/chosen": -88.66514587402344,
      "logps/ref_chosen": -87.02560424804688,
      "logps/ref_rejected": -91.06854248046875,
      "logps/rejected": -100.09973907470703,
      "loss": 0.9558,
      "margin_dpo/margin_mean": 7.391656875610352,
      "margin_dpo/margin_std": 9.42247200012207,
      "step": 153
    },
    {
      "epoch": 0.2261380323054332,
      "fcm_dpo/beta": 0.096708282828331,
      "fcm_dpo/delta": -0.32071927189826965,
      "fcm_dpo/margin": 11.730560302734375,
      "fcm_dpo/q_t": 0.2757687270641327,
      "grad_norm": 22.773046493530273,
      "learning_rate": 4.771163179548808e-07,
      "logits/chosen": -0.9404456615447998,
      "logits/rejected": -0.6278473734855652,
      "logps/chosen": -91.3301010131836,
      "logps/ref_chosen": -91.10566711425781,
      "logps/ref_rejected": -111.37238311767578,
      "logps/rejected": -123.32737731933594,
      "loss": 0.7088,
      "margin_dpo/margin_mean": 11.730561256408691,
      "margin_dpo/margin_std": 9.345605850219727,
      "step": 154
    },
    {
      "epoch": 0.2276064610866373,
      "fcm_dpo/beta": 0.09173603355884552,
      "fcm_dpo/delta": -0.07401001453399658,
      "fcm_dpo/margin": 9.972280502319336,
      "fcm_dpo/q_t": 0.3208170533180237,
      "grad_norm": 28.64071273803711,
      "learning_rate": 4.7657694675916247e-07,
      "logits/chosen": -0.5264374017715454,
      "logits/rejected": -0.19909337162971497,
      "logps/chosen": -95.87799072265625,
      "logps/ref_chosen": -94.56097412109375,
      "logps/ref_rejected": -84.77359008789062,
      "logps/rejected": -96.06289672851562,
      "loss": 0.8954,
      "margin_dpo/margin_mean": 9.972282409667969,
      "margin_dpo/margin_std": 11.702651977539062,
      "step": 155
    },
    {
      "epoch": 0.2290748898678414,
      "fcm_dpo/beta": 0.0929301306605339,
      "fcm_dpo/delta": 0.06338099390268326,
      "fcm_dpo/margin": 8.519001007080078,
      "fcm_dpo/q_t": 0.3422147035598755,
      "grad_norm": 27.464357376098633,
      "learning_rate": 4.7603160505017893e-07,
      "logits/chosen": -0.9737367630004883,
      "logits/rejected": -0.5267705321311951,
      "logps/chosen": -97.08377075195312,
      "logps/ref_chosen": -94.65538787841797,
      "logps/ref_rejected": -84.41934204101562,
      "logps/rejected": -95.36672973632812,
      "loss": 0.918,
      "margin_dpo/margin_mean": 8.519001007080078,
      "margin_dpo/margin_std": 10.034339904785156,
      "step": 156
    },
    {
      "epoch": 0.2305433186490455,
      "fcm_dpo/beta": 0.09153124690055847,
      "fcm_dpo/delta": -0.16219165921211243,
      "fcm_dpo/margin": 10.88486385345459,
      "fcm_dpo/q_t": 0.30015528202056885,
      "grad_norm": 23.287403106689453,
      "learning_rate": 4.7548030719819154e-07,
      "logits/chosen": -1.3203184604644775,
      "logits/rejected": -0.8898429870605469,
      "logps/chosen": -99.07041931152344,
      "logps/ref_chosen": -97.01239013671875,
      "logps/ref_rejected": -111.70133972167969,
      "logps/rejected": -124.64424133300781,
      "loss": 0.7785,
      "margin_dpo/margin_mean": 10.884864807128906,
      "margin_dpo/margin_std": 9.999959945678711,
      "step": 157
    },
    {
      "epoch": 0.23201174743024963,
      "fcm_dpo/beta": 0.0887395441532135,
      "fcm_dpo/delta": -0.15405318140983582,
      "fcm_dpo/margin": 11.145757675170898,
      "fcm_dpo/q_t": 0.297393798828125,
      "grad_norm": 23.948062896728516,
      "learning_rate": 4.7492306773041136e-07,
      "logits/chosen": -1.0246334075927734,
      "logits/rejected": -0.7327722311019897,
      "logps/chosen": -96.69784545898438,
      "logps/ref_chosen": -94.71296691894531,
      "logps/ref_rejected": -120.93733978271484,
      "logps/rejected": -134.06797790527344,
      "loss": 0.7641,
      "margin_dpo/margin_mean": 11.145757675170898,
      "margin_dpo/margin_std": 9.565597534179688,
      "step": 158
    },
    {
      "epoch": 0.23348017621145375,
      "fcm_dpo/beta": 0.08581481873989105,
      "fcm_dpo/delta": -0.12313660234212875,
      "fcm_dpo/margin": 11.194160461425781,
      "fcm_dpo/q_t": 0.31181782484054565,
      "grad_norm": 20.498376846313477,
      "learning_rate": 4.743599013306165e-07,
      "logits/chosen": -1.4571746587753296,
      "logits/rejected": -1.0124716758728027,
      "logps/chosen": -109.18254852294922,
      "logps/ref_chosen": -108.2182388305664,
      "logps/ref_rejected": -94.83901977539062,
      "logps/rejected": -106.99748229980469,
      "loss": 0.8284,
      "margin_dpo/margin_mean": 11.194160461425781,
      "margin_dpo/margin_std": 11.655582427978516,
      "step": 159
    },
    {
      "epoch": 0.23494860499265785,
      "fcm_dpo/beta": 0.08508424460887909,
      "fcm_dpo/delta": -0.025002997368574142,
      "fcm_dpo/margin": 10.25672721862793,
      "fcm_dpo/q_t": 0.3192032277584076,
      "grad_norm": 20.28183364868164,
      "learning_rate": 4.737908228387656e-07,
      "logits/chosen": -1.2797681093215942,
      "logits/rejected": -0.7797454595565796,
      "logps/chosen": -98.57347106933594,
      "logps/ref_chosen": -98.16340637207031,
      "logps/ref_rejected": -102.28097534179688,
      "logps/rejected": -112.9477767944336,
      "loss": 0.8396,
      "margin_dpo/margin_mean": 10.25672721862793,
      "margin_dpo/margin_std": 9.986027717590332,
      "step": 160
    },
    {
      "epoch": 0.23641703377386197,
      "fcm_dpo/beta": 0.0846460685133934,
      "fcm_dpo/delta": 0.005213946104049683,
      "fcm_dpo/margin": 9.975399017333984,
      "fcm_dpo/q_t": 0.3322407305240631,
      "grad_norm": 22.53128433227539,
      "learning_rate": 4.7321584725060594e-07,
      "logits/chosen": -1.2773491144180298,
      "logits/rejected": -0.8251501321792603,
      "logps/chosen": -88.7325439453125,
      "logps/ref_chosen": -86.59187316894531,
      "logps/ref_rejected": -89.01643371582031,
      "logps/rejected": -101.13250732421875,
      "loss": 0.8939,
      "margin_dpo/margin_mean": 9.975399017333984,
      "margin_dpo/margin_std": 11.190435409545898,
      "step": 161
    },
    {
      "epoch": 0.23788546255506607,
      "fcm_dpo/beta": 0.08452028781175613,
      "fcm_dpo/delta": -0.03935426473617554,
      "fcm_dpo/margin": 10.478342056274414,
      "fcm_dpo/q_t": 0.3177115321159363,
      "grad_norm": 20.945478439331055,
      "learning_rate": 4.7263498971727905e-07,
      "logits/chosen": -0.6811781525611877,
      "logits/rejected": -0.29194605350494385,
      "logps/chosen": -98.99219512939453,
      "logps/ref_chosen": -97.40288543701172,
      "logps/ref_rejected": -95.86923217773438,
      "logps/rejected": -107.93688201904297,
      "loss": 0.8561,
      "margin_dpo/margin_mean": 10.478342056274414,
      "margin_dpo/margin_std": 10.870805740356445,
      "step": 162
    },
    {
      "epoch": 0.2393538913362702,
      "fcm_dpo/beta": 0.08143004029989243,
      "fcm_dpo/delta": -0.242702916264534,
      "fcm_dpo/margin": 13.094063758850098,
      "fcm_dpo/q_t": 0.2967764139175415,
      "grad_norm": 17.559616088867188,
      "learning_rate": 4.720482655449212e-07,
      "logits/chosen": -1.1392264366149902,
      "logits/rejected": -0.7611545324325562,
      "logps/chosen": -83.36274719238281,
      "logps/ref_chosen": -82.1753921508789,
      "logps/ref_rejected": -82.86473846435547,
      "logps/rejected": -97.14615631103516,
      "loss": 0.7736,
      "margin_dpo/margin_mean": 13.094064712524414,
      "margin_dpo/margin_std": 13.233580589294434,
      "step": 163
    },
    {
      "epoch": 0.24082232011747431,
      "fcm_dpo/beta": 0.08133898675441742,
      "fcm_dpo/delta": 0.14469550549983978,
      "fcm_dpo/margin": 8.78353214263916,
      "fcm_dpo/q_t": 0.3501974046230316,
      "grad_norm": 20.816787719726562,
      "learning_rate": 4.714556901942599e-07,
      "logits/chosen": -0.9682873487472534,
      "logits/rejected": -0.5573686957359314,
      "logps/chosen": -94.12632751464844,
      "logps/ref_chosen": -90.47427368164062,
      "logps/ref_rejected": -87.02809143066406,
      "logps/rejected": -99.46368408203125,
      "loss": 0.9361,
      "margin_dpo/margin_mean": 8.783533096313477,
      "margin_dpo/margin_std": 9.51855182647705,
      "step": 164
    },
    {
      "epoch": 0.2422907488986784,
      "fcm_dpo/beta": 0.08096069097518921,
      "fcm_dpo/delta": -0.20310097932815552,
      "fcm_dpo/margin": 12.753755569458008,
      "fcm_dpo/q_t": 0.3054801821708679,
      "grad_norm": 18.420368194580078,
      "learning_rate": 4.708572792802069e-07,
      "logits/chosen": -1.2523810863494873,
      "logits/rejected": -0.8975613117218018,
      "logps/chosen": -88.87324523925781,
      "logps/ref_chosen": -87.313232421875,
      "logps/ref_rejected": -81.902587890625,
      "logps/rejected": -96.21635437011719,
      "loss": 0.7986,
      "margin_dpo/margin_mean": 12.753755569458008,
      "margin_dpo/margin_std": 14.236822128295898,
      "step": 165
    },
    {
      "epoch": 0.24375917767988253,
      "fcm_dpo/beta": 0.0783786028623581,
      "fcm_dpo/delta": -0.11244317889213562,
      "fcm_dpo/margin": 12.143659591674805,
      "fcm_dpo/q_t": 0.322204053401947,
      "grad_norm": 20.77262306213379,
      "learning_rate": 4.702530485714461e-07,
      "logits/chosen": -0.9464759230613708,
      "logits/rejected": -0.7219276428222656,
      "logps/chosen": -86.5793228149414,
      "logps/ref_chosen": -81.58606719970703,
      "logps/ref_rejected": -103.16517639160156,
      "logps/rejected": -120.30209350585938,
      "loss": 0.8471,
      "margin_dpo/margin_mean": 12.143659591674805,
      "margin_dpo/margin_std": 14.19489860534668,
      "step": 166
    },
    {
      "epoch": 0.24522760646108663,
      "fcm_dpo/beta": 0.0751272439956665,
      "fcm_dpo/delta": -0.183059424161911,
      "fcm_dpo/margin": 13.46567440032959,
      "fcm_dpo/q_t": 0.2956295907497406,
      "grad_norm": 20.29990005493164,
      "learning_rate": 4.6964301399001877e-07,
      "logits/chosen": -0.7697356939315796,
      "logits/rejected": -0.5615593194961548,
      "logps/chosen": -87.39271545410156,
      "logps/ref_chosen": -83.56167602539062,
      "logps/ref_rejected": -99.05305480957031,
      "logps/rejected": -116.34976959228516,
      "loss": 0.7512,
      "margin_dpo/margin_mean": 13.465675354003906,
      "margin_dpo/margin_std": 11.612232208251953,
      "step": 167
    },
    {
      "epoch": 0.24669603524229075,
      "fcm_dpo/beta": 0.07545551657676697,
      "fcm_dpo/delta": 0.006653144955635071,
      "fcm_dpo/margin": 11.17927360534668,
      "fcm_dpo/q_t": 0.33007046580314636,
      "grad_norm": 22.19552230834961,
      "learning_rate": 4.690271916109034e-07,
      "logits/chosen": -0.9670852422714233,
      "logits/rejected": -0.6387625336647034,
      "logps/chosen": -83.56878662109375,
      "logps/ref_chosen": -79.93123626708984,
      "logps/ref_rejected": -80.10216522216797,
      "logps/rejected": -94.91899108886719,
      "loss": 0.8696,
      "margin_dpo/margin_mean": 11.17927360534668,
      "margin_dpo/margin_std": 11.96432113647461,
      "step": 168
    },
    {
      "epoch": 0.24816446402349487,
      "fcm_dpo/beta": 0.07446470856666565,
      "fcm_dpo/delta": -0.08862879127264023,
      "fcm_dpo/margin": 12.494746208190918,
      "fcm_dpo/q_t": 0.3135685920715332,
      "grad_norm": 20.065807342529297,
      "learning_rate": 4.6840559766159235e-07,
      "logits/chosen": -0.9137426018714905,
      "logits/rejected": -0.6862479448318481,
      "logps/chosen": -85.2474365234375,
      "logps/ref_chosen": -82.90370178222656,
      "logps/ref_rejected": -90.12191009521484,
      "logps/rejected": -104.96038818359375,
      "loss": 0.8135,
      "margin_dpo/margin_mean": 12.494745254516602,
      "margin_dpo/margin_std": 12.275469779968262,
      "step": 169
    },
    {
      "epoch": 0.24963289280469897,
      "fcm_dpo/beta": 0.07337189465761185,
      "fcm_dpo/delta": 0.003176748752593994,
      "fcm_dpo/margin": 11.533134460449219,
      "fcm_dpo/q_t": 0.3243024945259094,
      "grad_norm": 20.527284622192383,
      "learning_rate": 4.6777824852166437e-07,
      "logits/chosen": -1.0314369201660156,
      "logits/rejected": -0.7973779439926147,
      "logps/chosen": -79.22702026367188,
      "logps/ref_chosen": -74.98197174072266,
      "logps/ref_rejected": -82.16560363769531,
      "logps/rejected": -97.94378662109375,
      "loss": 0.8447,
      "margin_dpo/margin_mean": 11.533134460449219,
      "margin_dpo/margin_std": 10.969135284423828,
      "step": 170
    },
    {
      "epoch": 0.2511013215859031,
      "fcm_dpo/beta": 0.07268837094306946,
      "fcm_dpo/delta": -0.11282499134540558,
      "fcm_dpo/margin": 13.098483085632324,
      "fcm_dpo/q_t": 0.32213613390922546,
      "grad_norm": 22.617277145385742,
      "learning_rate": 4.6714516072235273e-07,
      "logits/chosen": -1.1407670974731445,
      "logits/rejected": -0.6412278413772583,
      "logps/chosen": -123.88941955566406,
      "logps/ref_chosen": -118.97853088378906,
      "logps/ref_rejected": -115.76930236816406,
      "logps/rejected": -133.77867126464844,
      "loss": 0.8676,
      "margin_dpo/margin_mean": 13.098483085632324,
      "margin_dpo/margin_std": 15.950417518615723,
      "step": 171
    },
    {
      "epoch": 0.2525697503671072,
      "fcm_dpo/beta": 0.0713869035243988,
      "fcm_dpo/delta": -0.04091132432222366,
      "fcm_dpo/margin": 12.417065620422363,
      "fcm_dpo/q_t": 0.30859920382499695,
      "grad_norm": 16.54987907409668,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": -1.8607536554336548,
      "logits/rejected": -1.2343878746032715,
      "logps/chosen": -106.86568450927734,
      "logps/ref_chosen": -101.15379333496094,
      "logps/ref_rejected": -91.00568389892578,
      "logps/rejected": -109.1346435546875,
      "loss": 0.7824,
      "margin_dpo/margin_mean": 12.417064666748047,
      "margin_dpo/margin_std": 9.314888000488281,
      "step": 172
    },
    {
      "epoch": 0.2540381791483113,
      "fcm_dpo/beta": 0.07276591658592224,
      "fcm_dpo/delta": 0.10682529211044312,
      "fcm_dpo/margin": 10.32400131225586,
      "fcm_dpo/q_t": 0.34479019045829773,
      "grad_norm": 18.29812240600586,
      "learning_rate": 4.6586183602616687e-07,
      "logits/chosen": -0.8446004390716553,
      "logits/rejected": -0.4139299988746643,
      "logps/chosen": -104.5085678100586,
      "logps/ref_chosen": -99.05668640136719,
      "logps/ref_rejected": -83.8507308959961,
      "logps/rejected": -99.62661743164062,
      "loss": 0.9065,
      "margin_dpo/margin_mean": 10.32400131225586,
      "margin_dpo/margin_std": 11.158248901367188,
      "step": 173
    },
    {
      "epoch": 0.2555066079295154,
      "fcm_dpo/beta": 0.07212045043706894,
      "fcm_dpo/delta": -0.11132963001728058,
      "fcm_dpo/margin": 13.184260368347168,
      "fcm_dpo/q_t": 0.3065446615219116,
      "grad_norm": 21.124048233032227,
      "learning_rate": 4.652116329460919e-07,
      "logits/chosen": -1.4102823734283447,
      "logits/rejected": -1.3270244598388672,
      "logps/chosen": -85.65483856201172,
      "logps/ref_chosen": -80.40890502929688,
      "logps/ref_rejected": -108.20171356201172,
      "logps/rejected": -126.63190460205078,
      "loss": 0.8031,
      "margin_dpo/margin_mean": 13.184259414672852,
      "margin_dpo/margin_std": 12.659872055053711,
      "step": 174
    },
    {
      "epoch": 0.25697503671071953,
      "fcm_dpo/beta": 0.07079780846834183,
      "fcm_dpo/delta": -0.12347761541604996,
      "fcm_dpo/margin": 13.580202102661133,
      "fcm_dpo/q_t": 0.30228978395462036,
      "grad_norm": 21.37623405456543,
      "learning_rate": 4.645557588393406e-07,
      "logits/chosen": -0.736810564994812,
      "logits/rejected": -0.47296372056007385,
      "logps/chosen": -90.78504943847656,
      "logps/ref_chosen": -82.71649169921875,
      "logps/ref_rejected": -96.66712951660156,
      "logps/rejected": -118.31588745117188,
      "loss": 0.7768,
      "margin_dpo/margin_mean": 13.580202102661133,
      "margin_dpo/margin_std": 11.554082870483398,
      "step": 175
    },
    {
      "epoch": 0.25844346549192365,
      "fcm_dpo/beta": 0.06876523047685623,
      "fcm_dpo/delta": -0.11029569059610367,
      "fcm_dpo/margin": 13.813409805297852,
      "fcm_dpo/q_t": 0.3185492753982544,
      "grad_norm": 17.933225631713867,
      "learning_rate": 4.638942309888058e-07,
      "logits/chosen": -1.2823872566223145,
      "logits/rejected": -1.0857605934143066,
      "logps/chosen": -85.75364685058594,
      "logps/ref_chosen": -79.62930297851562,
      "logps/ref_rejected": -102.6885986328125,
      "logps/rejected": -122.62635040283203,
      "loss": 0.8524,
      "margin_dpo/margin_mean": 13.813409805297852,
      "margin_dpo/margin_std": 15.199995040893555,
      "step": 176
    },
    {
      "epoch": 0.2599118942731278,
      "fcm_dpo/beta": 0.06667226552963257,
      "fcm_dpo/delta": -0.22711145877838135,
      "fcm_dpo/margin": 15.801198959350586,
      "fcm_dpo/q_t": 0.3023749887943268,
      "grad_norm": 16.617761611938477,
      "learning_rate": 4.6322706682636137e-07,
      "logits/chosen": -1.1350969076156616,
      "logits/rejected": -0.7791039943695068,
      "logps/chosen": -99.27539825439453,
      "logps/ref_chosen": -92.56109619140625,
      "logps/ref_rejected": -103.33839416503906,
      "logps/rejected": -125.85388946533203,
      "loss": 0.7779,
      "margin_dpo/margin_mean": 15.801198959350586,
      "margin_dpo/margin_std": 16.495779037475586,
      "step": 177
    },
    {
      "epoch": 0.26138032305433184,
      "fcm_dpo/beta": 0.0618768036365509,
      "fcm_dpo/delta": -0.3499090075492859,
      "fcm_dpo/margin": 18.723678588867188,
      "fcm_dpo/q_t": 0.2736192047595978,
      "grad_norm": 19.030345916748047,
      "learning_rate": 4.6255428393240354e-07,
      "logits/chosen": -1.4820408821105957,
      "logits/rejected": -1.190525770187378,
      "logps/chosen": -90.46121215820312,
      "logps/ref_chosen": -84.26579284667969,
      "logps/ref_rejected": -110.79730987548828,
      "logps/rejected": -135.71640014648438,
      "loss": 0.6981,
      "margin_dpo/margin_mean": 18.723678588867188,
      "margin_dpo/margin_std": 16.16928482055664,
      "step": 178
    },
    {
      "epoch": 0.26284875183553597,
      "fcm_dpo/beta": 0.06120564043521881,
      "fcm_dpo/delta": 0.0761781707406044,
      "fcm_dpo/margin": 12.736023902893066,
      "fcm_dpo/q_t": 0.34814584255218506,
      "grad_norm": 19.742097854614258,
      "learning_rate": 4.6187590003538724e-07,
      "logits/chosen": -1.2575550079345703,
      "logits/rejected": -0.8413559198379517,
      "logps/chosen": -102.05105590820312,
      "logps/ref_chosen": -93.4479751586914,
      "logps/ref_rejected": -94.25541687011719,
      "logps/rejected": -115.59451293945312,
      "loss": 0.9468,
      "margin_dpo/margin_mean": 12.73602294921875,
      "margin_dpo/margin_std": 15.995349884033203,
      "step": 179
    },
    {
      "epoch": 0.2643171806167401,
      "fcm_dpo/beta": 0.06053383648395538,
      "fcm_dpo/delta": -0.10252348333597183,
      "fcm_dpo/margin": 15.557160377502441,
      "fcm_dpo/q_t": 0.3080785870552063,
      "grad_norm": 20.96506118774414,
      "learning_rate": 4.611919330113591e-07,
      "logits/chosen": -1.0803053379058838,
      "logits/rejected": -0.8237367272377014,
      "logps/chosen": -95.61346435546875,
      "logps/ref_chosen": -88.83554077148438,
      "logps/ref_rejected": -103.90696716308594,
      "logps/rejected": -126.24205017089844,
      "loss": 0.7905,
      "margin_dpo/margin_mean": 15.557161331176758,
      "margin_dpo/margin_std": 14.514302253723145,
      "step": 180
    },
    {
      "epoch": 0.2657856093979442,
      "fcm_dpo/beta": 0.06162776052951813,
      "fcm_dpo/delta": 0.10773488879203796,
      "fcm_dpo/margin": 12.171871185302734,
      "fcm_dpo/q_t": 0.34699884057044983,
      "grad_norm": 19.01604652404785,
      "learning_rate": 4.605024008834863e-07,
      "logits/chosen": -0.9782782793045044,
      "logits/rejected": -0.5275274515151978,
      "logps/chosen": -86.51460266113281,
      "logps/ref_chosen": -79.23396301269531,
      "logps/ref_rejected": -69.17442321777344,
      "logps/rejected": -88.6269302368164,
      "loss": 0.9037,
      "margin_dpo/margin_mean": 12.171870231628418,
      "margin_dpo/margin_std": 13.407722473144531,
      "step": 181
    },
    {
      "epoch": 0.26725403817914833,
      "fcm_dpo/beta": 0.06035807728767395,
      "fcm_dpo/delta": -0.1120065227150917,
      "fcm_dpo/margin": 15.727120399475098,
      "fcm_dpo/q_t": 0.3143818974494934,
      "grad_norm": 16.87702178955078,
      "learning_rate": 4.598073218215817e-07,
      "logits/chosen": -0.9282777905464172,
      "logits/rejected": -0.6541332006454468,
      "logps/chosen": -86.01435089111328,
      "logps/ref_chosen": -76.08218383789062,
      "logps/ref_rejected": -94.9686050415039,
      "logps/rejected": -120.62789916992188,
      "loss": 0.8428,
      "margin_dpo/margin_mean": 15.727121353149414,
      "margin_dpo/margin_std": 16.90331268310547,
      "step": 182
    },
    {
      "epoch": 0.2687224669603524,
      "fcm_dpo/beta": 0.05917463079094887,
      "fcm_dpo/delta": -0.11101389676332474,
      "fcm_dpo/margin": 16.041793823242188,
      "fcm_dpo/q_t": 0.3082314133644104,
      "grad_norm": 18.056352615356445,
      "learning_rate": 4.5910671414162484e-07,
      "logits/chosen": -1.2095495462417603,
      "logits/rejected": -0.6235491037368774,
      "logps/chosen": -103.69279479980469,
      "logps/ref_chosen": -93.90790557861328,
      "logps/ref_rejected": -83.56061553955078,
      "logps/rejected": -109.38729858398438,
      "loss": 0.7971,
      "margin_dpo/margin_mean": 16.041793823242188,
      "margin_dpo/margin_std": 14.786785125732422,
      "step": 183
    },
    {
      "epoch": 0.2701908957415565,
      "fcm_dpo/beta": 0.061080534011125565,
      "fcm_dpo/delta": 0.21639560163021088,
      "fcm_dpo/margin": 10.603964805603027,
      "fcm_dpo/q_t": 0.36788150668144226,
      "grad_norm": 20.342153549194336,
      "learning_rate": 4.5840059630527985e-07,
      "logits/chosen": -1.476677656173706,
      "logits/rejected": -1.261820912361145,
      "logps/chosen": -95.60768127441406,
      "logps/ref_chosen": -86.86790466308594,
      "logps/ref_rejected": -84.28456115722656,
      "logps/rejected": -103.6282958984375,
      "loss": 1.0314,
      "margin_dpo/margin_mean": 10.603963851928711,
      "margin_dpo/margin_std": 15.710251808166504,
      "step": 184
    },
    {
      "epoch": 0.27165932452276065,
      "fcm_dpo/beta": 0.06259152293205261,
      "fcm_dpo/delta": 0.04567991569638252,
      "fcm_dpo/margin": 12.893394470214844,
      "fcm_dpo/q_t": 0.34303832054138184,
      "grad_norm": 23.124479293823242,
      "learning_rate": 4.5768898691940836e-07,
      "logits/chosen": -1.7452738285064697,
      "logits/rejected": -1.1792840957641602,
      "logps/chosen": -103.65872955322266,
      "logps/ref_chosen": -94.02142333984375,
      "logps/ref_rejected": -81.45668029785156,
      "logps/rejected": -103.98738098144531,
      "loss": 0.943,
      "margin_dpo/margin_mean": 12.89339542388916,
      "margin_dpo/margin_std": 16.560266494750977,
      "step": 185
    },
    {
      "epoch": 0.27312775330396477,
      "fcm_dpo/beta": 0.06136384233832359,
      "fcm_dpo/delta": -0.06328608095645905,
      "fcm_dpo/margin": 14.769530296325684,
      "fcm_dpo/q_t": 0.3183685541152954,
      "grad_norm": 21.316608428955078,
      "learning_rate": 4.5697190473557947e-07,
      "logits/chosen": -2.0271401405334473,
      "logits/rejected": -1.2335484027862549,
      "logps/chosen": -125.817626953125,
      "logps/ref_chosen": -114.19168090820312,
      "logps/ref_rejected": -93.33912658691406,
      "logps/rejected": -119.73458862304688,
      "loss": 0.8408,
      "margin_dpo/margin_mean": 14.769529342651367,
      "margin_dpo/margin_std": 15.083984375,
      "step": 186
    },
    {
      "epoch": 0.2745961820851689,
      "fcm_dpo/beta": 0.06224139779806137,
      "fcm_dpo/delta": 0.0447879321873188,
      "fcm_dpo/margin": 12.98681354522705,
      "fcm_dpo/q_t": 0.34211674332618713,
      "grad_norm": 18.177583694458008,
      "learning_rate": 4.5624936864957555e-07,
      "logits/chosen": -1.5803933143615723,
      "logits/rejected": -1.344294786453247,
      "logps/chosen": -86.82991790771484,
      "logps/ref_chosen": -76.86955261230469,
      "logps/ref_rejected": -88.16494750976562,
      "logps/rejected": -111.11212158203125,
      "loss": 0.9328,
      "margin_dpo/margin_mean": 12.986814498901367,
      "margin_dpo/margin_std": 16.151668548583984,
      "step": 187
    },
    {
      "epoch": 0.27606461086637296,
      "fcm_dpo/beta": 0.06076783686876297,
      "fcm_dpo/delta": -0.08916963636875153,
      "fcm_dpo/margin": 15.28190803527832,
      "fcm_dpo/q_t": 0.3073975741863251,
      "grad_norm": 17.986310958862305,
      "learning_rate": 4.5552139770089454e-07,
      "logits/chosen": -1.4565420150756836,
      "logits/rejected": -1.1993229389190674,
      "logps/chosen": -86.23423767089844,
      "logps/ref_chosen": -79.1948013305664,
      "logps/ref_rejected": -96.82889556884766,
      "logps/rejected": -119.1502456665039,
      "loss": 0.8328,
      "margin_dpo/margin_mean": 15.281907081604004,
      "margin_dpo/margin_std": 14.838127136230469,
      "step": 188
    },
    {
      "epoch": 0.2775330396475771,
      "fcm_dpo/beta": 0.06060121953487396,
      "fcm_dpo/delta": -0.12682175636291504,
      "fcm_dpo/margin": 15.897202491760254,
      "fcm_dpo/q_t": 0.3206084072589874,
      "grad_norm": 16.118820190429688,
      "learning_rate": 4.5478801107224794e-07,
      "logits/chosen": -1.2777214050292969,
      "logits/rejected": -0.6506960988044739,
      "logps/chosen": -93.4096450805664,
      "logps/ref_chosen": -83.8927993774414,
      "logps/ref_rejected": -79.46589660644531,
      "logps/rejected": -104.87994384765625,
      "loss": 0.8542,
      "margin_dpo/margin_mean": 15.89720344543457,
      "margin_dpo/margin_std": 18.334991455078125,
      "step": 189
    },
    {
      "epoch": 0.2790014684287812,
      "fcm_dpo/beta": 0.05765543505549431,
      "fcm_dpo/delta": -0.19643330574035645,
      "fcm_dpo/margin": 17.790395736694336,
      "fcm_dpo/q_t": 0.29139959812164307,
      "grad_norm": 17.58658790588379,
      "learning_rate": 4.5404922808905543e-07,
      "logits/chosen": -1.5000858306884766,
      "logits/rejected": -0.9203025102615356,
      "logps/chosen": -110.40105438232422,
      "logps/ref_chosen": -103.44146728515625,
      "logps/ref_rejected": -109.21502685546875,
      "logps/rejected": -133.9650115966797,
      "loss": 0.7545,
      "margin_dpo/margin_mean": 17.790395736694336,
      "margin_dpo/margin_std": 15.214214324951172,
      "step": 190
    },
    {
      "epoch": 0.28046989720998533,
      "fcm_dpo/beta": 0.05509544536471367,
      "fcm_dpo/delta": -0.27554386854171753,
      "fcm_dpo/margin": 19.890722274780273,
      "fcm_dpo/q_t": 0.28646570444107056,
      "grad_norm": 18.667753219604492,
      "learning_rate": 4.5330506821893565e-07,
      "logits/chosen": -1.3162940740585327,
      "logits/rejected": -0.9771643280982971,
      "logps/chosen": -105.68276977539062,
      "logps/ref_chosen": -99.78551483154297,
      "logps/ref_rejected": -116.00028228759766,
      "logps/rejected": -141.78826904296875,
      "loss": 0.7436,
      "margin_dpo/margin_mean": 19.89072036743164,
      "margin_dpo/margin_std": 18.173255920410156,
      "step": 191
    },
    {
      "epoch": 0.28193832599118945,
      "fcm_dpo/beta": 0.05472166836261749,
      "fcm_dpo/delta": 0.08773260563611984,
      "fcm_dpo/margin": 14.054750442504883,
      "fcm_dpo/q_t": 0.3466401994228363,
      "grad_norm": 20.07705307006836,
      "learning_rate": 4.5255555107119336e-07,
      "logits/chosen": -1.1917040348052979,
      "logits/rejected": -0.9342058897018433,
      "logps/chosen": -113.84033203125,
      "logps/ref_chosen": -100.86032104492188,
      "logps/ref_rejected": -109.44500732421875,
      "logps/rejected": -136.47976684570312,
      "loss": 0.9351,
      "margin_dpo/margin_mean": 14.054752349853516,
      "margin_dpo/margin_std": 17.19097137451172,
      "step": 192
    },
    {
      "epoch": 0.2834067547723935,
      "fcm_dpo/beta": 0.05689571797847748,
      "fcm_dpo/delta": 0.17024913430213928,
      "fcm_dpo/margin": 12.118256568908691,
      "fcm_dpo/q_t": 0.3621511459350586,
      "grad_norm": 21.269683837890625,
      "learning_rate": 4.5180069639630236e-07,
      "logits/chosen": -1.5149171352386475,
      "logits/rejected": -0.9463554620742798,
      "logps/chosen": -102.46221160888672,
      "logps/ref_chosen": -92.93831634521484,
      "logps/ref_rejected": -83.10917663574219,
      "logps/rejected": -104.75133514404297,
      "loss": 1.0219,
      "margin_dpo/margin_mean": 12.118255615234375,
      "margin_dpo/margin_std": 17.301712036132812,
      "step": 193
    },
    {
      "epoch": 0.28487518355359764,
      "fcm_dpo/beta": 0.05779755860567093,
      "fcm_dpo/delta": 0.03512994945049286,
      "fcm_dpo/margin": 14.118998527526855,
      "fcm_dpo/q_t": 0.33766230940818787,
      "grad_norm": 17.252609252929688,
      "learning_rate": 4.510405240853854e-07,
      "logits/chosen": -2.011484146118164,
      "logits/rejected": -1.64884614944458,
      "logps/chosen": -73.9320297241211,
      "logps/ref_chosen": -63.79157257080078,
      "logps/ref_rejected": -68.72665405273438,
      "logps/rejected": -92.98609924316406,
      "loss": 0.9078,
      "margin_dpo/margin_mean": 14.118997573852539,
      "margin_dpo/margin_std": 16.229263305664062,
      "step": 194
    },
    {
      "epoch": 0.28634361233480177,
      "fcm_dpo/beta": 0.05644117295742035,
      "fcm_dpo/delta": -0.14289246499538422,
      "fcm_dpo/margin": 17.347488403320312,
      "fcm_dpo/q_t": 0.3069530725479126,
      "grad_norm": 17.122520446777344,
      "learning_rate": 4.5027505416968985e-07,
      "logits/chosen": -1.070321798324585,
      "logits/rejected": -0.9118151664733887,
      "logps/chosen": -93.67105102539062,
      "logps/ref_chosen": -83.88545989990234,
      "logps/ref_rejected": -104.00125122070312,
      "logps/rejected": -131.13433837890625,
      "loss": 0.8372,
      "margin_dpo/margin_mean": 17.347488403320312,
      "margin_dpo/margin_std": 18.290515899658203,
      "step": 195
    },
    {
      "epoch": 0.2878120411160059,
      "fcm_dpo/beta": 0.05479153245687485,
      "fcm_dpo/delta": -0.11973267793655396,
      "fcm_dpo/margin": 17.488330841064453,
      "fcm_dpo/q_t": 0.3044339418411255,
      "grad_norm": 14.206181526184082,
      "learning_rate": 4.495043068200599e-07,
      "logits/chosen": -0.9788313508033752,
      "logits/rejected": -0.5322003960609436,
      "logps/chosen": -100.54130554199219,
      "logps/ref_chosen": -89.4013671875,
      "logps/ref_rejected": -83.34735107421875,
      "logps/rejected": -111.97561645507812,
      "loss": 0.7886,
      "margin_dpo/margin_mean": 17.488330841064453,
      "margin_dpo/margin_std": 15.582159042358398,
      "step": 196
    },
    {
      "epoch": 0.28928046989721,
      "fcm_dpo/beta": 0.0564768947660923,
      "fcm_dpo/delta": 0.24834321439266205,
      "fcm_dpo/margin": 10.927431106567383,
      "fcm_dpo/q_t": 0.36820292472839355,
      "grad_norm": 18.218454360961914,
      "learning_rate": 4.4872830234640493e-07,
      "logits/chosen": -1.0139613151550293,
      "logits/rejected": -0.7055380344390869,
      "logps/chosen": -98.77421569824219,
      "logps/ref_chosen": -86.84070587158203,
      "logps/ref_rejected": -82.79289245605469,
      "logps/rejected": -105.65383911132812,
      "loss": 1.0011,
      "margin_dpo/margin_mean": 10.927431106567383,
      "margin_dpo/margin_std": 14.292451858520508,
      "step": 197
    },
    {
      "epoch": 0.2907488986784141,
      "fcm_dpo/beta": 0.0565267875790596,
      "fcm_dpo/delta": -0.09485112130641937,
      "fcm_dpo/margin": 16.559494018554688,
      "fcm_dpo/q_t": 0.3122091293334961,
      "grad_norm": 20.28647804260254,
      "learning_rate": 4.479470611971645e-07,
      "logits/chosen": -1.3765474557876587,
      "logits/rejected": -1.164352297782898,
      "logps/chosen": -99.16595458984375,
      "logps/ref_chosen": -85.04824829101562,
      "logps/ref_rejected": -104.10365295410156,
      "logps/rejected": -134.78085327148438,
      "loss": 0.8164,
      "margin_dpo/margin_mean": 16.55949592590332,
      "margin_dpo/margin_std": 16.41266441345215,
      "step": 198
    },
    {
      "epoch": 0.2922173274596182,
      "fcm_dpo/beta": 0.05642756074666977,
      "fcm_dpo/delta": 0.020511126145720482,
      "fcm_dpo/margin": 14.728890419006348,
      "fcm_dpo/q_t": 0.33377328515052795,
      "grad_norm": 18.18387222290039,
      "learning_rate": 4.471606039587695e-07,
      "logits/chosen": -1.5072638988494873,
      "logits/rejected": -1.2555978298187256,
      "logps/chosen": -95.25479125976562,
      "logps/ref_chosen": -81.78669738769531,
      "logps/ref_rejected": -90.790771484375,
      "logps/rejected": -118.98776245117188,
      "loss": 0.8888,
      "margin_dpo/margin_mean": 14.728889465332031,
      "margin_dpo/margin_std": 16.250869750976562,
      "step": 199
    },
    {
      "epoch": 0.2936857562408223,
      "fcm_dpo/beta": 0.055008381605148315,
      "fcm_dpo/delta": -0.11476686596870422,
      "fcm_dpo/margin": 17.302513122558594,
      "fcm_dpo/q_t": 0.3144181966781616,
      "grad_norm": 15.63186264038086,
      "learning_rate": 4.4636895135509966e-07,
      "logits/chosen": -1.6174651384353638,
      "logits/rejected": -1.1445786952972412,
      "logps/chosen": -96.79918670654297,
      "logps/ref_chosen": -84.26809692382812,
      "logps/ref_rejected": -86.96786499023438,
      "logps/rejected": -116.80148315429688,
      "loss": 0.8588,
      "margin_dpo/margin_mean": 17.302513122558594,
      "margin_dpo/margin_std": 18.646202087402344,
      "step": 200
    },
    {
      "epoch": 0.2936857562408223,
      "eval_fcm_dpo/beta": 0.05511855334043503,
      "eval_logits/chosen": -1.145448923110962,
      "eval_logits/rejected": -0.8443020582199097,
      "eval_logps/chosen": -115.77802276611328,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -121.74003601074219,
      "eval_loss": 0.5018326044082642,
      "eval_margin_dpo/margin_mean": 12.387815475463867,
      "eval_margin_dpo/margin_std": 17.043996810913086,
      "eval_runtime": 43.7855,
      "eval_samples_per_second": 53.42,
      "eval_steps_per_second": 1.69,
      "step": 200
    },
    {
      "epoch": 0.29515418502202645,
      "fcm_dpo/beta": 0.05490724742412567,
      "fcm_dpo/delta": -0.03479336202144623,
      "fcm_dpo/margin": 16.05862045288086,
      "fcm_dpo/q_t": 0.3281078338623047,
      "grad_norm": 19.102737426757812,
      "learning_rate": 4.455721242469372e-07,
      "logits/chosen": -1.08148193359375,
      "logits/rejected": -0.6773242950439453,
      "logps/chosen": -122.24678039550781,
      "logps/ref_chosen": -107.77249145507812,
      "logps/ref_rejected": -119.79248046875,
      "logps/rejected": -150.32537841796875,
      "loss": 0.9179,
      "margin_dpo/margin_mean": 16.05862045288086,
      "margin_dpo/margin_std": 19.691104888916016,
      "step": 201
    },
    {
      "epoch": 0.2966226138032305,
      "fcm_dpo/beta": 0.05517326667904854,
      "fcm_dpo/delta": 0.041220732033252716,
      "fcm_dpo/margin": 14.717092514038086,
      "fcm_dpo/q_t": 0.340792715549469,
      "grad_norm": 17.2861328125,
      "learning_rate": 4.4477014363141755e-07,
      "logits/chosen": -1.263124942779541,
      "logits/rejected": -1.1197762489318848,
      "logps/chosen": -89.5303955078125,
      "logps/ref_chosen": -75.97245788574219,
      "logps/ref_rejected": -94.4599838256836,
      "logps/rejected": -122.73501586914062,
      "loss": 0.9107,
      "margin_dpo/margin_mean": 14.717092514038086,
      "margin_dpo/margin_std": 17.112964630126953,
      "step": 202
    },
    {
      "epoch": 0.29809104258443464,
      "fcm_dpo/beta": 0.054431818425655365,
      "fcm_dpo/delta": -0.08932656049728394,
      "fcm_dpo/margin": 17.103050231933594,
      "fcm_dpo/q_t": 0.31066063046455383,
      "grad_norm": 15.397363662719727,
      "learning_rate": 4.439630306414758e-07,
      "logits/chosen": -1.5567055940628052,
      "logits/rejected": -1.1206634044647217,
      "logps/chosen": -106.02995300292969,
      "logps/ref_chosen": -94.96715545654297,
      "logps/ref_rejected": -92.8876724243164,
      "logps/rejected": -121.05351257324219,
      "loss": 0.8194,
      "margin_dpo/margin_mean": 17.103050231933594,
      "margin_dpo/margin_std": 16.513595581054688,
      "step": 203
    },
    {
      "epoch": 0.29955947136563876,
      "fcm_dpo/beta": 0.054940156638622284,
      "fcm_dpo/delta": 0.06543414294719696,
      "fcm_dpo/margin": 14.365680694580078,
      "fcm_dpo/q_t": 0.34287261962890625,
      "grad_norm": 21.70539665222168,
      "learning_rate": 4.431508065452897e-07,
      "logits/chosen": -1.2361195087432861,
      "logits/rejected": -0.5873404741287231,
      "logps/chosen": -131.6267852783203,
      "logps/ref_chosen": -116.35719299316406,
      "logps/ref_rejected": -93.39759063720703,
      "logps/rejected": -123.03286743164062,
      "loss": 0.94,
      "margin_dpo/margin_mean": 14.365680694580078,
      "margin_dpo/margin_std": 17.704381942749023,
      "step": 204
    },
    {
      "epoch": 0.3010279001468429,
      "fcm_dpo/beta": 0.05323153734207153,
      "fcm_dpo/delta": -0.18734723329544067,
      "fcm_dpo/margin": 19.112470626831055,
      "fcm_dpo/q_t": 0.28901487588882446,
      "grad_norm": 14.131471633911133,
      "learning_rate": 4.4233349274571974e-07,
      "logits/chosen": -1.951904535293579,
      "logits/rejected": -1.5425834655761719,
      "logps/chosen": -102.00930786132812,
      "logps/ref_chosen": -88.85934448242188,
      "logps/ref_rejected": -91.8544921875,
      "logps/rejected": -124.1169204711914,
      "loss": 0.7412,
      "margin_dpo/margin_mean": 19.112470626831055,
      "margin_dpo/margin_std": 15.50132942199707,
      "step": 205
    },
    {
      "epoch": 0.302496328928047,
      "fcm_dpo/beta": 0.052048660814762115,
      "fcm_dpo/delta": -0.06987091153860092,
      "fcm_dpo/margin": 17.530094146728516,
      "fcm_dpo/q_t": 0.30802714824676514,
      "grad_norm": 19.163528442382812,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": -1.3110017776489258,
      "logits/rejected": -1.1106235980987549,
      "logps/chosen": -90.1287841796875,
      "logps/ref_chosen": -76.54634857177734,
      "logps/ref_rejected": -102.95314025878906,
      "logps/rejected": -134.065673828125,
      "loss": 0.7868,
      "margin_dpo/margin_mean": 17.530094146728516,
      "margin_dpo/margin_std": 14.55693531036377,
      "step": 206
    },
    {
      "epoch": 0.3039647577092511,
      "fcm_dpo/beta": 0.051451846957206726,
      "fcm_dpo/delta": -0.09747027605772018,
      "fcm_dpo/margin": 18.237651824951172,
      "fcm_dpo/q_t": 0.32565534114837646,
      "grad_norm": 22.39365005493164,
      "learning_rate": 4.4068368231789365e-07,
      "logits/chosen": -1.6113066673278809,
      "logits/rejected": -1.1651461124420166,
      "logps/chosen": -101.52843475341797,
      "logps/ref_chosen": -86.23164367675781,
      "logps/ref_rejected": -90.65512084960938,
      "logps/rejected": -124.18955993652344,
      "loss": 0.893,
      "margin_dpo/margin_mean": 18.237651824951172,
      "margin_dpo/margin_std": 22.46469497680664,
      "step": 207
    },
    {
      "epoch": 0.3054331864904552,
      "fcm_dpo/beta": 0.050518698990345,
      "fcm_dpo/delta": -0.14123016595840454,
      "fcm_dpo/margin": 19.343996047973633,
      "fcm_dpo/q_t": 0.2982601523399353,
      "grad_norm": 17.023468017578125,
      "learning_rate": 4.398512291636768e-07,
      "logits/chosen": -1.5571520328521729,
      "logits/rejected": -1.182025671005249,
      "logps/chosen": -109.33761596679688,
      "logps/ref_chosen": -94.1595458984375,
      "logps/ref_rejected": -100.96233367919922,
      "logps/rejected": -135.48440551757812,
      "loss": 0.7912,
      "margin_dpo/margin_mean": 19.343996047973633,
      "margin_dpo/margin_std": 17.425840377807617,
      "step": 208
    },
    {
      "epoch": 0.3069016152716593,
      "fcm_dpo/beta": 0.04946083575487137,
      "fcm_dpo/delta": 0.011037945747375488,
      "fcm_dpo/margin": 16.961767196655273,
      "fcm_dpo/q_t": 0.33165478706359863,
      "grad_norm": 15.817395210266113,
      "learning_rate": 4.3901377325300857e-07,
      "logits/chosen": -1.7526047229766846,
      "logits/rejected": -1.2873879671096802,
      "logps/chosen": -97.22732543945312,
      "logps/ref_chosen": -84.17056274414062,
      "logps/ref_rejected": -87.61955261230469,
      "logps/rejected": -117.63807678222656,
      "loss": 0.8862,
      "margin_dpo/margin_mean": 16.961767196655273,
      "margin_dpo/margin_std": 18.429622650146484,
      "step": 209
    },
    {
      "epoch": 0.30837004405286345,
      "fcm_dpo/beta": 0.049842916429042816,
      "fcm_dpo/delta": 0.037534188479185104,
      "fcm_dpo/margin": 16.353267669677734,
      "fcm_dpo/q_t": 0.33743494749069214,
      "grad_norm": 15.248353958129883,
      "learning_rate": 4.381713366536311e-07,
      "logits/chosen": -1.5737419128417969,
      "logits/rejected": -1.1732791662216187,
      "logps/chosen": -95.85877990722656,
      "logps/ref_chosen": -81.17117309570312,
      "logps/ref_rejected": -84.17478942871094,
      "logps/rejected": -115.21566009521484,
      "loss": 0.8877,
      "margin_dpo/margin_mean": 16.353267669677734,
      "margin_dpo/margin_std": 17.86672592163086,
      "step": 210
    },
    {
      "epoch": 0.30983847283406757,
      "fcm_dpo/beta": 0.05027975142002106,
      "fcm_dpo/delta": 0.0004709279164671898,
      "fcm_dpo/margin": 16.896448135375977,
      "fcm_dpo/q_t": 0.33016592264175415,
      "grad_norm": 16.873620986938477,
      "learning_rate": 4.373239415645323e-07,
      "logits/chosen": -1.735175371170044,
      "logits/rejected": -1.1475024223327637,
      "logps/chosen": -125.00082397460938,
      "logps/ref_chosen": -108.71270751953125,
      "logps/ref_rejected": -93.55564880371094,
      "logps/rejected": -126.74020385742188,
      "loss": 0.8806,
      "margin_dpo/margin_mean": 16.896448135375977,
      "margin_dpo/margin_std": 18.872371673583984,
      "step": 211
    },
    {
      "epoch": 0.31130690161527164,
      "fcm_dpo/beta": 0.04791909083724022,
      "fcm_dpo/delta": -0.2870158553123474,
      "fcm_dpo/margin": 23.030895233154297,
      "fcm_dpo/q_t": 0.28026482462882996,
      "grad_norm": 13.476890563964844,
      "learning_rate": 4.3647161031536086e-07,
      "logits/chosen": -1.4513945579528809,
      "logits/rejected": -1.0123846530914307,
      "logps/chosen": -108.43475341796875,
      "logps/ref_chosen": -98.36195373535156,
      "logps/ref_rejected": -109.88999938964844,
      "logps/rejected": -142.99371337890625,
      "loss": 0.7281,
      "margin_dpo/margin_mean": 23.030893325805664,
      "margin_dpo/margin_std": 19.890701293945312,
      "step": 212
    },
    {
      "epoch": 0.31277533039647576,
      "fcm_dpo/beta": 0.04703017696738243,
      "fcm_dpo/delta": -0.029142513871192932,
      "fcm_dpo/margin": 18.629619598388672,
      "fcm_dpo/q_t": 0.3192410171031952,
      "grad_norm": 20.05600357055664,
      "learning_rate": 4.3561436536583774e-07,
      "logits/chosen": -1.5639327764511108,
      "logits/rejected": -0.8274385929107666,
      "logps/chosen": -121.58412170410156,
      "logps/ref_chosen": -108.05531311035156,
      "logps/ref_rejected": -100.14414978027344,
      "logps/rejected": -132.30258178710938,
      "loss": 0.8464,
      "margin_dpo/margin_mean": 18.629619598388672,
      "margin_dpo/margin_std": 18.510005950927734,
      "step": 213
    },
    {
      "epoch": 0.3142437591776799,
      "fcm_dpo/beta": 0.047874145209789276,
      "fcm_dpo/delta": 0.08061401546001434,
      "fcm_dpo/margin": 16.192214965820312,
      "fcm_dpo/q_t": 0.3512595295906067,
      "grad_norm": 16.127655029296875,
      "learning_rate": 4.3475222930516473e-07,
      "logits/chosen": -1.3913880586624146,
      "logits/rejected": -1.266923189163208,
      "logps/chosen": -92.62548828125,
      "logps/ref_chosen": -77.80473327636719,
      "logps/ref_rejected": -84.27578735351562,
      "logps/rejected": -115.28874206542969,
      "loss": 0.9673,
      "margin_dpo/margin_mean": 16.192214965820312,
      "margin_dpo/margin_std": 21.93199348449707,
      "step": 214
    },
    {
      "epoch": 0.315712187958884,
      "fcm_dpo/beta": 0.046690285205841064,
      "fcm_dpo/delta": -0.16893821954727173,
      "fcm_dpo/margin": 21.456758499145508,
      "fcm_dpo/q_t": 0.2978704571723938,
      "grad_norm": 14.699638366699219,
      "learning_rate": 4.3388522485142885e-07,
      "logits/chosen": -1.5712075233459473,
      "logits/rejected": -1.0760598182678223,
      "logps/chosen": -97.10356140136719,
      "logps/ref_chosen": -85.1138916015625,
      "logps/ref_rejected": -96.86151123046875,
      "logps/rejected": -130.30795288085938,
      "loss": 0.7625,
      "margin_dpo/margin_mean": 21.456756591796875,
      "margin_dpo/margin_std": 19.08936309814453,
      "step": 215
    },
    {
      "epoch": 0.31718061674008813,
      "fcm_dpo/beta": 0.0466710701584816,
      "fcm_dpo/delta": 0.01144382357597351,
      "fcm_dpo/margin": 17.97332191467285,
      "fcm_dpo/q_t": 0.3272448778152466,
      "grad_norm": 13.613797187805176,
      "learning_rate": 4.330133748510036e-07,
      "logits/chosen": -1.5654511451721191,
      "logits/rejected": -1.1545265913009644,
      "logps/chosen": -94.93984985351562,
      "logps/ref_chosen": -80.59230041503906,
      "logps/ref_rejected": -81.41983795166016,
      "logps/rejected": -113.74070739746094,
      "loss": 0.8434,
      "margin_dpo/margin_mean": 17.97332000732422,
      "margin_dpo/margin_std": 17.3824462890625,
      "step": 216
    },
    {
      "epoch": 0.3186490455212922,
      "fcm_dpo/beta": 0.04719318449497223,
      "fcm_dpo/delta": 0.07027594745159149,
      "fcm_dpo/margin": 16.616840362548828,
      "fcm_dpo/q_t": 0.34417039155960083,
      "grad_norm": 17.98988151550293,
      "learning_rate": 4.3213670227794757e-07,
      "logits/chosen": -1.81255042552948,
      "logits/rejected": -1.5056719779968262,
      "logps/chosen": -110.1501693725586,
      "logps/ref_chosen": -93.47257995605469,
      "logps/ref_rejected": -103.488525390625,
      "logps/rejected": -136.782958984375,
      "loss": 0.9225,
      "margin_dpo/margin_mean": 16.616840362548828,
      "margin_dpo/margin_std": 19.80947494506836,
      "step": 217
    },
    {
      "epoch": 0.3201174743024963,
      "fcm_dpo/beta": 0.04671292006969452,
      "fcm_dpo/delta": -0.0367327444255352,
      "fcm_dpo/margin": 18.908458709716797,
      "fcm_dpo/q_t": 0.3229714035987854,
      "grad_norm": 16.900039672851562,
      "learning_rate": 4.3125523023339815e-07,
      "logits/chosen": -1.0308185815811157,
      "logits/rejected": -0.6163084506988525,
      "logps/chosen": -104.5977783203125,
      "logps/ref_chosen": -89.05883026123047,
      "logps/ref_rejected": -94.30680847167969,
      "logps/rejected": -128.7542266845703,
      "loss": 0.8795,
      "margin_dpo/margin_mean": 18.908458709716797,
      "margin_dpo/margin_std": 20.99794578552246,
      "step": 218
    },
    {
      "epoch": 0.32158590308370044,
      "fcm_dpo/beta": 0.04732338711619377,
      "fcm_dpo/delta": 0.014991968870162964,
      "fcm_dpo/margin": 17.627521514892578,
      "fcm_dpo/q_t": 0.33559125661849976,
      "grad_norm": 22.87677764892578,
      "learning_rate": 4.303689819449636e-07,
      "logits/chosen": -1.599898099899292,
      "logits/rejected": -1.204871654510498,
      "logps/chosen": -115.39610290527344,
      "logps/ref_chosen": -101.00733947753906,
      "logps/ref_rejected": -92.46794128417969,
      "logps/rejected": -124.48422241210938,
      "loss": 0.9291,
      "margin_dpo/margin_mean": 17.627521514892578,
      "margin_dpo/margin_std": 21.77505874633789,
      "step": 219
    },
    {
      "epoch": 0.32305433186490456,
      "fcm_dpo/beta": 0.04759221524000168,
      "fcm_dpo/delta": 0.10844795405864716,
      "fcm_dpo/margin": 15.754082679748535,
      "fcm_dpo/q_t": 0.3434942364692688,
      "grad_norm": 16.797428131103516,
      "learning_rate": 4.2947798076611047e-07,
      "logits/chosen": -1.211072325706482,
      "logits/rejected": -0.8570730090141296,
      "logps/chosen": -110.34193420410156,
      "logps/ref_chosen": -95.53721618652344,
      "logps/ref_rejected": -94.30703735351562,
      "logps/rejected": -124.86582946777344,
      "loss": 0.901,
      "margin_dpo/margin_mean": 15.754082679748535,
      "margin_dpo/margin_std": 16.457195281982422,
      "step": 220
    },
    {
      "epoch": 0.3245227606461087,
      "fcm_dpo/beta": 0.046583935618400574,
      "fcm_dpo/delta": -0.26324182748794556,
      "fcm_dpo/margin": 23.2933349609375,
      "fcm_dpo/q_t": 0.2837619185447693,
      "grad_norm": 15.483804702758789,
      "learning_rate": 4.285822501755485e-07,
      "logits/chosen": -1.017369270324707,
      "logits/rejected": -0.8445158004760742,
      "logps/chosen": -97.19378662109375,
      "logps/ref_chosen": -82.84486389160156,
      "logps/ref_rejected": -110.81179809570312,
      "logps/rejected": -148.45407104492188,
      "loss": 0.7347,
      "margin_dpo/margin_mean": 23.2933349609375,
      "margin_dpo/margin_std": 19.25533676147461,
      "step": 221
    },
    {
      "epoch": 0.32599118942731276,
      "fcm_dpo/beta": 0.045510318130254745,
      "fcm_dpo/delta": -0.0532805472612381,
      "fcm_dpo/margin": 19.718502044677734,
      "fcm_dpo/q_t": 0.32435593008995056,
      "grad_norm": 23.144872665405273,
      "learning_rate": 4.276818137766118e-07,
      "logits/chosen": -1.720402479171753,
      "logits/rejected": -1.287516474723816,
      "logps/chosen": -113.3494873046875,
      "logps/ref_chosen": -95.14198303222656,
      "logps/ref_rejected": -106.80441284179688,
      "logps/rejected": -144.73040771484375,
      "loss": 0.9196,
      "margin_dpo/margin_mean": 19.7185001373291,
      "margin_dpo/margin_std": 24.308671951293945,
      "step": 222
    },
    {
      "epoch": 0.3274596182085169,
      "fcm_dpo/beta": 0.045756347477436066,
      "fcm_dpo/delta": 0.13066641986370087,
      "fcm_dpo/margin": 15.932899475097656,
      "fcm_dpo/q_t": 0.3532084822654724,
      "grad_norm": 15.819351196289062,
      "learning_rate": 4.2677669529663686e-07,
      "logits/chosen": -2.0023083686828613,
      "logits/rejected": -1.5972869396209717,
      "logps/chosen": -102.81034851074219,
      "logps/ref_chosen": -85.57511138916016,
      "logps/ref_rejected": -86.45237731933594,
      "logps/rejected": -119.62051391601562,
      "loss": 0.9563,
      "margin_dpo/margin_mean": 15.932899475097656,
      "margin_dpo/margin_std": 19.981346130371094,
      "step": 223
    },
    {
      "epoch": 0.328928046989721,
      "fcm_dpo/beta": 0.04605341702699661,
      "fcm_dpo/delta": -0.03015466220676899,
      "fcm_dpo/margin": 19.053667068481445,
      "fcm_dpo/q_t": 0.3233034014701843,
      "grad_norm": 16.37089729309082,
      "learning_rate": 4.2586691858633747e-07,
      "logits/chosen": -1.5879364013671875,
      "logits/rejected": -1.1492383480072021,
      "logps/chosen": -99.3148193359375,
      "logps/ref_chosen": -82.72380065917969,
      "logps/ref_rejected": -82.59538269042969,
      "logps/rejected": -118.24006652832031,
      "loss": 0.8544,
      "margin_dpo/margin_mean": 19.053668975830078,
      "margin_dpo/margin_std": 20.131755828857422,
      "step": 224
    },
    {
      "epoch": 0.3303964757709251,
      "fcm_dpo/beta": 0.045157790184020996,
      "fcm_dpo/delta": -0.1496858149766922,
      "fcm_dpo/margin": 21.81402587890625,
      "fcm_dpo/q_t": 0.3073573112487793,
      "grad_norm": 16.657323837280273,
      "learning_rate": 4.249525076191759e-07,
      "logits/chosen": -1.70444917678833,
      "logits/rejected": -1.226616382598877,
      "logps/chosen": -114.02447509765625,
      "logps/ref_chosen": -95.67768859863281,
      "logps/ref_rejected": -105.09687805175781,
      "logps/rejected": -145.2576904296875,
      "loss": 0.8305,
      "margin_dpo/margin_mean": 21.81402587890625,
      "margin_dpo/margin_std": 23.370323181152344,
      "step": 225
    },
    {
      "epoch": 0.33186490455212925,
      "fcm_dpo/beta": 0.04409945756196976,
      "fcm_dpo/delta": -0.017373599112033844,
      "fcm_dpo/margin": 19.608936309814453,
      "fcm_dpo/q_t": 0.32043924927711487,
      "grad_norm": 14.112857818603516,
      "learning_rate": 4.2403348649073167e-07,
      "logits/chosen": -1.6110211610794067,
      "logits/rejected": -1.0937122106552124,
      "logps/chosen": -108.33169555664062,
      "logps/ref_chosen": -93.46092224121094,
      "logps/ref_rejected": -86.7017593383789,
      "logps/rejected": -121.18147277832031,
      "loss": 0.8342,
      "margin_dpo/margin_mean": 19.608936309814453,
      "margin_dpo/margin_std": 18.334829330444336,
      "step": 226
    },
    {
      "epoch": 0.3333333333333333,
      "fcm_dpo/beta": 0.044242776930332184,
      "fcm_dpo/delta": -0.023924846202135086,
      "fcm_dpo/margin": 19.70592498779297,
      "fcm_dpo/q_t": 0.3298208713531494,
      "grad_norm": 16.250980377197266,
      "learning_rate": 4.2310987941806615e-07,
      "logits/chosen": -1.4222127199172974,
      "logits/rejected": -1.094136357307434,
      "logps/chosen": -114.05935668945312,
      "logps/ref_chosen": -92.81427001953125,
      "logps/ref_rejected": -104.73693084716797,
      "logps/rejected": -145.68792724609375,
      "loss": 0.8972,
      "margin_dpo/margin_mean": 19.7059268951416,
      "margin_dpo/margin_std": 22.74959945678711,
      "step": 227
    },
    {
      "epoch": 0.33480176211453744,
      "fcm_dpo/beta": 0.04522550851106644,
      "fcm_dpo/delta": 0.13866844773292542,
      "fcm_dpo/margin": 15.936595916748047,
      "fcm_dpo/q_t": 0.34774237871170044,
      "grad_norm": 16.323331832885742,
      "learning_rate": 4.2218171073908463e-07,
      "logits/chosen": -1.6550034284591675,
      "logits/rejected": -1.2243372201919556,
      "logps/chosen": -113.41688537597656,
      "logps/ref_chosen": -94.03712463378906,
      "logps/ref_rejected": -96.02151489257812,
      "logps/rejected": -131.33787536621094,
      "loss": 0.9329,
      "margin_dpo/margin_mean": 15.936595916748047,
      "margin_dpo/margin_std": 17.845264434814453,
      "step": 228
    },
    {
      "epoch": 0.33627019089574156,
      "fcm_dpo/beta": 0.043626703321933746,
      "fcm_dpo/delta": -0.2818097770214081,
      "fcm_dpo/margin": 25.238357543945312,
      "fcm_dpo/q_t": 0.28765520453453064,
      "grad_norm": 13.921157836914062,
      "learning_rate": 4.212490049118951e-07,
      "logits/chosen": -1.539412021636963,
      "logits/rejected": -0.9021925926208496,
      "logps/chosen": -111.91642761230469,
      "logps/ref_chosen": -95.57766723632812,
      "logps/ref_rejected": -89.17379760742188,
      "logps/rejected": -130.75091552734375,
      "loss": 0.7562,
      "margin_dpo/margin_mean": 25.238357543945312,
      "margin_dpo/margin_std": 23.75576400756836,
      "step": 229
    },
    {
      "epoch": 0.3377386196769457,
      "fcm_dpo/beta": 0.04161980003118515,
      "fcm_dpo/delta": -0.20371034741401672,
      "fcm_dpo/margin": 24.80811309814453,
      "fcm_dpo/q_t": 0.28747209906578064,
      "grad_norm": 15.197046279907227,
      "learning_rate": 4.203117865141635e-07,
      "logits/chosen": -1.5952270030975342,
      "logits/rejected": -1.4780795574188232,
      "logps/chosen": -81.2739028930664,
      "logps/ref_chosen": -63.713626861572266,
      "logps/ref_rejected": -91.90870666503906,
      "logps/rejected": -134.277099609375,
      "loss": 0.7406,
      "margin_dpo/margin_mean": 24.80811309814453,
      "margin_dpo/margin_std": 20.30365753173828,
      "step": 230
    },
    {
      "epoch": 0.3392070484581498,
      "fcm_dpo/beta": 0.04121699184179306,
      "fcm_dpo/delta": 0.03683965280652046,
      "fcm_dpo/margin": 19.79311752319336,
      "fcm_dpo/q_t": 0.3291405439376831,
      "grad_norm": 13.925002098083496,
      "learning_rate": 4.1937008024246625e-07,
      "logits/chosen": -1.4601212739944458,
      "logits/rejected": -0.9463979005813599,
      "logps/chosen": -116.09357452392578,
      "logps/ref_chosen": -95.45567321777344,
      "logps/ref_rejected": -80.95568084716797,
      "logps/rejected": -121.3866958618164,
      "loss": 0.846,
      "margin_dpo/margin_mean": 19.79311752319336,
      "margin_dpo/margin_std": 18.4172306060791,
      "step": 231
    },
    {
      "epoch": 0.3406754772393539,
      "fcm_dpo/beta": 0.04198054224252701,
      "fcm_dpo/delta": 0.0867529809474945,
      "fcm_dpo/margin": 18.341995239257812,
      "fcm_dpo/q_t": 0.3432447910308838,
      "grad_norm": 15.27546215057373,
      "learning_rate": 4.1842391091163933e-07,
      "logits/chosen": -1.2151339054107666,
      "logits/rejected": -0.749545693397522,
      "logps/chosen": -116.75117492675781,
      "logps/ref_chosen": -96.89726257324219,
      "logps/ref_rejected": -89.76461791992188,
      "logps/rejected": -127.96051788330078,
      "loss": 0.9195,
      "margin_dpo/margin_mean": 18.341995239257812,
      "margin_dpo/margin_std": 20.981639862060547,
      "step": 232
    },
    {
      "epoch": 0.342143906020558,
      "fcm_dpo/beta": 0.040949709713459015,
      "fcm_dpo/delta": -0.18371593952178955,
      "fcm_dpo/margin": 24.765342712402344,
      "fcm_dpo/q_t": 0.29807668924331665,
      "grad_norm": 14.443903923034668,
      "learning_rate": 4.174733034541245e-07,
      "logits/chosen": -1.4963655471801758,
      "logits/rejected": -1.233656644821167,
      "logps/chosen": -110.30583190917969,
      "logps/ref_chosen": -89.05032348632812,
      "logps/ref_rejected": -112.75917053222656,
      "logps/rejected": -158.77999877929688,
      "loss": 0.7868,
      "margin_dpo/margin_mean": 24.765342712402344,
      "margin_dpo/margin_std": 23.211368560791016,
      "step": 233
    },
    {
      "epoch": 0.3436123348017621,
      "fcm_dpo/beta": 0.0389174222946167,
      "fcm_dpo/delta": -0.19708919525146484,
      "fcm_dpo/margin": 26.18871307373047,
      "fcm_dpo/q_t": 0.30054956674575806,
      "grad_norm": 13.936091423034668,
      "learning_rate": 4.165182829193126e-07,
      "logits/chosen": -1.3125296831130981,
      "logits/rejected": -1.2004642486572266,
      "logps/chosen": -94.45536804199219,
      "logps/ref_chosen": -74.318359375,
      "logps/ref_rejected": -106.38758850097656,
      "logps/rejected": -152.7133026123047,
      "loss": 0.8011,
      "margin_dpo/margin_mean": 26.18871307373047,
      "margin_dpo/margin_std": 25.016361236572266,
      "step": 234
    },
    {
      "epoch": 0.34508076358296624,
      "fcm_dpo/beta": 0.03963829576969147,
      "fcm_dpo/delta": 0.07630567252635956,
      "fcm_dpo/margin": 19.665775299072266,
      "fcm_dpo/q_t": 0.3366130590438843,
      "grad_norm": 13.919268608093262,
      "learning_rate": 4.1555887447288255e-07,
      "logits/chosen": -1.2706738710403442,
      "logits/rejected": -0.7282851934432983,
      "logps/chosen": -122.56187438964844,
      "logps/ref_chosen": -98.217041015625,
      "logps/ref_rejected": -97.24677276611328,
      "logps/rejected": -141.25738525390625,
      "loss": 0.8743,
      "margin_dpo/margin_mean": 19.665773391723633,
      "margin_dpo/margin_std": 19.334230422973633,
      "step": 235
    },
    {
      "epoch": 0.3465491923641703,
      "fcm_dpo/beta": 0.03917301446199417,
      "fcm_dpo/delta": -0.12850745022296906,
      "fcm_dpo/margin": 24.66576385498047,
      "fcm_dpo/q_t": 0.3191804885864258,
      "grad_norm": 14.044709205627441,
      "learning_rate": 4.1459510339613946e-07,
      "logits/chosen": -1.3997790813446045,
      "logits/rejected": -1.3525149822235107,
      "logps/chosen": -98.97593688964844,
      "logps/ref_chosen": -78.83773040771484,
      "logps/ref_rejected": -109.06343078613281,
      "logps/rejected": -153.86740112304688,
      "loss": 0.8368,
      "margin_dpo/margin_mean": 24.66576385498047,
      "margin_dpo/margin_std": 28.19268035888672,
      "step": 236
    },
    {
      "epoch": 0.34801762114537443,
      "fcm_dpo/beta": 0.0390484444797039,
      "fcm_dpo/delta": 0.07136844098567963,
      "fcm_dpo/margin": 20.084590911865234,
      "fcm_dpo/q_t": 0.35290807485580444,
      "grad_norm": 15.641119003295898,
      "learning_rate": 4.136269950853473e-07,
      "logits/chosen": -1.503021240234375,
      "logits/rejected": -1.09787118434906,
      "logps/chosen": -108.82412719726562,
      "logps/ref_chosen": -85.21128845214844,
      "logps/ref_rejected": -99.90999603271484,
      "logps/rejected": -143.60743713378906,
      "loss": 0.9832,
      "margin_dpo/margin_mean": 20.084590911865234,
      "margin_dpo/margin_std": 28.457015991210938,
      "step": 237
    },
    {
      "epoch": 0.34948604992657856,
      "fcm_dpo/beta": 0.03921890631318092,
      "fcm_dpo/delta": -0.0024761436507105827,
      "fcm_dpo/margin": 21.72978401184082,
      "fcm_dpo/q_t": 0.33124884963035583,
      "grad_norm": 13.521815299987793,
      "learning_rate": 4.126545750510605e-07,
      "logits/chosen": -1.3132723569869995,
      "logits/rejected": -1.1299904584884644,
      "logps/chosen": -104.59783935546875,
      "logps/ref_chosen": -78.73123168945312,
      "logps/ref_rejected": -95.41840362548828,
      "logps/rejected": -143.01480102539062,
      "loss": 0.8794,
      "margin_dpo/margin_mean": 21.729782104492188,
      "margin_dpo/margin_std": 24.03033447265625,
      "step": 238
    },
    {
      "epoch": 0.3509544787077827,
      "fcm_dpo/beta": 0.03920455649495125,
      "fcm_dpo/delta": 0.04809597134590149,
      "fcm_dpo/margin": 20.524173736572266,
      "fcm_dpo/q_t": 0.33716443181037903,
      "grad_norm": 14.763937950134277,
      "learning_rate": 4.116778689174514e-07,
      "logits/chosen": -1.7374763488769531,
      "logits/rejected": -1.263242244720459,
      "logps/chosen": -116.79473876953125,
      "logps/ref_chosen": -92.60093688964844,
      "logps/ref_rejected": -100.51769256591797,
      "logps/rejected": -145.23565673828125,
      "loss": 0.9014,
      "margin_dpo/margin_mean": 20.524173736572266,
      "margin_dpo/margin_std": 22.263370513916016,
      "step": 239
    },
    {
      "epoch": 0.3524229074889868,
      "fcm_dpo/beta": 0.040073223412036896,
      "fcm_dpo/delta": 0.10381583124399185,
      "fcm_dpo/margin": 18.81613540649414,
      "fcm_dpo/q_t": 0.34458890557289124,
      "grad_norm": 18.043434143066406,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -1.850398063659668,
      "logits/rejected": -1.5278594493865967,
      "logps/chosen": -110.80873107910156,
      "logps/ref_chosen": -86.15977478027344,
      "logps/ref_rejected": -80.45567321777344,
      "logps/rejected": -123.92076110839844,
      "loss": 0.9218,
      "margin_dpo/margin_mean": 18.81613540649414,
      "margin_dpo/margin_std": 21.0098876953125,
      "step": 240
    },
    {
      "epoch": 0.35389133627019087,
      "fcm_dpo/beta": 0.03847186267375946,
      "fcm_dpo/delta": -0.420759916305542,
      "fcm_dpo/margin": 31.7427978515625,
      "fcm_dpo/q_t": 0.2649722695350647,
      "grad_norm": 13.3577299118042,
      "learning_rate": 4.097117014129903e-07,
      "logits/chosen": -1.5077540874481201,
      "logits/rejected": -0.8860681056976318,
      "logps/chosen": -121.54011535644531,
      "logps/ref_chosen": -101.04594421386719,
      "logps/ref_rejected": -94.04934692382812,
      "logps/rejected": -146.28631591796875,
      "loss": 0.6882,
      "margin_dpo/margin_mean": 31.7427978515625,
      "margin_dpo/margin_std": 26.658802032470703,
      "step": 241
    },
    {
      "epoch": 0.355359765051395,
      "fcm_dpo/beta": 0.03743920102715492,
      "fcm_dpo/delta": -0.012355677783489227,
      "fcm_dpo/margin": 22.97677993774414,
      "fcm_dpo/q_t": 0.32101404666900635,
      "grad_norm": 14.553584098815918,
      "learning_rate": 4.087222918524807e-07,
      "logits/chosen": -1.6811339855194092,
      "logits/rejected": -1.0449347496032715,
      "logps/chosen": -121.91845703125,
      "logps/ref_chosen": -95.67266845703125,
      "logps/ref_rejected": -90.65454864501953,
      "logps/rejected": -139.8771209716797,
      "loss": 0.8409,
      "margin_dpo/margin_mean": 22.97677993774414,
      "margin_dpo/margin_std": 21.574127197265625,
      "step": 242
    },
    {
      "epoch": 0.3568281938325991,
      "fcm_dpo/beta": 0.03756431117653847,
      "fcm_dpo/delta": -0.008666589856147766,
      "fcm_dpo/margin": 22.756086349487305,
      "fcm_dpo/q_t": 0.32629698514938354,
      "grad_norm": 13.971424102783203,
      "learning_rate": 4.07728699811968e-07,
      "logits/chosen": -1.6577210426330566,
      "logits/rejected": -0.8642560243606567,
      "logps/chosen": -123.25171661376953,
      "logps/ref_chosen": -98.03140258789062,
      "logps/ref_rejected": -83.18806457519531,
      "logps/rejected": -131.16445922851562,
      "loss": 0.8891,
      "margin_dpo/margin_mean": 22.756086349487305,
      "margin_dpo/margin_std": 24.21973419189453,
      "step": 243
    },
    {
      "epoch": 0.35829662261380324,
      "fcm_dpo/beta": 0.0368984192609787,
      "fcm_dpo/delta": -0.012304404750466347,
      "fcm_dpo/margin": 23.336746215820312,
      "fcm_dpo/q_t": 0.32805073261260986,
      "grad_norm": 13.78069019317627,
      "learning_rate": 4.067309514735267e-07,
      "logits/chosen": -1.9630992412567139,
      "logits/rejected": -1.6161465644836426,
      "logps/chosen": -115.62930297851562,
      "logps/ref_chosen": -88.8939208984375,
      "logps/ref_rejected": -102.57278442382812,
      "logps/rejected": -152.64491271972656,
      "loss": 0.8663,
      "margin_dpo/margin_mean": 23.336746215820312,
      "margin_dpo/margin_std": 24.9739933013916,
      "step": 244
    },
    {
      "epoch": 0.35976505139500736,
      "fcm_dpo/beta": 0.03636704385280609,
      "fcm_dpo/delta": -0.11052923649549484,
      "fcm_dpo/margin": 26.122928619384766,
      "fcm_dpo/q_t": 0.3047110438346863,
      "grad_norm": 13.41275691986084,
      "learning_rate": 4.057290731287531e-07,
      "logits/chosen": -2.0480587482452393,
      "logits/rejected": -1.2761878967285156,
      "logps/chosen": -126.63717651367188,
      "logps/ref_chosen": -104.19400024414062,
      "logps/ref_rejected": -92.65645599365234,
      "logps/rejected": -141.22256469726562,
      "loss": 0.7874,
      "margin_dpo/margin_mean": 26.122928619384766,
      "margin_dpo/margin_std": 22.928936004638672,
      "step": 245
    },
    {
      "epoch": 0.36123348017621143,
      "fcm_dpo/beta": 0.03667043149471283,
      "fcm_dpo/delta": 0.12247386574745178,
      "fcm_dpo/margin": 20.089324951171875,
      "fcm_dpo/q_t": 0.3481150269508362,
      "grad_norm": 15.194877624511719,
      "learning_rate": 4.047230911780736e-07,
      "logits/chosen": -1.530800461769104,
      "logits/rejected": -1.014233946800232,
      "logps/chosen": -132.90155029296875,
      "logps/ref_chosen": -103.21904754638672,
      "logps/ref_rejected": -90.99221801757812,
      "logps/rejected": -140.7640380859375,
      "loss": 0.9273,
      "margin_dpo/margin_mean": 20.089324951171875,
      "margin_dpo/margin_std": 22.886226654052734,
      "step": 246
    },
    {
      "epoch": 0.36270190895741555,
      "fcm_dpo/beta": 0.03562181442975998,
      "fcm_dpo/delta": -0.22008347511291504,
      "fcm_dpo/margin": 29.32817268371582,
      "fcm_dpo/q_t": 0.2943752408027649,
      "grad_norm": 12.687663078308105,
      "learning_rate": 4.0371303213004814e-07,
      "logits/chosen": -2.017427921295166,
      "logits/rejected": -1.7663198709487915,
      "logps/chosen": -113.63011169433594,
      "logps/ref_chosen": -86.99437713623047,
      "logps/ref_rejected": -111.33802795410156,
      "logps/rejected": -167.30191040039062,
      "loss": 0.7633,
      "margin_dpo/margin_mean": 29.32817268371582,
      "margin_dpo/margin_std": 27.21298599243164,
      "step": 247
    },
    {
      "epoch": 0.3641703377386197,
      "fcm_dpo/beta": 0.03615882247686386,
      "fcm_dpo/delta": 0.11238771677017212,
      "fcm_dpo/margin": 20.618698120117188,
      "fcm_dpo/q_t": 0.3473028540611267,
      "grad_norm": 15.50047492980957,
      "learning_rate": 4.0269892260067197e-07,
      "logits/chosen": -1.745754599571228,
      "logits/rejected": -1.6654396057128906,
      "logps/chosen": -105.63675689697266,
      "logps/ref_chosen": -74.7855224609375,
      "logps/ref_rejected": -98.27689361572266,
      "logps/rejected": -149.746826171875,
      "loss": 0.9163,
      "margin_dpo/margin_mean": 20.618698120117188,
      "margin_dpo/margin_std": 23.03470230102539,
      "step": 248
    },
    {
      "epoch": 0.3656387665198238,
      "fcm_dpo/beta": 0.03689437732100487,
      "fcm_dpo/delta": 0.13981816172599792,
      "fcm_dpo/margin": 19.527952194213867,
      "fcm_dpo/q_t": 0.3553723394870758,
      "grad_norm": 17.28803825378418,
      "learning_rate": 4.0168078931267426e-07,
      "logits/chosen": -2.111118793487549,
      "logits/rejected": -1.628953456878662,
      "logps/chosen": -128.77549743652344,
      "logps/ref_chosen": -95.70379638671875,
      "logps/ref_rejected": -87.14646911621094,
      "logps/rejected": -139.74612426757812,
      "loss": 0.9751,
      "margin_dpo/margin_mean": 19.5279541015625,
      "margin_dpo/margin_std": 25.350357055664062,
      "step": 249
    },
    {
      "epoch": 0.3671071953010279,
      "fcm_dpo/beta": 0.03755343705415726,
      "fcm_dpo/delta": 0.10731877386569977,
      "fcm_dpo/margin": 19.97228240966797,
      "fcm_dpo/q_t": 0.34544017910957336,
      "grad_norm": 20.92696189880371,
      "learning_rate": 4.006586590948141e-07,
      "logits/chosen": -1.7340309619903564,
      "logits/rejected": -0.8914738893508911,
      "logps/chosen": -144.86044311523438,
      "logps/ref_chosen": -114.05221557617188,
      "logps/ref_rejected": -81.08767700195312,
      "logps/rejected": -131.86819458007812,
      "loss": 0.9467,
      "margin_dpo/margin_mean": 19.97228240966797,
      "margin_dpo/margin_std": 23.23900032043457,
      "step": 250
    },
    {
      "epoch": 0.368575624082232,
      "fcm_dpo/beta": 0.0382770374417305,
      "fcm_dpo/delta": 0.0928206592798233,
      "fcm_dpo/margin": 19.94355583190918,
      "fcm_dpo/q_t": 0.3502398431301117,
      "grad_norm": 21.311126708984375,
      "learning_rate": 3.9963255888117325e-07,
      "logits/chosen": -1.6832396984100342,
      "logits/rejected": -1.006043553352356,
      "logps/chosen": -127.50480651855469,
      "logps/ref_chosen": -97.71128845214844,
      "logps/ref_rejected": -83.52742004394531,
      "logps/rejected": -133.26449584960938,
      "loss": 0.9595,
      "margin_dpo/margin_mean": 19.94355583190918,
      "margin_dpo/margin_std": 25.059249877929688,
      "step": 251
    },
    {
      "epoch": 0.3700440528634361,
      "fcm_dpo/beta": 0.03917098790407181,
      "fcm_dpo/delta": 0.06389188766479492,
      "fcm_dpo/margin": 20.19826889038086,
      "fcm_dpo/q_t": 0.33797770738601685,
      "grad_norm": 16.346147537231445,
      "learning_rate": 3.9860251571044666e-07,
      "logits/chosen": -1.793283462524414,
      "logits/rejected": -1.3007917404174805,
      "logps/chosen": -138.48342895507812,
      "logps/ref_chosen": -108.9861068725586,
      "logps/ref_rejected": -91.56423950195312,
      "logps/rejected": -141.25982666015625,
      "loss": 0.9099,
      "margin_dpo/margin_mean": 20.19826889038086,
      "margin_dpo/margin_std": 22.625965118408203,
      "step": 252
    },
    {
      "epoch": 0.37151248164464024,
      "fcm_dpo/beta": 0.04027726128697395,
      "fcm_dpo/delta": 0.06578607112169266,
      "fcm_dpo/margin": 19.479793548583984,
      "fcm_dpo/q_t": 0.3372243344783783,
      "grad_norm": 16.55797004699707,
      "learning_rate": 3.9756855672522986e-07,
      "logits/chosen": -1.936989426612854,
      "logits/rejected": -1.7282047271728516,
      "logps/chosen": -132.63905334472656,
      "logps/ref_chosen": -100.21630096435547,
      "logps/ref_rejected": -105.67670440673828,
      "logps/rejected": -157.57925415039062,
      "loss": 0.9225,
      "margin_dpo/margin_mean": 19.479793548583984,
      "margin_dpo/margin_std": 21.367464065551758,
      "step": 253
    },
    {
      "epoch": 0.37298091042584436,
      "fcm_dpo/beta": 0.04016454517841339,
      "fcm_dpo/delta": 0.1159537136554718,
      "fcm_dpo/margin": 18.474327087402344,
      "fcm_dpo/q_t": 0.34890618920326233,
      "grad_norm": 16.3812255859375,
      "learning_rate": 3.965307091713037e-07,
      "logits/chosen": -1.952610731124878,
      "logits/rejected": -1.4213777780532837,
      "logps/chosen": -130.75515747070312,
      "logps/ref_chosen": -98.73518371582031,
      "logps/ref_rejected": -93.73825073242188,
      "logps/rejected": -144.23255920410156,
      "loss": 0.9529,
      "margin_dpo/margin_mean": 18.474328994750977,
      "margin_dpo/margin_std": 22.191251754760742,
      "step": 254
    },
    {
      "epoch": 0.3744493392070485,
      "fcm_dpo/beta": 0.03990204632282257,
      "fcm_dpo/delta": -0.15113875269889832,
      "fcm_dpo/margin": 24.718217849731445,
      "fcm_dpo/q_t": 0.30290794372558594,
      "grad_norm": 16.08772087097168,
      "learning_rate": 3.954890003969163e-07,
      "logits/chosen": -1.6271870136260986,
      "logits/rejected": -1.2749335765838623,
      "logps/chosen": -124.388427734375,
      "logps/ref_chosen": -90.382568359375,
      "logps/ref_rejected": -97.07624816894531,
      "logps/rejected": -155.80032348632812,
      "loss": 0.8062,
      "margin_dpo/margin_mean": 24.718219757080078,
      "margin_dpo/margin_std": 23.882701873779297,
      "step": 255
    },
    {
      "epoch": 0.37591776798825255,
      "fcm_dpo/beta": 0.03883664309978485,
      "fcm_dpo/delta": -0.16052256524562836,
      "fcm_dpo/margin": 25.614931106567383,
      "fcm_dpo/q_t": 0.3072439134120941,
      "grad_norm": 16.105266571044922,
      "learning_rate": 3.944434578520628e-07,
      "logits/chosen": -1.953477144241333,
      "logits/rejected": -1.5968413352966309,
      "logps/chosen": -119.0958023071289,
      "logps/ref_chosen": -88.7528076171875,
      "logps/ref_rejected": -98.49383544921875,
      "logps/rejected": -154.45175170898438,
      "loss": 0.8515,
      "margin_dpo/margin_mean": 25.614931106567383,
      "margin_dpo/margin_std": 27.952342987060547,
      "step": 256
    },
    {
      "epoch": 0.37738619676945667,
      "fcm_dpo/beta": 0.03744344413280487,
      "fcm_dpo/delta": -0.061067692935466766,
      "fcm_dpo/margin": 24.065053939819336,
      "fcm_dpo/q_t": 0.32052546739578247,
      "grad_norm": 17.075380325317383,
      "learning_rate": 3.933941090877615e-07,
      "logits/chosen": -1.7229259014129639,
      "logits/rejected": -1.3566169738769531,
      "logps/chosen": -115.55632019042969,
      "logps/ref_chosen": -82.80352020263672,
      "logps/ref_rejected": -85.8677978515625,
      "logps/rejected": -142.68563842773438,
      "loss": 0.864,
      "margin_dpo/margin_mean": 24.065053939819336,
      "margin_dpo/margin_std": 25.16031265258789,
      "step": 257
    },
    {
      "epoch": 0.3788546255506608,
      "fcm_dpo/beta": 0.03788026422262192,
      "fcm_dpo/delta": -0.03581918030977249,
      "fcm_dpo/margin": 23.28742027282715,
      "fcm_dpo/q_t": 0.31651657819747925,
      "grad_norm": 15.347949981689453,
      "learning_rate": 3.923409817553284e-07,
      "logits/chosen": -1.954576015472412,
      "logits/rejected": -1.6011340618133545,
      "logps/chosen": -122.21308135986328,
      "logps/ref_chosen": -90.187744140625,
      "logps/ref_rejected": -103.47068786621094,
      "logps/rejected": -158.783447265625,
      "loss": 0.8404,
      "margin_dpo/margin_mean": 23.28742218017578,
      "margin_dpo/margin_std": 22.27655601501465,
      "step": 258
    },
    {
      "epoch": 0.3803230543318649,
      "fcm_dpo/beta": 0.03793717548251152,
      "fcm_dpo/delta": 0.11459681391716003,
      "fcm_dpo/margin": 19.601879119873047,
      "fcm_dpo/q_t": 0.3489711880683899,
      "grad_norm": 16.28948974609375,
      "learning_rate": 3.9128410360564793e-07,
      "logits/chosen": -1.276770830154419,
      "logits/rejected": -1.0280699729919434,
      "logps/chosen": -126.50442504882812,
      "logps/ref_chosen": -90.77254486083984,
      "logps/ref_rejected": -94.58816528320312,
      "logps/rejected": -149.9219207763672,
      "loss": 0.9648,
      "margin_dpo/margin_mean": 19.601879119873047,
      "margin_dpo/margin_std": 25.395401000976562,
      "step": 259
    },
    {
      "epoch": 0.38179148311306904,
      "fcm_dpo/beta": 0.039135679602622986,
      "fcm_dpo/delta": 0.13330310583114624,
      "fcm_dpo/margin": 18.564983367919922,
      "fcm_dpo/q_t": 0.35755765438079834,
      "grad_norm": 18.12778663635254,
      "learning_rate": 3.9022350248844246e-07,
      "logits/chosen": -1.5422040224075317,
      "logits/rejected": -1.5151578187942505,
      "logps/chosen": -111.06421661376953,
      "logps/ref_chosen": -75.59269714355469,
      "logps/ref_rejected": -100.84554290771484,
      "logps/rejected": -154.8820343017578,
      "loss": 1.0363,
      "margin_dpo/margin_mean": 18.564983367919922,
      "margin_dpo/margin_std": 27.767108917236328,
      "step": 260
    },
    {
      "epoch": 0.3832599118942731,
      "fcm_dpo/beta": 0.038379594683647156,
      "fcm_dpo/delta": -0.1646619439125061,
      "fcm_dpo/margin": 25.985095977783203,
      "fcm_dpo/q_t": 0.2999562621116638,
      "grad_norm": 14.235591888427734,
      "learning_rate": 3.891592063515376e-07,
      "logits/chosen": -2.0275070667266846,
      "logits/rejected": -1.6710395812988281,
      "logps/chosen": -123.13007354736328,
      "logps/ref_chosen": -94.25491333007812,
      "logps/ref_rejected": -93.65699005126953,
      "logps/rejected": -148.51724243164062,
      "loss": 0.7772,
      "margin_dpo/margin_mean": 25.985097885131836,
      "margin_dpo/margin_std": 24.243711471557617,
      "step": 261
    },
    {
      "epoch": 0.38472834067547723,
      "fcm_dpo/beta": 0.037830717861652374,
      "fcm_dpo/delta": -0.05457323044538498,
      "fcm_dpo/margin": 23.777236938476562,
      "fcm_dpo/q_t": 0.3187333941459656,
      "grad_norm": 15.138924598693848,
      "learning_rate": 3.880912432401264e-07,
      "logits/chosen": -1.6778860092163086,
      "logits/rejected": -1.1244826316833496,
      "logps/chosen": -117.18390655517578,
      "logps/ref_chosen": -85.26730346679688,
      "logps/ref_rejected": -90.82609558105469,
      "logps/rejected": -146.51992797851562,
      "loss": 0.8426,
      "margin_dpo/margin_mean": 23.777236938476562,
      "margin_dpo/margin_std": 24.42955780029297,
      "step": 262
    },
    {
      "epoch": 0.38619676945668135,
      "fcm_dpo/beta": 0.03582540154457092,
      "fcm_dpo/delta": -0.3481602668762207,
      "fcm_dpo/margin": 32.29640197753906,
      "fcm_dpo/q_t": 0.2730727195739746,
      "grad_norm": 13.927531242370605,
      "learning_rate": 3.870196412960302e-07,
      "logits/chosen": -1.8874688148498535,
      "logits/rejected": -1.3352882862091064,
      "logps/chosen": -121.56033325195312,
      "logps/ref_chosen": -95.75790405273438,
      "logps/ref_rejected": -101.83377075195312,
      "logps/rejected": -159.93260192871094,
      "loss": 0.7319,
      "margin_dpo/margin_mean": 32.29640197753906,
      "margin_dpo/margin_std": 28.878860473632812,
      "step": 263
    },
    {
      "epoch": 0.3876651982378855,
      "fcm_dpo/beta": 0.03467824310064316,
      "fcm_dpo/delta": -0.09107442945241928,
      "fcm_dpo/margin": 26.880929946899414,
      "fcm_dpo/q_t": 0.31235480308532715,
      "grad_norm": 13.216853141784668,
      "learning_rate": 3.8594442875695665e-07,
      "logits/chosen": -1.9384407997131348,
      "logits/rejected": -1.6622414588928223,
      "logps/chosen": -118.2774658203125,
      "logps/ref_chosen": -90.6226577758789,
      "logps/ref_rejected": -100.32554626464844,
      "logps/rejected": -154.8612823486328,
      "loss": 0.8316,
      "margin_dpo/margin_mean": 26.88092803955078,
      "margin_dpo/margin_std": 26.818939208984375,
      "step": 264
    },
    {
      "epoch": 0.3891336270190896,
      "fcm_dpo/beta": 0.034085676074028015,
      "fcm_dpo/delta": -0.047373127192258835,
      "fcm_dpo/margin": 26.153051376342773,
      "fcm_dpo/q_t": 0.3252333402633667,
      "grad_norm": 12.807720184326172,
      "learning_rate": 3.848656339557562e-07,
      "logits/chosen": -1.829958200454712,
      "logits/rejected": -1.4463666677474976,
      "logps/chosen": -122.73773193359375,
      "logps/ref_chosen": -92.37232971191406,
      "logps/ref_rejected": -94.62757873535156,
      "logps/rejected": -151.14602661132812,
      "loss": 0.8721,
      "margin_dpo/margin_mean": 26.153051376342773,
      "margin_dpo/margin_std": 28.59259605407715,
      "step": 265
    },
    {
      "epoch": 0.39060205580029367,
      "fcm_dpo/beta": 0.03454948216676712,
      "fcm_dpo/delta": 0.06971698999404907,
      "fcm_dpo/margin": 22.744022369384766,
      "fcm_dpo/q_t": 0.3368097245693207,
      "grad_norm": 16.510560989379883,
      "learning_rate": 3.8378328531967507e-07,
      "logits/chosen": -2.1291229724884033,
      "logits/rejected": -1.4051086902618408,
      "logps/chosen": -136.4056396484375,
      "logps/ref_chosen": -102.20002746582031,
      "logps/ref_rejected": -74.36642456054688,
      "logps/rejected": -131.31605529785156,
      "loss": 0.8945,
      "margin_dpo/margin_mean": 22.744022369384766,
      "margin_dpo/margin_std": 23.8177433013916,
      "step": 266
    },
    {
      "epoch": 0.3920704845814978,
      "fcm_dpo/beta": 0.034008704125881195,
      "fcm_dpo/delta": -0.09747851639986038,
      "fcm_dpo/margin": 27.53805160522461,
      "fcm_dpo/q_t": 0.30594033002853394,
      "grad_norm": 13.239112854003906,
      "learning_rate": 3.8269741136960646e-07,
      "logits/chosen": -1.465944766998291,
      "logits/rejected": -0.8757386207580566,
      "logps/chosen": -133.18771362304688,
      "logps/ref_chosen": -104.28599548339844,
      "logps/ref_rejected": -95.98719024658203,
      "logps/rejected": -152.4269561767578,
      "loss": 0.7845,
      "margin_dpo/margin_mean": 27.538053512573242,
      "margin_dpo/margin_std": 23.816219329833984,
      "step": 267
    },
    {
      "epoch": 0.3935389133627019,
      "fcm_dpo/beta": 0.03355433791875839,
      "fcm_dpo/delta": -0.09911436587572098,
      "fcm_dpo/margin": 28.003337860107422,
      "fcm_dpo/q_t": 0.30674782395362854,
      "grad_norm": 13.598925590515137,
      "learning_rate": 3.8160804071933894e-07,
      "logits/chosen": -1.3583710193634033,
      "logits/rejected": -1.1435065269470215,
      "logps/chosen": -114.95004272460938,
      "logps/ref_chosen": -86.69622039794922,
      "logps/ref_rejected": -109.19184112548828,
      "logps/rejected": -165.44898986816406,
      "loss": 0.7982,
      "margin_dpo/margin_mean": 28.003337860107422,
      "margin_dpo/margin_std": 25.024091720581055,
      "step": 268
    },
    {
      "epoch": 0.39500734214390604,
      "fcm_dpo/beta": 0.03295326232910156,
      "fcm_dpo/delta": -0.11522669345140457,
      "fcm_dpo/margin": 28.961397171020508,
      "fcm_dpo/q_t": 0.3074827194213867,
      "grad_norm": 13.537002563476562,
      "learning_rate": 3.8051520207480204e-07,
      "logits/chosen": -1.6168208122253418,
      "logits/rejected": -1.1353833675384521,
      "logps/chosen": -134.0711669921875,
      "logps/ref_chosen": -104.97181701660156,
      "logps/ref_rejected": -112.47647094726562,
      "logps/rejected": -170.53720092773438,
      "loss": 0.808,
      "margin_dpo/margin_mean": 28.961397171020508,
      "margin_dpo/margin_std": 27.48983383178711,
      "step": 269
    },
    {
      "epoch": 0.3964757709251101,
      "fcm_dpo/beta": 0.03269010782241821,
      "fcm_dpo/delta": -0.0122775137424469,
      "fcm_dpo/margin": 26.342090606689453,
      "fcm_dpo/q_t": 0.3317517638206482,
      "grad_norm": 14.160609245300293,
      "learning_rate": 3.794189242333106e-07,
      "logits/chosen": -2.4010086059570312,
      "logits/rejected": -1.9900360107421875,
      "logps/chosen": -132.8267822265625,
      "logps/ref_chosen": -101.07383728027344,
      "logps/ref_rejected": -117.75289154052734,
      "logps/rejected": -175.84793090820312,
      "loss": 0.9059,
      "margin_dpo/margin_mean": 26.342090606689453,
      "margin_dpo/margin_std": 31.022964477539062,
      "step": 270
    },
    {
      "epoch": 0.39794419970631423,
      "fcm_dpo/beta": 0.0318414531648159,
      "fcm_dpo/delta": -0.1332307755947113,
      "fcm_dpo/margin": 30.452823638916016,
      "fcm_dpo/q_t": 0.2993617653846741,
      "grad_norm": 12.96033763885498,
      "learning_rate": 3.7831923608280514e-07,
      "logits/chosen": -1.2467565536499023,
      "logits/rejected": -0.7933920621871948,
      "logps/chosen": -123.05168151855469,
      "logps/ref_chosen": -96.72459411621094,
      "logps/ref_rejected": -98.5244140625,
      "logps/rejected": -155.3043212890625,
      "loss": 0.8195,
      "margin_dpo/margin_mean": 30.45282745361328,
      "margin_dpo/margin_std": 29.1860294342041,
      "step": 271
    },
    {
      "epoch": 0.39941262848751835,
      "fcm_dpo/beta": 0.03125782310962677,
      "fcm_dpo/delta": -0.09906208515167236,
      "fcm_dpo/margin": 30.06342124938965,
      "fcm_dpo/q_t": 0.31359294056892395,
      "grad_norm": 14.11276912689209,
      "learning_rate": 3.772161666010912e-07,
      "logits/chosen": -1.4559786319732666,
      "logits/rejected": -1.2856930494308472,
      "logps/chosen": -102.26274871826172,
      "logps/ref_chosen": -80.97721862792969,
      "logps/ref_rejected": -108.55535888671875,
      "logps/rejected": -159.90431213378906,
      "loss": 0.831,
      "margin_dpo/margin_mean": 30.06342124938965,
      "margin_dpo/margin_std": 30.461017608642578,
      "step": 272
    },
    {
      "epoch": 0.4008810572687225,
      "fcm_dpo/beta": 0.030184127390384674,
      "fcm_dpo/delta": -0.10749184340238571,
      "fcm_dpo/margin": 31.21113395690918,
      "fcm_dpo/q_t": 0.311345636844635,
      "grad_norm": 13.089473724365234,
      "learning_rate": 3.761097448550755e-07,
      "logits/chosen": -1.5064098834991455,
      "logits/rejected": -1.1125901937484741,
      "logps/chosen": -120.93928527832031,
      "logps/ref_chosen": -92.22460174560547,
      "logps/ref_rejected": -97.3630599975586,
      "logps/rejected": -157.28887939453125,
      "loss": 0.8571,
      "margin_dpo/margin_mean": 31.211132049560547,
      "margin_dpo/margin_std": 32.5693359375,
      "step": 273
    },
    {
      "epoch": 0.4023494860499266,
      "fcm_dpo/beta": 0.030762720853090286,
      "fcm_dpo/delta": 0.08655337244272232,
      "fcm_dpo/margin": 25.03656578063965,
      "fcm_dpo/q_t": 0.3384164571762085,
      "grad_norm": 14.684450149536133,
      "learning_rate": 3.75e-07,
      "logits/chosen": -1.8288625478744507,
      "logits/rejected": -1.4254413843154907,
      "logps/chosen": -112.06315612792969,
      "logps/ref_chosen": -82.26089477539062,
      "logps/ref_rejected": -83.87699890136719,
      "logps/rejected": -138.7158203125,
      "loss": 0.8814,
      "margin_dpo/margin_mean": 25.03656768798828,
      "margin_dpo/margin_std": 25.01040267944336,
      "step": 274
    },
    {
      "epoch": 0.40381791483113066,
      "fcm_dpo/beta": 0.03131476789712906,
      "fcm_dpo/delta": 0.010461606085300446,
      "fcm_dpo/margin": 26.782379150390625,
      "fcm_dpo/q_t": 0.32816171646118164,
      "grad_norm": 14.658917427062988,
      "learning_rate": 3.738869612786737e-07,
      "logits/chosen": -1.936241626739502,
      "logits/rejected": -1.6021487712860107,
      "logps/chosen": -104.04966735839844,
      "logps/ref_chosen": -79.68696594238281,
      "logps/ref_rejected": -98.7509765625,
      "logps/rejected": -149.89605712890625,
      "loss": 0.8889,
      "margin_dpo/margin_mean": 26.782379150390625,
      "margin_dpo/margin_std": 28.50400161743164,
      "step": 275
    },
    {
      "epoch": 0.4052863436123348,
      "fcm_dpo/beta": 0.0309858750551939,
      "fcm_dpo/delta": 0.06398846209049225,
      "fcm_dpo/margin": 25.471071243286133,
      "fcm_dpo/q_t": 0.33972838521003723,
      "grad_norm": 15.50623607635498,
      "learning_rate": 3.7277065802070204e-07,
      "logits/chosen": -1.787797212600708,
      "logits/rejected": -1.2390464544296265,
      "logps/chosen": -118.15924835205078,
      "logps/ref_chosen": -86.53970336914062,
      "logps/ref_rejected": -77.85394287109375,
      "logps/rejected": -134.94456481933594,
      "loss": 0.9182,
      "margin_dpo/margin_mean": 25.471071243286133,
      "margin_dpo/margin_std": 28.662944793701172,
      "step": 276
    },
    {
      "epoch": 0.4067547723935389,
      "fcm_dpo/beta": 0.0317692831158638,
      "fcm_dpo/delta": 0.08927221596240997,
      "fcm_dpo/margin": 24.16355323791504,
      "fcm_dpo/q_t": 0.3370756506919861,
      "grad_norm": 16.35001564025879,
      "learning_rate": 3.71651119641714e-07,
      "logits/chosen": -2.0091605186462402,
      "logits/rejected": -1.5972063541412354,
      "logps/chosen": -115.12983703613281,
      "logps/ref_chosen": -84.24411010742188,
      "logps/ref_rejected": -98.83421325683594,
      "logps/rejected": -153.88348388671875,
      "loss": 0.8847,
      "margin_dpo/margin_mean": 24.163555145263672,
      "margin_dpo/margin_std": 23.77988624572754,
      "step": 277
    },
    {
      "epoch": 0.40822320117474303,
      "fcm_dpo/beta": 0.0317951962351799,
      "fcm_dpo/delta": 0.00917709618806839,
      "fcm_dpo/margin": 26.432132720947266,
      "fcm_dpo/q_t": 0.32769787311553955,
      "grad_norm": 14.036833763122559,
      "learning_rate": 3.705283756425872e-07,
      "logits/chosen": -1.9561946392059326,
      "logits/rejected": -1.744471788406372,
      "logps/chosen": -112.0636978149414,
      "logps/ref_chosen": -82.431884765625,
      "logps/ref_rejected": -97.85691833496094,
      "logps/rejected": -153.92086791992188,
      "loss": 0.8902,
      "margin_dpo/margin_mean": 26.432132720947266,
      "margin_dpo/margin_std": 28.450267791748047,
      "step": 278
    },
    {
      "epoch": 0.40969162995594716,
      "fcm_dpo/beta": 0.03182877600193024,
      "fcm_dpo/delta": -0.011635363101959229,
      "fcm_dpo/margin": 27.01772689819336,
      "fcm_dpo/q_t": 0.3264973759651184,
      "grad_norm": 14.092531204223633,
      "learning_rate": 3.6940245560867e-07,
      "logits/chosen": -1.6284323930740356,
      "logits/rejected": -1.3937866687774658,
      "logps/chosen": -116.97288513183594,
      "logps/ref_chosen": -85.16799926757812,
      "logps/ref_rejected": -94.12664794921875,
      "logps/rejected": -152.94924926757812,
      "loss": 0.8698,
      "margin_dpo/margin_mean": 27.01772689819336,
      "margin_dpo/margin_std": 28.68284034729004,
      "step": 279
    },
    {
      "epoch": 0.4111600587371512,
      "fcm_dpo/beta": 0.03129285201430321,
      "fcm_dpo/delta": -0.15949808061122894,
      "fcm_dpo/margin": 31.758270263671875,
      "fcm_dpo/q_t": 0.29896456003189087,
      "grad_norm": 13.670494079589844,
      "learning_rate": 3.6827338920900253e-07,
      "logits/chosen": -1.997270107269287,
      "logits/rejected": -1.7795908451080322,
      "logps/chosen": -115.86480712890625,
      "logps/ref_chosen": -85.85641479492188,
      "logps/ref_rejected": -104.11859130859375,
      "logps/rejected": -165.88525390625,
      "loss": 0.7889,
      "margin_dpo/margin_mean": 31.758270263671875,
      "margin_dpo/margin_std": 28.985366821289062,
      "step": 280
    },
    {
      "epoch": 0.41262848751835535,
      "fcm_dpo/beta": 0.03030802309513092,
      "fcm_dpo/delta": -0.14983683824539185,
      "fcm_dpo/margin": 32.501609802246094,
      "fcm_dpo/q_t": 0.30702778697013855,
      "grad_norm": 11.849176406860352,
      "learning_rate": 3.6714120619553435e-07,
      "logits/chosen": -1.8329683542251587,
      "logits/rejected": -1.104137659072876,
      "logps/chosen": -117.03921508789062,
      "logps/ref_chosen": -86.55082702636719,
      "logps/ref_rejected": -88.62866973876953,
      "logps/rejected": -151.61868286132812,
      "loss": 0.8201,
      "margin_dpo/margin_mean": 32.501609802246094,
      "margin_dpo/margin_std": 33.797786712646484,
      "step": 281
    },
    {
      "epoch": 0.41409691629955947,
      "fcm_dpo/beta": 0.028760947287082672,
      "fcm_dpo/delta": -0.26554107666015625,
      "fcm_dpo/margin": 37.71876907348633,
      "fcm_dpo/q_t": 0.28032368421554565,
      "grad_norm": 11.502086639404297,
      "learning_rate": 3.660059364023408e-07,
      "logits/chosen": -1.680617332458496,
      "logits/rejected": -1.3186070919036865,
      "logps/chosen": -135.4449462890625,
      "logps/ref_chosen": -105.10511016845703,
      "logps/ref_rejected": -102.85336303710938,
      "logps/rejected": -170.91197204589844,
      "loss": 0.7132,
      "margin_dpo/margin_mean": 37.71876525878906,
      "margin_dpo/margin_std": 30.542362213134766,
      "step": 282
    },
    {
      "epoch": 0.4155653450807636,
      "fcm_dpo/beta": 0.029016414657235146,
      "fcm_dpo/delta": 0.11088653653860092,
      "fcm_dpo/margin": 25.754066467285156,
      "fcm_dpo/q_t": 0.35040855407714844,
      "grad_norm": 17.384960174560547,
      "learning_rate": 3.6486760974483685e-07,
      "logits/chosen": -1.991034746170044,
      "logits/rejected": -1.643927812576294,
      "logps/chosen": -130.67234802246094,
      "logps/ref_chosen": -95.05259704589844,
      "logps/ref_rejected": -103.54454803466797,
      "logps/rejected": -164.91836547851562,
      "loss": 0.9866,
      "margin_dpo/margin_mean": 25.754066467285156,
      "margin_dpo/margin_std": 34.688026428222656,
      "step": 283
    },
    {
      "epoch": 0.4170337738619677,
      "fcm_dpo/beta": 0.028016947209835052,
      "fcm_dpo/delta": -0.22263917326927185,
      "fcm_dpo/margin": 37.3685188293457,
      "fcm_dpo/q_t": 0.28344595432281494,
      "grad_norm": 11.067302703857422,
      "learning_rate": 3.6372625621898863e-07,
      "logits/chosen": -1.5473668575286865,
      "logits/rejected": -1.2372541427612305,
      "logps/chosen": -116.82695007324219,
      "logps/ref_chosen": -87.66647338867188,
      "logps/ref_rejected": -98.75103759765625,
      "logps/rejected": -165.280029296875,
      "loss": 0.7178,
      "margin_dpo/margin_mean": 37.36852264404297,
      "margin_dpo/margin_std": 29.709064483642578,
      "step": 284
    },
    {
      "epoch": 0.4185022026431718,
      "fcm_dpo/beta": 0.028491143137216568,
      "fcm_dpo/delta": 0.10801464319229126,
      "fcm_dpo/margin": 26.28765106201172,
      "fcm_dpo/q_t": 0.3376986086368561,
      "grad_norm": 18.838285446166992,
      "learning_rate": 3.625819059005228e-07,
      "logits/chosen": -2.3627262115478516,
      "logits/rejected": -2.08278751373291,
      "logps/chosen": -128.71075439453125,
      "logps/ref_chosen": -94.43303680419922,
      "logps/ref_rejected": -104.07194519042969,
      "logps/rejected": -164.6373291015625,
      "loss": 0.8901,
      "margin_dpo/margin_mean": 26.28765106201172,
      "margin_dpo/margin_std": 25.488826751708984,
      "step": 285
    },
    {
      "epoch": 0.4199706314243759,
      "fcm_dpo/beta": 0.02817067690193653,
      "fcm_dpo/delta": -0.06162068247795105,
      "fcm_dpo/margin": 32.15382766723633,
      "fcm_dpo/q_t": 0.312236487865448,
      "grad_norm": 14.387185096740723,
      "learning_rate": 3.614345889441346e-07,
      "logits/chosen": -1.7623761892318726,
      "logits/rejected": -1.3088512420654297,
      "logps/chosen": -140.90280151367188,
      "logps/ref_chosen": -103.72039794921875,
      "logps/ref_rejected": -96.25775909423828,
      "logps/rejected": -165.593994140625,
      "loss": 0.8213,
      "margin_dpo/margin_mean": 32.153831481933594,
      "margin_dpo/margin_std": 29.81972885131836,
      "step": 286
    },
    {
      "epoch": 0.42143906020558003,
      "fcm_dpo/beta": 0.0293489508330822,
      "fcm_dpo/delta": 0.2245098203420639,
      "fcm_dpo/margin": 21.69069480895996,
      "fcm_dpo/q_t": 0.37523770332336426,
      "grad_norm": 15.388336181640625,
      "learning_rate": 3.6028433558269275e-07,
      "logits/chosen": -2.1079835891723633,
      "logits/rejected": -1.5590211153030396,
      "logps/chosen": -137.75592041015625,
      "logps/ref_chosen": -93.88988494873047,
      "logps/ref_rejected": -83.33365631103516,
      "logps/rejected": -148.89036560058594,
      "loss": 1.0341,
      "margin_dpo/margin_mean": 21.69069480895996,
      "margin_dpo/margin_std": 31.85369873046875,
      "step": 287
    },
    {
      "epoch": 0.42290748898678415,
      "fcm_dpo/beta": 0.028778456151485443,
      "fcm_dpo/delta": -0.13858668506145477,
      "fcm_dpo/margin": 33.86872100830078,
      "fcm_dpo/q_t": 0.3073822855949402,
      "grad_norm": 13.900410652160645,
      "learning_rate": 3.5913117612644327e-07,
      "logits/chosen": -1.7789084911346436,
      "logits/rejected": -1.3899840116500854,
      "logps/chosen": -129.65771484375,
      "logps/ref_chosen": -88.15602111816406,
      "logps/ref_rejected": -93.28195190429688,
      "logps/rejected": -168.65237426757812,
      "loss": 0.8138,
      "margin_dpo/margin_mean": 33.86872100830078,
      "margin_dpo/margin_std": 33.53106689453125,
      "step": 288
    },
    {
      "epoch": 0.4243759177679883,
      "fcm_dpo/beta": 0.02798629179596901,
      "fcm_dpo/delta": -0.18543705344200134,
      "fcm_dpo/margin": 36.33726501464844,
      "fcm_dpo/q_t": 0.31428423523902893,
      "grad_norm": 12.854353904724121,
      "learning_rate": 3.5797514096221024e-07,
      "logits/chosen": -2.2386109828948975,
      "logits/rejected": -2.0394985675811768,
      "logps/chosen": -116.66321563720703,
      "logps/ref_chosen": -75.39292907714844,
      "logps/ref_rejected": -93.15428161621094,
      "logps/rejected": -170.7618408203125,
      "loss": 0.8394,
      "margin_dpo/margin_mean": 36.33726501464844,
      "margin_dpo/margin_std": 40.785675048828125,
      "step": 289
    },
    {
      "epoch": 0.42584434654919234,
      "fcm_dpo/beta": 0.02720276080071926,
      "fcm_dpo/delta": -0.1718001663684845,
      "fcm_dpo/margin": 36.90747833251953,
      "fcm_dpo/q_t": 0.30523180961608887,
      "grad_norm": 13.576156616210938,
      "learning_rate": 3.568162605525952e-07,
      "logits/chosen": -1.2316092252731323,
      "logits/rejected": -1.231179118156433,
      "logps/chosen": -133.9448699951172,
      "logps/ref_chosen": -88.04197692871094,
      "logps/ref_rejected": -123.212158203125,
      "logps/rejected": -206.02252197265625,
      "loss": 0.8157,
      "margin_dpo/margin_mean": 36.90747833251953,
      "margin_dpo/margin_std": 38.05936050415039,
      "step": 290
    },
    {
      "epoch": 0.42731277533039647,
      "fcm_dpo/beta": 0.027082961052656174,
      "fcm_dpo/delta": 0.08572201430797577,
      "fcm_dpo/margin": 28.44838523864746,
      "fcm_dpo/q_t": 0.33763566613197327,
      "grad_norm": 14.184231758117676,
      "learning_rate": 3.5565456543517485e-07,
      "logits/chosen": -2.211665153503418,
      "logits/rejected": -1.781091570854187,
      "logps/chosen": -138.7436065673828,
      "logps/ref_chosen": -94.09524536132812,
      "logps/ref_rejected": -96.05006408691406,
      "logps/rejected": -169.14682006835938,
      "loss": 0.9222,
      "margin_dpo/margin_mean": 28.448383331298828,
      "margin_dpo/margin_std": 32.05262756347656,
      "step": 291
    },
    {
      "epoch": 0.4287812041116006,
      "fcm_dpo/beta": 0.02674213796854019,
      "fcm_dpo/delta": -0.09594199061393738,
      "fcm_dpo/margin": 35.03672790527344,
      "fcm_dpo/q_t": 0.3002822995185852,
      "grad_norm": 12.321151733398438,
      "learning_rate": 3.5449008622169583e-07,
      "logits/chosen": -2.063217878341675,
      "logits/rejected": -1.6173973083496094,
      "logps/chosen": -132.1011505126953,
      "logps/ref_chosen": -88.25041198730469,
      "logps/ref_rejected": -96.41764068603516,
      "logps/rejected": -175.30511474609375,
      "loss": 0.7616,
      "margin_dpo/margin_mean": 35.03672790527344,
      "margin_dpo/margin_std": 26.89871597290039,
      "step": 292
    },
    {
      "epoch": 0.4302496328928047,
      "fcm_dpo/beta": 0.027519384399056435,
      "fcm_dpo/delta": 0.16997066140174866,
      "fcm_dpo/margin": 25.03589630126953,
      "fcm_dpo/q_t": 0.364579975605011,
      "grad_norm": 12.596840858459473,
      "learning_rate": 3.5332285359726846e-07,
      "logits/chosen": -2.3722639083862305,
      "logits/rejected": -1.8270535469055176,
      "logps/chosen": -134.981689453125,
      "logps/ref_chosen": -87.37654113769531,
      "logps/ref_rejected": -85.75579833984375,
      "logps/rejected": -158.39686584472656,
      "loss": 0.9934,
      "margin_dpo/margin_mean": 25.03589630126953,
      "margin_dpo/margin_std": 33.70214080810547,
      "step": 293
    },
    {
      "epoch": 0.43171806167400884,
      "fcm_dpo/beta": 0.02852361649274826,
      "fcm_dpo/delta": 0.2223101556301117,
      "fcm_dpo/margin": 22.482465744018555,
      "fcm_dpo/q_t": 0.37413203716278076,
      "grad_norm": 14.340274810791016,
      "learning_rate": 3.5215289831955786e-07,
      "logits/chosen": -1.5175267457962036,
      "logits/rejected": -1.3194074630737305,
      "logps/chosen": -123.25398254394531,
      "logps/ref_chosen": -73.50794982910156,
      "logps/ref_rejected": -88.08877563476562,
      "logps/rejected": -160.31727600097656,
      "loss": 1.0366,
      "margin_dpo/margin_mean": 22.482467651367188,
      "margin_dpo/margin_std": 34.42567443847656,
      "step": 294
    },
    {
      "epoch": 0.4331864904552129,
      "fcm_dpo/beta": 0.028234336525201797,
      "fcm_dpo/delta": -0.08705528825521469,
      "fcm_dpo/margin": 32.85896301269531,
      "fcm_dpo/q_t": 0.3156839907169342,
      "grad_norm": 13.020578384399414,
      "learning_rate": 3.509802512179737e-07,
      "logits/chosen": -1.6198558807373047,
      "logits/rejected": -1.4796977043151855,
      "logps/chosen": -125.44113159179688,
      "logps/ref_chosen": -77.76548767089844,
      "logps/ref_rejected": -94.24726867675781,
      "logps/rejected": -174.78189086914062,
      "loss": 0.8424,
      "margin_dpo/margin_mean": 32.85896301269531,
      "margin_dpo/margin_std": 34.66865539550781,
      "step": 295
    },
    {
      "epoch": 0.434654919236417,
      "fcm_dpo/beta": 0.02758461982011795,
      "fcm_dpo/delta": -0.12351512908935547,
      "fcm_dpo/margin": 34.80379104614258,
      "fcm_dpo/q_t": 0.3101541996002197,
      "grad_norm": 13.690773963928223,
      "learning_rate": 3.498049431928577e-07,
      "logits/chosen": -2.44696044921875,
      "logits/rejected": -2.000422716140747,
      "logps/chosen": -149.84738159179688,
      "logps/ref_chosen": -97.85641479492188,
      "logps/ref_rejected": -100.81631469726562,
      "logps/rejected": -187.61105346679688,
      "loss": 0.8245,
      "margin_dpo/margin_mean": 34.80379104614258,
      "margin_dpo/margin_std": 36.020755767822266,
      "step": 296
    },
    {
      "epoch": 0.43612334801762115,
      "fcm_dpo/beta": 0.027776416391134262,
      "fcm_dpo/delta": 0.026497282087802887,
      "fcm_dpo/margin": 29.71851921081543,
      "fcm_dpo/q_t": 0.3335018455982208,
      "grad_norm": 14.7246675491333,
      "learning_rate": 3.486270052146694e-07,
      "logits/chosen": -0.898391604423523,
      "logits/rejected": -0.5614018440246582,
      "logps/chosen": -142.22756958007812,
      "logps/ref_chosen": -88.56583404541016,
      "logps/ref_rejected": -101.55656433105469,
      "logps/rejected": -184.9368133544922,
      "loss": 0.8784,
      "margin_dpo/margin_mean": 29.718517303466797,
      "margin_dpo/margin_std": 31.882976531982422,
      "step": 297
    },
    {
      "epoch": 0.43759177679882527,
      "fcm_dpo/beta": 0.027368562296032906,
      "fcm_dpo/delta": -0.12677723169326782,
      "fcm_dpo/margin": 35.24578857421875,
      "fcm_dpo/q_t": 0.31449317932128906,
      "grad_norm": 16.305553436279297,
      "learning_rate": 3.474464683231698e-07,
      "logits/chosen": -1.7980400323867798,
      "logits/rejected": -1.693596363067627,
      "logps/chosen": -143.30047607421875,
      "logps/ref_chosen": -94.88043975830078,
      "logps/ref_rejected": -122.31101989746094,
      "logps/rejected": -205.97686767578125,
      "loss": 0.8283,
      "margin_dpo/margin_mean": 35.24578857421875,
      "margin_dpo/margin_std": 37.54370880126953,
      "step": 298
    },
    {
      "epoch": 0.4390602055800294,
      "fcm_dpo/beta": 0.026351025328040123,
      "fcm_dpo/delta": -0.15279431641101837,
      "fcm_dpo/margin": 37.456703186035156,
      "fcm_dpo/q_t": 0.3005375564098358,
      "grad_norm": 12.541773796081543,
      "learning_rate": 3.462633636266041e-07,
      "logits/chosen": -1.1973588466644287,
      "logits/rejected": -0.9508827924728394,
      "logps/chosen": -130.56175231933594,
      "logps/ref_chosen": -80.40835571289062,
      "logps/ref_rejected": -89.53717041015625,
      "logps/rejected": -177.1472625732422,
      "loss": 0.7802,
      "margin_dpo/margin_mean": 37.456703186035156,
      "margin_dpo/margin_std": 33.8513298034668,
      "step": 299
    },
    {
      "epoch": 0.44052863436123346,
      "fcm_dpo/beta": 0.026305727660655975,
      "fcm_dpo/delta": 0.02844477817416191,
      "fcm_dpo/margin": 31.3201904296875,
      "fcm_dpo/q_t": 0.33717983961105347,
      "grad_norm": 13.94998836517334,
      "learning_rate": 3.4507772230088147e-07,
      "logits/chosen": -1.4799878597259521,
      "logits/rejected": -1.256316065788269,
      "logps/chosen": -142.23712158203125,
      "logps/ref_chosen": -88.15890502929688,
      "logps/ref_rejected": -100.93919372558594,
      "logps/rejected": -186.3376007080078,
      "loss": 0.9115,
      "margin_dpo/margin_mean": 31.3201904296875,
      "margin_dpo/margin_std": 36.975074768066406,
      "step": 300
    },
    {
      "epoch": 0.44052863436123346,
      "eval_fcm_dpo/beta": 0.026378802955150604,
      "eval_logits/chosen": -1.7435120344161987,
      "eval_logits/rejected": -1.4185765981674194,
      "eval_logps/chosen": -155.74705505371094,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -175.83782958984375,
      "eval_loss": 0.4887809157371521,
      "eval_margin_dpo/margin_mean": 26.516590118408203,
      "eval_margin_dpo/margin_std": 33.869022369384766,
      "eval_runtime": 43.7724,
      "eval_samples_per_second": 53.435,
      "eval_steps_per_second": 1.691,
      "step": 300
    },
    {
      "epoch": 0.4419970631424376,
      "fcm_dpo/beta": 0.025531355291604996,
      "fcm_dpo/delta": -0.1449037790298462,
      "fcm_dpo/margin": 38.26066589355469,
      "fcm_dpo/q_t": 0.3068852424621582,
      "grad_norm": 12.821732521057129,
      "learning_rate": 3.4388957558875316e-07,
      "logits/chosen": -1.85687255859375,
      "logits/rejected": -1.4238810539245605,
      "logps/chosen": -135.0188751220703,
      "logps/ref_chosen": -85.50491333007812,
      "logps/ref_rejected": -98.91683959960938,
      "logps/rejected": -186.6914825439453,
      "loss": 0.8059,
      "margin_dpo/margin_mean": 38.26066589355469,
      "margin_dpo/margin_std": 37.27458953857422,
      "step": 301
    },
    {
      "epoch": 0.4434654919236417,
      "fcm_dpo/beta": 0.025708243250846863,
      "fcm_dpo/delta": 0.04868558049201965,
      "fcm_dpo/margin": 31.31707763671875,
      "fcm_dpo/q_t": 0.3379690647125244,
      "grad_norm": 14.223493576049805,
      "learning_rate": 3.426989547989902e-07,
      "logits/chosen": -1.2909713983535767,
      "logits/rejected": -1.102597951889038,
      "logps/chosen": -131.14703369140625,
      "logps/ref_chosen": -79.45040893554688,
      "logps/ref_rejected": -103.0591049194336,
      "logps/rejected": -186.07278442382812,
      "loss": 0.9164,
      "margin_dpo/margin_mean": 31.317073822021484,
      "margin_dpo/margin_std": 36.476539611816406,
      "step": 302
    },
    {
      "epoch": 0.44493392070484583,
      "fcm_dpo/beta": 0.025652453303337097,
      "fcm_dpo/delta": -0.0899156928062439,
      "fcm_dpo/margin": 36.312965393066406,
      "fcm_dpo/q_t": 0.3102775514125824,
      "grad_norm": 12.295307159423828,
      "learning_rate": 3.4150589130555773e-07,
      "logits/chosen": -1.6473402976989746,
      "logits/rejected": -1.2242443561553955,
      "logps/chosen": -141.21902465820312,
      "logps/ref_chosen": -95.59829711914062,
      "logps/ref_rejected": -92.0645751953125,
      "logps/rejected": -173.998291015625,
      "loss": 0.8163,
      "margin_dpo/margin_mean": 36.312965393066406,
      "margin_dpo/margin_std": 34.15446090698242,
      "step": 303
    },
    {
      "epoch": 0.44640234948604995,
      "fcm_dpo/beta": 0.025181569159030914,
      "fcm_dpo/delta": -0.0828293189406395,
      "fcm_dpo/margin": 36.74197769165039,
      "fcm_dpo/q_t": 0.30937981605529785,
      "grad_norm": 13.311530113220215,
      "learning_rate": 3.403104165467883e-07,
      "logits/chosen": -2.078594446182251,
      "logits/rejected": -1.6698689460754395,
      "logps/chosen": -146.96170043945312,
      "logps/ref_chosen": -99.4531478881836,
      "logps/ref_rejected": -98.08226013183594,
      "logps/rejected": -182.33279418945312,
      "loss": 0.8224,
      "margin_dpo/margin_mean": 36.74197769165039,
      "margin_dpo/margin_std": 34.821163177490234,
      "step": 304
    },
    {
      "epoch": 0.447870778267254,
      "fcm_dpo/beta": 0.024396728724241257,
      "fcm_dpo/delta": -0.0825144499540329,
      "fcm_dpo/margin": 37.75819778442383,
      "fcm_dpo/q_t": 0.31057485938072205,
      "grad_norm": 12.317713737487793,
      "learning_rate": 3.391125620245535e-07,
      "logits/chosen": -1.8674237728118896,
      "logits/rejected": -1.3910233974456787,
      "logps/chosen": -151.20004272460938,
      "logps/ref_chosen": -103.26339721679688,
      "logps/ref_rejected": -97.82897186279297,
      "logps/rejected": -183.52383422851562,
      "loss": 0.8155,
      "margin_dpo/margin_mean": 37.75819778442383,
      "margin_dpo/margin_std": 33.5549430847168,
      "step": 305
    },
    {
      "epoch": 0.44933920704845814,
      "fcm_dpo/beta": 0.023559659719467163,
      "fcm_dpo/delta": -0.24917837977409363,
      "fcm_dpo/margin": 45.43587875366211,
      "fcm_dpo/q_t": 0.2730311453342438,
      "grad_norm": 11.243393898010254,
      "learning_rate": 3.3791235930343417e-07,
      "logits/chosen": -2.1616029739379883,
      "logits/rejected": -1.6345137357711792,
      "logps/chosen": -141.40054321289062,
      "logps/ref_chosen": -100.3038330078125,
      "logps/ref_rejected": -92.23086547851562,
      "logps/rejected": -178.76345825195312,
      "loss": 0.6856,
      "margin_dpo/margin_mean": 45.435882568359375,
      "margin_dpo/margin_std": 30.269792556762695,
      "step": 306
    },
    {
      "epoch": 0.45080763582966227,
      "fcm_dpo/beta": 0.023632019758224487,
      "fcm_dpo/delta": 0.08696854114532471,
      "fcm_dpo/margin": 32.57453918457031,
      "fcm_dpo/q_t": 0.34471505880355835,
      "grad_norm": 13.035873413085938,
      "learning_rate": 3.367098400098881e-07,
      "logits/chosen": -1.7763289213180542,
      "logits/rejected": -1.4540574550628662,
      "logps/chosen": -144.75022888183594,
      "logps/ref_chosen": -96.64976501464844,
      "logps/ref_rejected": -93.34860229492188,
      "logps/rejected": -174.0236053466797,
      "loss": 0.9365,
      "margin_dpo/margin_mean": 32.57453918457031,
      "margin_dpo/margin_std": 39.58002471923828,
      "step": 307
    },
    {
      "epoch": 0.4522760646108664,
      "fcm_dpo/beta": 0.02385834977030754,
      "fcm_dpo/delta": 0.08316093683242798,
      "fcm_dpo/margin": 32.360687255859375,
      "fcm_dpo/q_t": 0.34850144386291504,
      "grad_norm": 14.49838638305664,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": -1.7061140537261963,
      "logits/rejected": -1.4719178676605225,
      "logps/chosen": -134.83767700195312,
      "logps/ref_chosen": -85.72467041015625,
      "logps/ref_rejected": -86.42317962646484,
      "logps/rejected": -167.89688110351562,
      "loss": 0.9396,
      "margin_dpo/margin_mean": 32.360687255859375,
      "margin_dpo/margin_std": 39.152732849121094,
      "step": 308
    },
    {
      "epoch": 0.45374449339207046,
      "fcm_dpo/beta": 0.024311821907758713,
      "fcm_dpo/delta": 0.010310536250472069,
      "fcm_dpo/margin": 34.5657958984375,
      "fcm_dpo/q_t": 0.32924583554267883,
      "grad_norm": 13.426247596740723,
      "learning_rate": 3.3429797851573183e-07,
      "logits/chosen": -1.7498160600662231,
      "logits/rejected": -1.3319708108901978,
      "logps/chosen": -136.19140625,
      "logps/ref_chosen": -83.15145874023438,
      "logps/ref_rejected": -86.58602142333984,
      "logps/rejected": -174.19175720214844,
      "loss": 0.8826,
      "margin_dpo/margin_mean": 34.5657958984375,
      "margin_dpo/margin_std": 37.12953567504883,
      "step": 309
    },
    {
      "epoch": 0.4552129221732746,
      "fcm_dpo/beta": 0.024511564522981644,
      "fcm_dpo/delta": 0.053582563996315,
      "fcm_dpo/margin": 32.65531921386719,
      "fcm_dpo/q_t": 0.33659428358078003,
      "grad_norm": 13.409820556640625,
      "learning_rate": 3.3308869986991487e-07,
      "logits/chosen": -2.131115198135376,
      "logits/rejected": -1.7278515100479126,
      "logps/chosen": -136.57640075683594,
      "logps/ref_chosen": -89.73799133300781,
      "logps/ref_rejected": -87.07083129882812,
      "logps/rejected": -166.5645751953125,
      "loss": 0.9131,
      "margin_dpo/margin_mean": 32.65532302856445,
      "margin_dpo/margin_std": 37.27210235595703,
      "step": 310
    },
    {
      "epoch": 0.4566813509544787,
      "fcm_dpo/beta": 0.02398892678320408,
      "fcm_dpo/delta": -0.13488170504570007,
      "fcm_dpo/margin": 40.49799346923828,
      "fcm_dpo/q_t": 0.30414602160453796,
      "grad_norm": 13.267024993896484,
      "learning_rate": 3.3187723175958346e-07,
      "logits/chosen": -1.6566599607467651,
      "logits/rejected": -0.9458609819412231,
      "logps/chosen": -147.330078125,
      "logps/ref_chosen": -88.52644348144531,
      "logps/ref_rejected": -82.26608276367188,
      "logps/rejected": -181.56771850585938,
      "loss": 0.7842,
      "margin_dpo/margin_mean": 40.49799346923828,
      "margin_dpo/margin_std": 37.099029541015625,
      "step": 311
    },
    {
      "epoch": 0.4581497797356828,
      "fcm_dpo/beta": 0.023747332394123077,
      "fcm_dpo/delta": -0.0758887305855751,
      "fcm_dpo/margin": 38.68158721923828,
      "fcm_dpo/q_t": 0.3152478337287903,
      "grad_norm": 13.058525085449219,
      "learning_rate": 3.306636061080487e-07,
      "logits/chosen": -1.466660737991333,
      "logits/rejected": -0.9308065176010132,
      "logps/chosen": -134.54989624023438,
      "logps/ref_chosen": -80.40069580078125,
      "logps/ref_rejected": -82.4471435546875,
      "logps/rejected": -175.27792358398438,
      "loss": 0.8461,
      "margin_dpo/margin_mean": 38.68158721923828,
      "margin_dpo/margin_std": 39.664390563964844,
      "step": 312
    },
    {
      "epoch": 0.45961820851688695,
      "fcm_dpo/beta": 0.023635469377040863,
      "fcm_dpo/delta": 0.02790502831339836,
      "fcm_dpo/margin": 34.8759765625,
      "fcm_dpo/q_t": 0.3363736867904663,
      "grad_norm": 11.786361694335938,
      "learning_rate": 3.2944785489547537e-07,
      "logits/chosen": -2.3114123344421387,
      "logits/rejected": -1.8879140615463257,
      "logps/chosen": -134.4079132080078,
      "logps/ref_chosen": -84.14190673828125,
      "logps/ref_rejected": -93.42684173583984,
      "logps/rejected": -178.56883239746094,
      "loss": 0.9262,
      "margin_dpo/margin_mean": 34.8759765625,
      "margin_dpo/margin_std": 42.29779815673828,
      "step": 313
    },
    {
      "epoch": 0.461086637298091,
      "fcm_dpo/beta": 0.02345544472336769,
      "fcm_dpo/delta": -0.014135785400867462,
      "fcm_dpo/margin": 36.76209259033203,
      "fcm_dpo/q_t": 0.32853952050209045,
      "grad_norm": 13.441848754882812,
      "learning_rate": 3.2823001015803857e-07,
      "logits/chosen": -1.984985113143921,
      "logits/rejected": -1.779584288597107,
      "logps/chosen": -139.60719299316406,
      "logps/ref_chosen": -88.80972290039062,
      "logps/ref_rejected": -104.15892028808594,
      "logps/rejected": -191.71847534179688,
      "loss": 0.8832,
      "margin_dpo/margin_mean": 36.76209259033203,
      "margin_dpo/margin_std": 40.621002197265625,
      "step": 314
    },
    {
      "epoch": 0.46255506607929514,
      "fcm_dpo/beta": 0.023481205105781555,
      "fcm_dpo/delta": -0.0164218470454216,
      "fcm_dpo/margin": 36.83607482910156,
      "fcm_dpo/q_t": 0.3287695348262787,
      "grad_norm": 13.707772254943848,
      "learning_rate": 3.270101039870797e-07,
      "logits/chosen": -1.4009931087493896,
      "logits/rejected": -1.214354395866394,
      "logps/chosen": -131.05386352539062,
      "logps/ref_chosen": -80.793701171875,
      "logps/ref_rejected": -92.13711547851562,
      "logps/rejected": -179.23336791992188,
      "loss": 0.9178,
      "margin_dpo/margin_mean": 36.83607482910156,
      "margin_dpo/margin_std": 44.07396697998047,
      "step": 315
    },
    {
      "epoch": 0.46402349486049926,
      "fcm_dpo/beta": 0.023532241582870483,
      "fcm_dpo/delta": -0.0018831361085176468,
      "fcm_dpo/margin": 36.19043731689453,
      "fcm_dpo/q_t": 0.3307906985282898,
      "grad_norm": 13.95568561553955,
      "learning_rate": 3.2578816852826086e-07,
      "logits/chosen": -1.5216803550720215,
      "logits/rejected": -1.5168416500091553,
      "logps/chosen": -138.3435516357422,
      "logps/ref_chosen": -84.18424987792969,
      "logps/ref_rejected": -105.6908187866211,
      "logps/rejected": -196.04055786132812,
      "loss": 0.8913,
      "margin_dpo/margin_mean": 36.19043731689453,
      "margin_dpo/margin_std": 41.48989486694336,
      "step": 316
    },
    {
      "epoch": 0.4654919236417034,
      "fcm_dpo/beta": 0.022490374743938446,
      "fcm_dpo/delta": -0.31194061040878296,
      "fcm_dpo/margin": 50.12665557861328,
      "fcm_dpo/q_t": 0.28289079666137695,
      "grad_norm": 13.945904731750488,
      "learning_rate": 3.2456423598071783e-07,
      "logits/chosen": -1.6591770648956299,
      "logits/rejected": -1.1557483673095703,
      "logps/chosen": -140.0654754638672,
      "logps/ref_chosen": -87.87348937988281,
      "logps/ref_rejected": -108.26519775390625,
      "logps/rejected": -210.58383178710938,
      "loss": 0.7508,
      "margin_dpo/margin_mean": 50.12665557861328,
      "margin_dpo/margin_std": 46.59101486206055,
      "step": 317
    },
    {
      "epoch": 0.4669603524229075,
      "fcm_dpo/beta": 0.02168363705277443,
      "fcm_dpo/delta": -0.09581036865711212,
      "fcm_dpo/margin": 43.18408966064453,
      "fcm_dpo/q_t": 0.30946460366249084,
      "grad_norm": 11.313399314880371,
      "learning_rate": 3.233383385962115e-07,
      "logits/chosen": -1.9839439392089844,
      "logits/rejected": -1.3179357051849365,
      "logps/chosen": -149.8084716796875,
      "logps/ref_chosen": -98.20553588867188,
      "logps/ref_rejected": -88.13629150390625,
      "logps/rejected": -182.92332458496094,
      "loss": 0.8149,
      "margin_dpo/margin_mean": 43.18408966064453,
      "margin_dpo/margin_std": 41.31195831298828,
      "step": 318
    },
    {
      "epoch": 0.4684287812041116,
      "fcm_dpo/beta": 0.021182440221309662,
      "fcm_dpo/delta": -0.12014549970626831,
      "fcm_dpo/margin": 45.22450256347656,
      "fcm_dpo/q_t": 0.30883777141571045,
      "grad_norm": 12.289721488952637,
      "learning_rate": 3.2211050867827805e-07,
      "logits/chosen": -1.5484849214553833,
      "logits/rejected": -1.4470410346984863,
      "logps/chosen": -126.79381561279297,
      "logps/ref_chosen": -82.50337219238281,
      "logps/ref_rejected": -118.65068054199219,
      "logps/rejected": -208.16561889648438,
      "loss": 0.8169,
      "margin_dpo/margin_mean": 45.22450256347656,
      "margin_dpo/margin_std": 44.638736724853516,
      "step": 319
    },
    {
      "epoch": 0.4698972099853157,
      "fcm_dpo/beta": 0.0210844948887825,
      "fcm_dpo/delta": 0.016227949410676956,
      "fcm_dpo/margin": 39.59357452392578,
      "fcm_dpo/q_t": 0.3285983204841614,
      "grad_norm": 11.511106491088867,
      "learning_rate": 3.208807785813777e-07,
      "logits/chosen": -2.1302032470703125,
      "logits/rejected": -1.9397833347320557,
      "logps/chosen": -132.902099609375,
      "logps/ref_chosen": -87.13722229003906,
      "logps/ref_rejected": -103.22412109375,
      "logps/rejected": -188.58255004882812,
      "loss": 0.8644,
      "margin_dpo/margin_mean": 39.59357452392578,
      "margin_dpo/margin_std": 41.0483512878418,
      "step": 320
    },
    {
      "epoch": 0.4713656387665198,
      "fcm_dpo/beta": 0.020797066390514374,
      "fcm_dpo/delta": -0.08268754184246063,
      "fcm_dpo/margin": 44.4124641418457,
      "fcm_dpo/q_t": 0.3112069368362427,
      "grad_norm": 11.961592674255371,
      "learning_rate": 3.1964918071004217e-07,
      "logits/chosen": -2.0953972339630127,
      "logits/rejected": -1.7250018119812012,
      "logps/chosen": -144.9522705078125,
      "logps/ref_chosen": -93.10466766357422,
      "logps/ref_rejected": -97.4681396484375,
      "logps/rejected": -193.72821044921875,
      "loss": 0.8341,
      "margin_dpo/margin_mean": 44.4124641418457,
      "margin_dpo/margin_std": 43.544097900390625,
      "step": 321
    },
    {
      "epoch": 0.47283406754772395,
      "fcm_dpo/beta": 0.020121555775403976,
      "fcm_dpo/delta": -0.17728671431541443,
      "fcm_dpo/margin": 50.00999450683594,
      "fcm_dpo/q_t": 0.2999124526977539,
      "grad_norm": 13.548306465148926,
      "learning_rate": 3.184157475180207e-07,
      "logits/chosen": -1.853193759918213,
      "logits/rejected": -1.5143083333969116,
      "logps/chosen": -141.55194091796875,
      "logps/ref_chosen": -92.19291687011719,
      "logps/ref_rejected": -100.89321899414062,
      "logps/rejected": -200.26223754882812,
      "loss": 0.8025,
      "margin_dpo/margin_mean": 50.00999450683594,
      "margin_dpo/margin_std": 47.50038146972656,
      "step": 322
    },
    {
      "epoch": 0.47430249632892807,
      "fcm_dpo/beta": 0.01991206780076027,
      "fcm_dpo/delta": -0.04079621657729149,
      "fcm_dpo/margin": 44.52705383300781,
      "fcm_dpo/q_t": 0.32042044401168823,
      "grad_norm": 11.607592582702637,
      "learning_rate": 3.171805115074251e-07,
      "logits/chosen": -2.05534291267395,
      "logits/rejected": -1.6881399154663086,
      "logps/chosen": -135.66567993164062,
      "logps/ref_chosen": -80.27210235595703,
      "logps/ref_rejected": -80.3033447265625,
      "logps/rejected": -180.22398376464844,
      "loss": 0.8341,
      "margin_dpo/margin_mean": 44.52705383300781,
      "margin_dpo/margin_std": 43.4844856262207,
      "step": 323
    },
    {
      "epoch": 0.47577092511013214,
      "fcm_dpo/beta": 0.0196855291724205,
      "fcm_dpo/delta": -0.09823101758956909,
      "fcm_dpo/margin": 47.70295715332031,
      "fcm_dpo/q_t": 0.3015575706958771,
      "grad_norm": 11.840632438659668,
      "learning_rate": 3.1594350522787295e-07,
      "logits/chosen": -1.6086432933807373,
      "logits/rejected": -0.9486087560653687,
      "logps/chosen": -153.7623748779297,
      "logps/ref_chosen": -95.72935485839844,
      "logps/ref_rejected": -94.16924285888672,
      "logps/rejected": -199.90521240234375,
      "loss": 0.7789,
      "margin_dpo/margin_mean": 47.70295715332031,
      "margin_dpo/margin_std": 39.34478759765625,
      "step": 324
    },
    {
      "epoch": 0.47723935389133626,
      "fcm_dpo/beta": 0.019420579075813293,
      "fcm_dpo/delta": -0.04867015779018402,
      "fcm_dpo/margin": 46.04854202270508,
      "fcm_dpo/q_t": 0.3144415020942688,
      "grad_norm": 10.232751846313477,
      "learning_rate": 3.147047612756302e-07,
      "logits/chosen": -1.746485710144043,
      "logits/rejected": -1.050299882888794,
      "logps/chosen": -152.50927734375,
      "logps/ref_chosen": -98.70686340332031,
      "logps/ref_rejected": -76.40809631347656,
      "logps/rejected": -176.25906372070312,
      "loss": 0.829,
      "margin_dpo/margin_mean": 46.04853820800781,
      "margin_dpo/margin_std": 43.221412658691406,
      "step": 325
    },
    {
      "epoch": 0.4787077826725404,
      "fcm_dpo/beta": 0.019502099603414536,
      "fcm_dpo/delta": 0.041523873805999756,
      "fcm_dpo/margin": 41.62855529785156,
      "fcm_dpo/q_t": 0.33376142382621765,
      "grad_norm": 13.21248722076416,
      "learning_rate": 3.134643122927519e-07,
      "logits/chosen": -2.4767727851867676,
      "logits/rejected": -1.8699431419372559,
      "logps/chosen": -168.19497680664062,
      "logps/ref_chosen": -104.66130828857422,
      "logps/ref_rejected": -86.82537841796875,
      "logps/rejected": -191.98760986328125,
      "loss": 0.8896,
      "margin_dpo/margin_mean": 41.62855529785156,
      "margin_dpo/margin_std": 44.87831115722656,
      "step": 326
    },
    {
      "epoch": 0.4801762114537445,
      "fcm_dpo/beta": 0.019457612186670303,
      "fcm_dpo/delta": -0.01857585832476616,
      "fcm_dpo/margin": 44.55260467529297,
      "fcm_dpo/q_t": 0.32346054911613464,
      "grad_norm": 12.107725143432617,
      "learning_rate": 3.1222219096622264e-07,
      "logits/chosen": -1.6513781547546387,
      "logits/rejected": -1.0370820760726929,
      "logps/chosen": -167.53587341308594,
      "logps/ref_chosen": -107.44023895263672,
      "logps/ref_rejected": -117.05323028564453,
      "logps/rejected": -221.70147705078125,
      "loss": 0.8525,
      "margin_dpo/margin_mean": 44.5526008605957,
      "margin_dpo/margin_std": 44.991912841796875,
      "step": 327
    },
    {
      "epoch": 0.48164464023494863,
      "fcm_dpo/beta": 0.01904640533030033,
      "fcm_dpo/delta": -0.14952132105827332,
      "fcm_dpo/margin": 51.70508575439453,
      "fcm_dpo/q_t": 0.30180302262306213,
      "grad_norm": 12.407594680786133,
      "learning_rate": 3.1097843002709427e-07,
      "logits/chosen": -1.4869141578674316,
      "logits/rejected": -1.2007091045379639,
      "logps/chosen": -150.2059783935547,
      "logps/ref_chosen": -86.6106948852539,
      "logps/ref_rejected": -97.40188598632812,
      "logps/rejected": -212.70225524902344,
      "loss": 0.7873,
      "margin_dpo/margin_mean": 51.70508575439453,
      "margin_dpo/margin_std": 47.76759338378906,
      "step": 328
    },
    {
      "epoch": 0.4831130690161527,
      "fcm_dpo/beta": 0.01887112483382225,
      "fcm_dpo/delta": 0.027751173824071884,
      "fcm_dpo/margin": 43.669822692871094,
      "fcm_dpo/q_t": 0.32791298627853394,
      "grad_norm": 13.570395469665527,
      "learning_rate": 3.0973306224962437e-07,
      "logits/chosen": -1.5214825868606567,
      "logits/rejected": -1.0124340057373047,
      "logps/chosen": -165.36264038085938,
      "logps/ref_chosen": -100.61308288574219,
      "logps/ref_rejected": -104.95370483398438,
      "logps/rejected": -213.37307739257812,
      "loss": 0.8621,
      "margin_dpo/margin_mean": 43.669822692871094,
      "margin_dpo/margin_std": 42.60850524902344,
      "step": 329
    },
    {
      "epoch": 0.4845814977973568,
      "fcm_dpo/beta": 0.018791306763887405,
      "fcm_dpo/delta": -0.051450639963150024,
      "fcm_dpo/margin": 47.70458984375,
      "fcm_dpo/q_t": 0.30650538206100464,
      "grad_norm": 11.095928192138672,
      "learning_rate": 3.084861204504122e-07,
      "logits/chosen": -1.7994372844696045,
      "logits/rejected": -1.7166423797607422,
      "logps/chosen": -135.48019409179688,
      "logps/ref_chosen": -76.95843505859375,
      "logps/ref_rejected": -92.81494140625,
      "logps/rejected": -199.041259765625,
      "loss": 0.7905,
      "margin_dpo/margin_mean": 47.704586029052734,
      "margin_dpo/margin_std": 37.720497131347656,
      "step": 330
    },
    {
      "epoch": 0.48604992657856094,
      "fcm_dpo/beta": 0.018632695078849792,
      "fcm_dpo/delta": -0.10063387453556061,
      "fcm_dpo/margin": 50.50764465332031,
      "fcm_dpo/q_t": 0.3032960295677185,
      "grad_norm": 15.277036666870117,
      "learning_rate": 3.072376374875335e-07,
      "logits/chosen": -1.9191899299621582,
      "logits/rejected": -1.6125640869140625,
      "logps/chosen": -145.8758544921875,
      "logps/ref_chosen": -83.05941772460938,
      "logps/ref_rejected": -94.13113403320312,
      "logps/rejected": -207.45521545410156,
      "loss": 0.7858,
      "margin_dpo/margin_mean": 50.50764465332031,
      "margin_dpo/margin_std": 42.87828063964844,
      "step": 331
    },
    {
      "epoch": 0.48751835535976507,
      "fcm_dpo/beta": 0.018833328038454056,
      "fcm_dpo/delta": 0.14665086567401886,
      "fcm_dpo/margin": 37.903587341308594,
      "fcm_dpo/q_t": 0.35199999809265137,
      "grad_norm": 12.529155731201172,
      "learning_rate": 3.059876462596758e-07,
      "logits/chosen": -2.0765371322631836,
      "logits/rejected": -1.6660399436950684,
      "logps/chosen": -149.06369018554688,
      "logps/ref_chosen": -81.93089294433594,
      "logps/ref_rejected": -82.84590148925781,
      "logps/rejected": -187.8822784423828,
      "loss": 0.9459,
      "margin_dpo/margin_mean": 37.903587341308594,
      "margin_dpo/margin_std": 45.02606201171875,
      "step": 332
    },
    {
      "epoch": 0.4889867841409692,
      "fcm_dpo/beta": 0.018546540290117264,
      "fcm_dpo/delta": -0.0819404274225235,
      "fcm_dpo/margin": 49.67053985595703,
      "fcm_dpo/q_t": 0.31946393847465515,
      "grad_norm": 12.655741691589355,
      "learning_rate": 3.0473617970527015e-07,
      "logits/chosen": -1.6623640060424805,
      "logits/rejected": -1.3435744047164917,
      "logps/chosen": -165.89602661132812,
      "logps/ref_chosen": -88.53257751464844,
      "logps/ref_rejected": -102.46601867675781,
      "logps/rejected": -229.5,
      "loss": 0.8978,
      "margin_dpo/margin_mean": 49.67053985595703,
      "margin_dpo/margin_std": 58.06535339355469,
      "step": 333
    },
    {
      "epoch": 0.49045521292217326,
      "fcm_dpo/beta": 0.018588945269584656,
      "fcm_dpo/delta": 0.01159542053937912,
      "fcm_dpo/margin": 45.11811828613281,
      "fcm_dpo/q_t": 0.3292047381401062,
      "grad_norm": 13.874351501464844,
      "learning_rate": 3.034832708016243e-07,
      "logits/chosen": -1.2766565084457397,
      "logits/rejected": -0.9840788841247559,
      "logps/chosen": -168.4891357421875,
      "logps/ref_chosen": -97.75778198242188,
      "logps/ref_rejected": -100.0058364868164,
      "logps/rejected": -215.85531616210938,
      "loss": 0.9167,
      "margin_dpo/margin_mean": 45.11811828613281,
      "margin_dpo/margin_std": 52.88778305053711,
      "step": 334
    },
    {
      "epoch": 0.4919236417033774,
      "fcm_dpo/beta": 0.01887522265315056,
      "fcm_dpo/delta": 0.058015674352645874,
      "fcm_dpo/margin": 42.20411682128906,
      "fcm_dpo/q_t": 0.3390098512172699,
      "grad_norm": 14.231380462646484,
      "learning_rate": 3.022289525640531e-07,
      "logits/chosen": -2.1570115089416504,
      "logits/rejected": -1.7516398429870605,
      "logps/chosen": -167.72457885742188,
      "logps/ref_chosen": -96.37603759765625,
      "logps/ref_rejected": -94.92401123046875,
      "logps/rejected": -208.47665405273438,
      "loss": 0.9201,
      "margin_dpo/margin_mean": 42.20411682128906,
      "margin_dpo/margin_std": 49.45446014404297,
      "step": 335
    },
    {
      "epoch": 0.4933920704845815,
      "fcm_dpo/beta": 0.01868092268705368,
      "fcm_dpo/delta": -0.11260117590427399,
      "fcm_dpo/margin": 50.96186828613281,
      "fcm_dpo/q_t": 0.3083822429180145,
      "grad_norm": 13.978559494018555,
      "learning_rate": 3.009732580450086e-07,
      "logits/chosen": -1.5539782047271729,
      "logits/rejected": -1.3770310878753662,
      "logps/chosen": -159.64161682128906,
      "logps/ref_chosen": -88.52952575683594,
      "logps/ref_rejected": -109.5557861328125,
      "logps/rejected": -231.62973022460938,
      "loss": 0.8231,
      "margin_dpo/margin_mean": 50.96186828613281,
      "margin_dpo/margin_std": 52.15556335449219,
      "step": 336
    },
    {
      "epoch": 0.4948604992657856,
      "fcm_dpo/beta": 0.01848072186112404,
      "fcm_dpo/delta": -0.04322163388133049,
      "fcm_dpo/margin": 48.11244583129883,
      "fcm_dpo/q_t": 0.3242299556732178,
      "grad_norm": 13.638964653015137,
      "learning_rate": 2.9971622033320914e-07,
      "logits/chosen": -2.27642822265625,
      "logits/rejected": -1.9854902029037476,
      "logps/chosen": -167.85414123535156,
      "logps/ref_chosen": -97.3123779296875,
      "logps/ref_rejected": -110.13134765625,
      "logps/rejected": -228.78553771972656,
      "loss": 0.8944,
      "margin_dpo/margin_mean": 48.11244583129883,
      "margin_dpo/margin_std": 54.82648468017578,
      "step": 337
    },
    {
      "epoch": 0.49632892804698975,
      "fcm_dpo/beta": 0.01835198514163494,
      "fcm_dpo/delta": 0.01314034778624773,
      "fcm_dpo/margin": 45.65751647949219,
      "fcm_dpo/q_t": 0.325469970703125,
      "grad_norm": 14.516743659973145,
      "learning_rate": 2.984578725527675e-07,
      "logits/chosen": -1.8865281343460083,
      "logits/rejected": -1.7105586528778076,
      "logps/chosen": -157.10043334960938,
      "logps/ref_chosen": -83.63011169433594,
      "logps/ref_rejected": -93.69235229492188,
      "logps/rejected": -212.8201904296875,
      "loss": 0.8727,
      "margin_dpo/margin_mean": 45.65751647949219,
      "margin_dpo/margin_std": 46.734107971191406,
      "step": 338
    },
    {
      "epoch": 0.4977973568281938,
      "fcm_dpo/beta": 0.018329763785004616,
      "fcm_dpo/delta": -0.07409132272005081,
      "fcm_dpo/margin": 50.01941680908203,
      "fcm_dpo/q_t": 0.31882256269454956,
      "grad_norm": 13.026541709899902,
      "learning_rate": 2.9719824786231796e-07,
      "logits/chosen": -2.9330520629882812,
      "logits/rejected": -2.518876552581787,
      "logps/chosen": -174.89187622070312,
      "logps/ref_chosen": -99.85182189941406,
      "logps/ref_rejected": -109.0743637084961,
      "logps/rejected": -234.1338348388672,
      "loss": 0.8771,
      "margin_dpo/margin_mean": 50.01941680908203,
      "margin_dpo/margin_std": 55.33876037597656,
      "step": 339
    },
    {
      "epoch": 0.49926578560939794,
      "fcm_dpo/beta": 0.018268339335918427,
      "fcm_dpo/delta": 0.02735849656164646,
      "fcm_dpo/margin": 45.13690185546875,
      "fcm_dpo/q_t": 0.33072808384895325,
      "grad_norm": 16.115903854370117,
      "learning_rate": 2.959373794541426e-07,
      "logits/chosen": -1.4078471660614014,
      "logits/rejected": -0.9559658765792847,
      "logps/chosen": -167.7802734375,
      "logps/ref_chosen": -84.97396850585938,
      "logps/ref_rejected": -78.80033874511719,
      "logps/rejected": -206.7435760498047,
      "loss": 0.8937,
      "margin_dpo/margin_mean": 45.13690185546875,
      "margin_dpo/margin_std": 51.02478790283203,
      "step": 340
    },
    {
      "epoch": 0.5007342143906021,
      "fcm_dpo/beta": 0.018367266282439232,
      "fcm_dpo/delta": 0.05237448215484619,
      "fcm_dpo/margin": 43.65577697753906,
      "fcm_dpo/q_t": 0.34013432264328003,
      "grad_norm": 15.512502670288086,
      "learning_rate": 2.946753005532965e-07,
      "logits/chosen": -1.4719302654266357,
      "logits/rejected": -1.1835095882415771,
      "logps/chosen": -173.26119995117188,
      "logps/ref_chosen": -85.26399230957031,
      "logps/ref_rejected": -106.08595275878906,
      "logps/rejected": -237.7389373779297,
      "loss": 0.9403,
      "margin_dpo/margin_mean": 43.65577697753906,
      "margin_dpo/margin_std": 53.599098205566406,
      "step": 341
    },
    {
      "epoch": 0.5022026431718062,
      "fcm_dpo/beta": 0.017494451254606247,
      "fcm_dpo/delta": -0.25881823897361755,
      "fcm_dpo/margin": 61.412132263183594,
      "fcm_dpo/q_t": 0.2814210057258606,
      "grad_norm": 13.322992324829102,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -1.374617338180542,
      "logits/rejected": -0.6580771803855896,
      "logps/chosen": -170.48294067382812,
      "logps/ref_chosen": -101.03860473632812,
      "logps/ref_rejected": -98.89851379394531,
      "logps/rejected": -229.75497436523438,
      "loss": 0.7611,
      "margin_dpo/margin_mean": 61.41213607788086,
      "margin_dpo/margin_std": 52.07002639770508,
      "step": 342
    },
    {
      "epoch": 0.5036710719530103,
      "fcm_dpo/beta": 0.017505202442407608,
      "fcm_dpo/delta": -0.06715908646583557,
      "fcm_dpo/margin": 51.96100616455078,
      "fcm_dpo/q_t": 0.3151499032974243,
      "grad_norm": 15.006378173828125,
      "learning_rate": 2.9214764433242476e-07,
      "logits/chosen": -1.9575207233428955,
      "logits/rejected": -1.8041001558303833,
      "logps/chosen": -157.88031005859375,
      "logps/ref_chosen": -83.74736022949219,
      "logps/ref_rejected": -116.57908630371094,
      "logps/rejected": -242.67303466796875,
      "loss": 0.8391,
      "margin_dpo/margin_mean": 51.96100997924805,
      "margin_dpo/margin_std": 50.270721435546875,
      "step": 343
    },
    {
      "epoch": 0.5051395007342144,
      "fcm_dpo/beta": 0.01725601777434349,
      "fcm_dpo/delta": -0.0286573339253664,
      "fcm_dpo/margin": 50.76178741455078,
      "fcm_dpo/q_t": 0.32474368810653687,
      "grad_norm": 13.056037902832031,
      "learning_rate": 2.9088213361849126e-07,
      "logits/chosen": -1.6103906631469727,
      "logits/rejected": -1.3356447219848633,
      "logps/chosen": -158.83035278320312,
      "logps/ref_chosen": -78.25396728515625,
      "logps/ref_rejected": -95.42870330810547,
      "logps/rejected": -226.76687622070312,
      "loss": 0.8756,
      "margin_dpo/margin_mean": 50.76179122924805,
      "margin_dpo/margin_std": 55.72649383544922,
      "step": 344
    },
    {
      "epoch": 0.5066079295154186,
      "fcm_dpo/beta": 0.01681181788444519,
      "fcm_dpo/delta": -0.10634002834558487,
      "fcm_dpo/margin": 56.23204803466797,
      "fcm_dpo/q_t": 0.3105340003967285,
      "grad_norm": 13.938704490661621,
      "learning_rate": 2.896155456223163e-07,
      "logits/chosen": -1.5161519050598145,
      "logits/rejected": -1.2150441408157349,
      "logps/chosen": -175.97979736328125,
      "logps/ref_chosen": -98.0794906616211,
      "logps/ref_rejected": -104.03477478027344,
      "logps/rejected": -238.16714477539062,
      "loss": 0.8146,
      "margin_dpo/margin_mean": 56.23204803466797,
      "margin_dpo/margin_std": 54.27198791503906,
      "step": 345
    },
    {
      "epoch": 0.5080763582966226,
      "fcm_dpo/beta": 0.017007581889629364,
      "fcm_dpo/delta": 0.06959997862577438,
      "fcm_dpo/margin": 46.195072174072266,
      "fcm_dpo/q_t": 0.3423931896686554,
      "grad_norm": 12.621767044067383,
      "learning_rate": 2.883479137196714e-07,
      "logits/chosen": -1.956493854522705,
      "logits/rejected": -1.6527358293533325,
      "logps/chosen": -170.6133270263672,
      "logps/ref_chosen": -80.68348693847656,
      "logps/ref_rejected": -85.89260864257812,
      "logps/rejected": -222.01751708984375,
      "loss": 0.9513,
      "margin_dpo/margin_mean": 46.19507598876953,
      "margin_dpo/margin_std": 58.70746994018555,
      "step": 346
    },
    {
      "epoch": 0.5095447870778267,
      "fcm_dpo/beta": 0.017326023429632187,
      "fcm_dpo/delta": 0.09608689695596695,
      "fcm_dpo/margin": 43.91358184814453,
      "fcm_dpo/q_t": 0.3499487340450287,
      "grad_norm": 14.319795608520508,
      "learning_rate": 2.8707927131383614e-07,
      "logits/chosen": -1.9093542098999023,
      "logits/rejected": -1.5683306455612183,
      "logps/chosen": -178.8655242919922,
      "logps/ref_chosen": -93.67938232421875,
      "logps/ref_rejected": -97.54251098632812,
      "logps/rejected": -226.64224243164062,
      "loss": 0.9717,
      "margin_dpo/margin_mean": 43.91358184814453,
      "margin_dpo/margin_std": 58.20018005371094,
      "step": 347
    },
    {
      "epoch": 0.5110132158590308,
      "fcm_dpo/beta": 0.01721569523215294,
      "fcm_dpo/delta": 0.02329825609922409,
      "fcm_dpo/margin": 47.98047637939453,
      "fcm_dpo/q_t": 0.3315523862838745,
      "grad_norm": 12.443164825439453,
      "learning_rate": 2.858096518347179e-07,
      "logits/chosen": -1.9739258289337158,
      "logits/rejected": -1.6834808588027954,
      "logps/chosen": -165.66465759277344,
      "logps/ref_chosen": -90.25224304199219,
      "logps/ref_rejected": -95.77027893066406,
      "logps/rejected": -219.1631622314453,
      "loss": 0.9045,
      "margin_dpo/margin_mean": 47.98047637939453,
      "margin_dpo/margin_std": 52.16603088378906,
      "step": 348
    },
    {
      "epoch": 0.5124816446402349,
      "fcm_dpo/beta": 0.01769782230257988,
      "fcm_dpo/delta": 0.037826307117938995,
      "fcm_dpo/margin": 45.96704864501953,
      "fcm_dpo/q_t": 0.3365427255630493,
      "grad_norm": 13.886954307556152,
      "learning_rate": 2.845390887379706e-07,
      "logits/chosen": -2.102787733078003,
      "logits/rejected": -1.881716251373291,
      "logps/chosen": -173.50393676757812,
      "logps/ref_chosen": -92.05809020996094,
      "logps/ref_rejected": -103.96527099609375,
      "logps/rejected": -231.378173828125,
      "loss": 0.9215,
      "margin_dpo/margin_mean": 45.96704864501953,
      "margin_dpo/margin_std": 53.656681060791016,
      "step": 349
    },
    {
      "epoch": 0.5139500734214391,
      "fcm_dpo/beta": 0.01799216866493225,
      "fcm_dpo/delta": 0.16198407113552094,
      "fcm_dpo/margin": 38.87583923339844,
      "fcm_dpo/q_t": 0.35764336585998535,
      "grad_norm": 15.808060646057129,
      "learning_rate": 2.8326761550411346e-07,
      "logits/chosen": -1.8025670051574707,
      "logits/rejected": -1.573468804359436,
      "logps/chosen": -175.3384552001953,
      "logps/ref_chosen": -87.05419921875,
      "logps/ref_rejected": -95.72193145751953,
      "logps/rejected": -222.88204956054688,
      "loss": 0.9587,
      "margin_dpo/margin_mean": 38.87583923339844,
      "margin_dpo/margin_std": 47.68621826171875,
      "step": 350
    },
    {
      "epoch": 0.5154185022026432,
      "fcm_dpo/beta": 0.017786063253879547,
      "fcm_dpo/delta": -0.17867089807987213,
      "fcm_dpo/margin": 56.837730407714844,
      "fcm_dpo/q_t": 0.29999619722366333,
      "grad_norm": 13.050240516662598,
      "learning_rate": 2.819952656376487e-07,
      "logits/chosen": -1.5670181512832642,
      "logits/rejected": -1.2213714122772217,
      "logps/chosen": -173.60598754882812,
      "logps/ref_chosen": -89.16903686523438,
      "logps/ref_rejected": -108.29801940917969,
      "logps/rejected": -249.57269287109375,
      "loss": 0.8264,
      "margin_dpo/margin_mean": 56.837730407714844,
      "margin_dpo/margin_std": 57.681922912597656,
      "step": 351
    },
    {
      "epoch": 0.5168869309838473,
      "fcm_dpo/beta": 0.01727548986673355,
      "fcm_dpo/delta": -0.09042805433273315,
      "fcm_dpo/margin": 53.94491195678711,
      "fcm_dpo/q_t": 0.3155178427696228,
      "grad_norm": 13.539352416992188,
      "learning_rate": 2.8072207266617854e-07,
      "logits/chosen": -1.7938660383224487,
      "logits/rejected": -1.2727184295654297,
      "logps/chosen": -175.17919921875,
      "logps/ref_chosen": -96.81147766113281,
      "logps/ref_rejected": -84.40483093261719,
      "logps/rejected": -216.71746826171875,
      "loss": 0.8342,
      "margin_dpo/margin_mean": 53.94491195678711,
      "margin_dpo/margin_std": 54.587371826171875,
      "step": 352
    },
    {
      "epoch": 0.5183553597650514,
      "fcm_dpo/beta": 0.017876289784908295,
      "fcm_dpo/delta": 0.26869261264801025,
      "fcm_dpo/margin": 33.43932342529297,
      "fcm_dpo/q_t": 0.3762671947479248,
      "grad_norm": 15.362466812133789,
      "learning_rate": 2.794480701395219e-07,
      "logits/chosen": -2.049345016479492,
      "logits/rejected": -1.686617136001587,
      "logps/chosen": -176.68536376953125,
      "logps/ref_chosen": -85.36248779296875,
      "logps/ref_rejected": -87.36732482910156,
      "logps/rejected": -212.1295166015625,
      "loss": 1.059,
      "margin_dpo/margin_mean": 33.43932342529297,
      "margin_dpo/margin_std": 50.67942810058594,
      "step": 353
    },
    {
      "epoch": 0.5198237885462555,
      "fcm_dpo/beta": 0.017914768308401108,
      "fcm_dpo/delta": -0.11532653868198395,
      "fcm_dpo/margin": 53.27729797363281,
      "fcm_dpo/q_t": 0.3020152449607849,
      "grad_norm": 12.614726066589355,
      "learning_rate": 2.781732916288303e-07,
      "logits/chosen": -1.9569498300552368,
      "logits/rejected": -1.506282091140747,
      "logps/chosen": -176.7467803955078,
      "logps/ref_chosen": -95.8011703491211,
      "logps/ref_rejected": -94.86614227294922,
      "logps/rejected": -229.0890350341797,
      "loss": 0.7736,
      "margin_dpo/margin_mean": 53.27729797363281,
      "margin_dpo/margin_std": 45.16783142089844,
      "step": 354
    },
    {
      "epoch": 0.5212922173274597,
      "fcm_dpo/beta": 0.017441321164369583,
      "fcm_dpo/delta": -0.04620569944381714,
      "fcm_dpo/margin": 51.005733489990234,
      "fcm_dpo/q_t": 0.3192770779132843,
      "grad_norm": 12.877270698547363,
      "learning_rate": 2.7689777072570284e-07,
      "logits/chosen": -2.404881477355957,
      "logits/rejected": -1.9403038024902344,
      "logps/chosen": -166.10281372070312,
      "logps/ref_chosen": -86.185546875,
      "logps/ref_rejected": -88.4615707397461,
      "logps/rejected": -219.3845672607422,
      "loss": 0.8567,
      "margin_dpo/margin_mean": 51.005733489990234,
      "margin_dpo/margin_std": 50.32048416137695,
      "step": 355
    },
    {
      "epoch": 0.5227606461086637,
      "fcm_dpo/beta": 0.018100222572684288,
      "fcm_dpo/delta": 0.17513401806354523,
      "fcm_dpo/margin": 37.909202575683594,
      "fcm_dpo/q_t": 0.3618093430995941,
      "grad_norm": 16.413511276245117,
      "learning_rate": 2.7562154104130176e-07,
      "logits/chosen": -1.8032923936843872,
      "logits/rejected": -1.3581966161727905,
      "logps/chosen": -185.53103637695312,
      "logps/ref_chosen": -90.818115234375,
      "logps/ref_rejected": -82.1273193359375,
      "logps/rejected": -214.7494354248047,
      "loss": 1.0311,
      "margin_dpo/margin_mean": 37.909202575683594,
      "margin_dpo/margin_std": 56.4036979675293,
      "step": 356
    },
    {
      "epoch": 0.5242290748898678,
      "fcm_dpo/beta": 0.01725354790687561,
      "fcm_dpo/delta": -0.25837036967277527,
      "fcm_dpo/margin": 62.15252685546875,
      "fcm_dpo/q_t": 0.28101831674575806,
      "grad_norm": 11.764604568481445,
      "learning_rate": 2.7434463620546594e-07,
      "logits/chosen": -2.014652729034424,
      "logits/rejected": -1.663172960281372,
      "logps/chosen": -164.86306762695312,
      "logps/ref_chosen": -87.5047607421875,
      "logps/ref_rejected": -92.24937438964844,
      "logps/rejected": -231.76019287109375,
      "loss": 0.7364,
      "margin_dpo/margin_mean": 62.15252685546875,
      "margin_dpo/margin_std": 49.077301025390625,
      "step": 357
    },
    {
      "epoch": 0.5256975036710719,
      "fcm_dpo/beta": 0.01754681020975113,
      "fcm_dpo/delta": 0.09242334961891174,
      "fcm_dpo/margin": 43.57078552246094,
      "fcm_dpo/q_t": 0.351772665977478,
      "grad_norm": 13.389106750488281,
      "learning_rate": 2.730670898658255e-07,
      "logits/chosen": -1.5166475772857666,
      "logits/rejected": -1.0220069885253906,
      "logps/chosen": -184.1082763671875,
      "logps/ref_chosen": -95.05546569824219,
      "logps/ref_rejected": -96.47611999511719,
      "logps/rejected": -229.0997314453125,
      "loss": 0.946,
      "margin_dpo/margin_mean": 43.57078552246094,
      "margin_dpo/margin_std": 55.526466369628906,
      "step": 358
    },
    {
      "epoch": 0.527165932452276,
      "fcm_dpo/beta": 0.017719190567731857,
      "fcm_dpo/delta": 0.1193242222070694,
      "fcm_dpo/margin": 41.656089782714844,
      "fcm_dpo/q_t": 0.36246833205223083,
      "grad_norm": 16.13714599609375,
      "learning_rate": 2.717889356869146e-07,
      "logits/chosen": -1.5216652154922485,
      "logits/rejected": -0.9881617426872253,
      "logps/chosen": -185.498046875,
      "logps/ref_chosen": -87.37332916259766,
      "logps/ref_rejected": -88.77188110351562,
      "logps/rejected": -228.55267333984375,
      "loss": 1.008,
      "margin_dpo/margin_mean": 41.65609359741211,
      "margin_dpo/margin_std": 59.36506652832031,
      "step": 359
    },
    {
      "epoch": 0.5286343612334802,
      "fcm_dpo/beta": 0.017821190878748894,
      "fcm_dpo/delta": -0.08467458188533783,
      "fcm_dpo/margin": 52.00859069824219,
      "fcm_dpo/q_t": 0.3092251420021057,
      "grad_norm": 14.783651351928711,
      "learning_rate": 2.7051020734928443e-07,
      "logits/chosen": -1.7419157028198242,
      "logits/rejected": -1.4053552150726318,
      "logps/chosen": -169.15045166015625,
      "logps/ref_chosen": -75.22344970703125,
      "logps/ref_rejected": -76.09400177001953,
      "logps/rejected": -222.02957153320312,
      "loss": 0.8292,
      "margin_dpo/margin_mean": 52.00859069824219,
      "margin_dpo/margin_std": 51.10163879394531,
      "step": 360
    },
    {
      "epoch": 0.5301027900146843,
      "fcm_dpo/beta": 0.017817631363868713,
      "fcm_dpo/delta": 0.017418846487998962,
      "fcm_dpo/margin": 46.802886962890625,
      "fcm_dpo/q_t": 0.34032896161079407,
      "grad_norm": 15.830314636230469,
      "learning_rate": 2.6923093854861593e-07,
      "logits/chosen": -1.6299736499786377,
      "logits/rejected": -1.3665854930877686,
      "logps/chosen": -176.39230346679688,
      "logps/ref_chosen": -81.95333099365234,
      "logps/ref_rejected": -96.67919921875,
      "logps/rejected": -237.92105102539062,
      "loss": 0.9434,
      "margin_dpo/margin_mean": 46.802886962890625,
      "margin_dpo/margin_std": 60.25688552856445,
      "step": 361
    },
    {
      "epoch": 0.5315712187958884,
      "fcm_dpo/beta": 0.017633792012929916,
      "fcm_dpo/delta": -0.048090770840644836,
      "fcm_dpo/margin": 50.67647933959961,
      "fcm_dpo/q_t": 0.3179241418838501,
      "grad_norm": 13.564910888671875,
      "learning_rate": 2.679511629948319e-07,
      "logits/chosen": -1.8477286100387573,
      "logits/rejected": -1.6221222877502441,
      "logps/chosen": -186.0591278076172,
      "logps/ref_chosen": -89.33971405029297,
      "logps/ref_rejected": -110.50196838378906,
      "logps/rejected": -257.89788818359375,
      "loss": 0.8685,
      "margin_dpo/margin_mean": 50.67647933959961,
      "margin_dpo/margin_std": 53.38725280761719,
      "step": 362
    },
    {
      "epoch": 0.5330396475770925,
      "fcm_dpo/beta": 0.016984201967716217,
      "fcm_dpo/delta": -0.2905126214027405,
      "fcm_dpo/margin": 65.30412292480469,
      "fcm_dpo/q_t": 0.28682398796081543,
      "grad_norm": 12.831151008605957,
      "learning_rate": 2.6667091441120816e-07,
      "logits/chosen": -1.8855948448181152,
      "logits/rejected": -1.4528388977050781,
      "logps/chosen": -172.280517578125,
      "logps/ref_chosen": -80.77499389648438,
      "logps/ref_rejected": -79.43267822265625,
      "logps/rejected": -236.2423095703125,
      "loss": 0.7683,
      "margin_dpo/margin_mean": 65.30412292480469,
      "margin_dpo/margin_std": 62.15265655517578,
      "step": 363
    },
    {
      "epoch": 0.5345080763582967,
      "fcm_dpo/beta": 0.016796786338090897,
      "fcm_dpo/delta": -0.002676934003829956,
      "fcm_dpo/margin": 50.632606506347656,
      "fcm_dpo/q_t": 0.32234978675842285,
      "grad_norm": 15.04246997833252,
      "learning_rate": 2.6539022653348575e-07,
      "logits/chosen": -1.8682522773742676,
      "logits/rejected": -1.6991798877716064,
      "logps/chosen": -169.2406005859375,
      "logps/ref_chosen": -76.52960205078125,
      "logps/ref_rejected": -97.56173706054688,
      "logps/rejected": -240.90536499023438,
      "loss": 0.8435,
      "margin_dpo/margin_mean": 50.632606506347656,
      "margin_dpo/margin_std": 46.46898651123047,
      "step": 364
    },
    {
      "epoch": 0.5359765051395007,
      "fcm_dpo/beta": 0.016505403444170952,
      "fcm_dpo/delta": -0.07894276827573776,
      "fcm_dpo/margin": 55.836181640625,
      "fcm_dpo/q_t": 0.317300021648407,
      "grad_norm": 13.384997367858887,
      "learning_rate": 2.641091331089811e-07,
      "logits/chosen": -1.6708288192749023,
      "logits/rejected": -1.5057486295700073,
      "logps/chosen": -172.15521240234375,
      "logps/ref_chosen": -80.54489135742188,
      "logps/ref_rejected": -99.32316589355469,
      "logps/rejected": -246.76966857910156,
      "loss": 0.8627,
      "margin_dpo/margin_mean": 55.836181640625,
      "margin_dpo/margin_std": 60.313575744628906,
      "step": 365
    },
    {
      "epoch": 0.5374449339207048,
      "fcm_dpo/beta": 0.01634395495057106,
      "fcm_dpo/delta": 0.0015336787328124046,
      "fcm_dpo/margin": 51.919151306152344,
      "fcm_dpo/q_t": 0.3336575925350189,
      "grad_norm": 11.990065574645996,
      "learning_rate": 2.6282766789569736e-07,
      "logits/chosen": -1.8974790573120117,
      "logits/rejected": -1.8965535163879395,
      "logps/chosen": -162.96044921875,
      "logps/ref_chosen": -72.23806762695312,
      "logps/ref_rejected": -91.67182922363281,
      "logps/rejected": -234.3133544921875,
      "loss": 0.8653,
      "margin_dpo/margin_mean": 51.919151306152344,
      "margin_dpo/margin_std": 56.507080078125,
      "step": 366
    },
    {
      "epoch": 0.5389133627019089,
      "fcm_dpo/beta": 0.016237348318099976,
      "fcm_dpo/delta": -0.053293175995349884,
      "fcm_dpo/margin": 55.335411071777344,
      "fcm_dpo/q_t": 0.31658363342285156,
      "grad_norm": 12.581396102905273,
      "learning_rate": 2.615458646614349e-07,
      "logits/chosen": -1.756191611289978,
      "logits/rejected": -1.4468399286270142,
      "logps/chosen": -173.73377990722656,
      "logps/ref_chosen": -85.42201232910156,
      "logps/ref_rejected": -83.51779174804688,
      "logps/rejected": -227.16497802734375,
      "loss": 0.8241,
      "margin_dpo/margin_mean": 55.335411071777344,
      "margin_dpo/margin_std": 53.84605407714844,
      "step": 367
    },
    {
      "epoch": 0.540381791483113,
      "fcm_dpo/beta": 0.016097472980618477,
      "fcm_dpo/delta": -0.09254693984985352,
      "fcm_dpo/margin": 57.9866828918457,
      "fcm_dpo/q_t": 0.30905455350875854,
      "grad_norm": 14.102049827575684,
      "learning_rate": 2.6026375718290083e-07,
      "logits/chosen": -1.4592063426971436,
      "logits/rejected": -1.4003169536590576,
      "logps/chosen": -160.42959594726562,
      "logps/ref_chosen": -73.84112548828125,
      "logps/ref_rejected": -104.69528198242188,
      "logps/rejected": -249.2704315185547,
      "loss": 0.8385,
      "margin_dpo/margin_mean": 57.98668670654297,
      "margin_dpo/margin_std": 56.874576568603516,
      "step": 368
    },
    {
      "epoch": 0.5418502202643172,
      "fcm_dpo/beta": 0.015790659934282303,
      "fcm_dpo/delta": -0.06228688731789589,
      "fcm_dpo/margin": 57.41761779785156,
      "fcm_dpo/q_t": 0.3094673752784729,
      "grad_norm": 13.252199172973633,
      "learning_rate": 2.589813792448196e-07,
      "logits/chosen": -1.6390339136123657,
      "logits/rejected": -1.2733685970306396,
      "logps/chosen": -190.71282958984375,
      "logps/ref_chosen": -98.46195983886719,
      "logps/ref_rejected": -99.15672302246094,
      "logps/rejected": -248.82521057128906,
      "loss": 0.7867,
      "margin_dpo/margin_mean": 57.41761779785156,
      "margin_dpo/margin_std": 47.635658264160156,
      "step": 369
    },
    {
      "epoch": 0.5433186490455213,
      "fcm_dpo/beta": 0.0161726176738739,
      "fcm_dpo/delta": 0.18013787269592285,
      "fcm_dpo/margin": 42.137481689453125,
      "fcm_dpo/q_t": 0.3603252172470093,
      "grad_norm": 13.428014755249023,
      "learning_rate": 2.5769876463904263e-07,
      "logits/chosen": -1.8658051490783691,
      "logits/rejected": -1.6577569246292114,
      "logps/chosen": -198.522705078125,
      "logps/ref_chosen": -96.77574157714844,
      "logps/ref_rejected": -102.74411010742188,
      "logps/rejected": -246.62857055664062,
      "loss": 1.002,
      "margin_dpo/margin_mean": 42.137481689453125,
      "margin_dpo/margin_std": 57.25678253173828,
      "step": 370
    },
    {
      "epoch": 0.5447870778267254,
      "fcm_dpo/beta": 0.01677476242184639,
      "fcm_dpo/delta": 0.16446369886398315,
      "fcm_dpo/margin": 41.4566535949707,
      "fcm_dpo/q_t": 0.3613440692424774,
      "grad_norm": 15.086153030395508,
      "learning_rate": 2.5641594716365744e-07,
      "logits/chosen": -1.7068711519241333,
      "logits/rejected": -1.3464912176132202,
      "logps/chosen": -201.3501434326172,
      "logps/ref_chosen": -102.99020385742188,
      "logps/ref_rejected": -105.35062408447266,
      "logps/rejected": -245.167236328125,
      "loss": 0.9969,
      "margin_dpo/margin_mean": 41.45665740966797,
      "margin_dpo/margin_std": 56.3448600769043,
      "step": 371
    },
    {
      "epoch": 0.5462555066079295,
      "fcm_dpo/beta": 0.01618875563144684,
      "fcm_dpo/delta": -0.26988279819488525,
      "fcm_dpo/margin": 67.3760986328125,
      "fcm_dpo/q_t": 0.28218111395835876,
      "grad_norm": 11.737579345703125,
      "learning_rate": 2.551329606220976e-07,
      "logits/chosen": -1.7287421226501465,
      "logits/rejected": -1.1179594993591309,
      "logps/chosen": -174.77841186523438,
      "logps/ref_chosen": -91.02623748779297,
      "logps/ref_rejected": -85.6102294921875,
      "logps/rejected": -236.738525390625,
      "loss": 0.7396,
      "margin_dpo/margin_mean": 67.37609100341797,
      "margin_dpo/margin_std": 57.7528076171875,
      "step": 372
    },
    {
      "epoch": 0.5477239353891337,
      "fcm_dpo/beta": 0.015897128731012344,
      "fcm_dpo/delta": -0.0048202937468886375,
      "fcm_dpo/margin": 53.741973876953125,
      "fcm_dpo/q_t": 0.32611092925071716,
      "grad_norm": 13.899093627929688,
      "learning_rate": 2.538498388222517e-07,
      "logits/chosen": -2.0247459411621094,
      "logits/rejected": -1.520031213760376,
      "logps/chosen": -195.14593505859375,
      "logps/ref_chosen": -100.07307434082031,
      "logps/ref_rejected": -91.11634826660156,
      "logps/rejected": -239.93118286132812,
      "loss": 0.8742,
      "margin_dpo/margin_mean": 53.741973876953125,
      "margin_dpo/margin_std": 56.848350524902344,
      "step": 373
    },
    {
      "epoch": 0.5491923641703378,
      "fcm_dpo/beta": 0.015747731551527977,
      "fcm_dpo/delta": -0.030241642147302628,
      "fcm_dpo/margin": 55.66587829589844,
      "fcm_dpo/q_t": 0.31644949316978455,
      "grad_norm": 11.643533706665039,
      "learning_rate": 2.525666155755725e-07,
      "logits/chosen": -2.1849594116210938,
      "logits/rejected": -1.7997386455535889,
      "logps/chosen": -183.28863525390625,
      "logps/ref_chosen": -98.01234436035156,
      "logps/ref_rejected": -100.75224304199219,
      "logps/rejected": -241.69439697265625,
      "loss": 0.8144,
      "margin_dpo/margin_mean": 55.66587829589844,
      "margin_dpo/margin_std": 48.722808837890625,
      "step": 374
    },
    {
      "epoch": 0.5506607929515418,
      "fcm_dpo/beta": 0.0154128298163414,
      "fcm_dpo/delta": -0.0929117500782013,
      "fcm_dpo/margin": 60.347686767578125,
      "fcm_dpo/q_t": 0.3120383322238922,
      "grad_norm": 12.819354057312012,
      "learning_rate": 2.512833246961859e-07,
      "logits/chosen": -1.465893268585205,
      "logits/rejected": -1.2448041439056396,
      "logps/chosen": -173.93695068359375,
      "logps/ref_chosen": -84.39627838134766,
      "logps/ref_rejected": -95.7188491821289,
      "logps/rejected": -245.60720825195312,
      "loss": 0.8251,
      "margin_dpo/margin_mean": 60.347686767578125,
      "margin_dpo/margin_std": 57.37383270263672,
      "step": 375
    },
    {
      "epoch": 0.5521292217327459,
      "fcm_dpo/beta": 0.015097381547093391,
      "fcm_dpo/delta": -0.16518311202526093,
      "fcm_dpo/margin": 66.08840942382812,
      "fcm_dpo/q_t": 0.29718315601348877,
      "grad_norm": 12.352048873901367,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.7917392253875732,
      "logits/rejected": -1.50510835647583,
      "logps/chosen": -180.96434020996094,
      "logps/ref_chosen": -95.73770141601562,
      "logps/ref_rejected": -111.97331237792969,
      "logps/rejected": -263.2883605957031,
      "loss": 0.7832,
      "margin_dpo/margin_mean": 66.08840942382812,
      "margin_dpo/margin_std": 59.88749313354492,
      "step": 376
    },
    {
      "epoch": 0.55359765051395,
      "fcm_dpo/beta": 0.01455208845436573,
      "fcm_dpo/delta": -0.15709523856639862,
      "fcm_dpo/margin": 67.94343566894531,
      "fcm_dpo/q_t": 0.3027934730052948,
      "grad_norm": 13.825796127319336,
      "learning_rate": 2.487166753038141e-07,
      "logits/chosen": -1.534698724746704,
      "logits/rejected": -1.4000335931777954,
      "logps/chosen": -172.19622802734375,
      "logps/ref_chosen": -78.92694854736328,
      "logps/ref_rejected": -105.89892578125,
      "logps/rejected": -267.11163330078125,
      "loss": 0.8043,
      "margin_dpo/margin_mean": 67.94343566894531,
      "margin_dpo/margin_std": 64.65965270996094,
      "step": 377
    },
    {
      "epoch": 0.5550660792951542,
      "fcm_dpo/beta": 0.014748094603419304,
      "fcm_dpo/delta": 0.0158497616648674,
      "fcm_dpo/margin": 56.504417419433594,
      "fcm_dpo/q_t": 0.32408106327056885,
      "grad_norm": 12.6776704788208,
      "learning_rate": 2.4743338442442754e-07,
      "logits/chosen": -1.9457380771636963,
      "logits/rejected": -1.6649649143218994,
      "logps/chosen": -166.82522583007812,
      "logps/ref_chosen": -75.37240600585938,
      "logps/ref_rejected": -93.70409393310547,
      "logps/rejected": -241.6613311767578,
      "loss": 0.8687,
      "margin_dpo/margin_mean": 56.50442123413086,
      "margin_dpo/margin_std": 55.60420227050781,
      "step": 378
    },
    {
      "epoch": 0.5565345080763583,
      "fcm_dpo/beta": 0.01469915546476841,
      "fcm_dpo/delta": 0.051874738186597824,
      "fcm_dpo/margin": 54.58234786987305,
      "fcm_dpo/q_t": 0.3433055281639099,
      "grad_norm": 12.682414054870605,
      "learning_rate": 2.461501611777483e-07,
      "logits/chosen": -1.7052133083343506,
      "logits/rejected": -1.5816318988800049,
      "logps/chosen": -190.00857543945312,
      "logps/ref_chosen": -88.85568237304688,
      "logps/ref_rejected": -121.38114166259766,
      "logps/rejected": -277.11639404296875,
      "loss": 0.9467,
      "margin_dpo/margin_mean": 54.58234786987305,
      "margin_dpo/margin_std": 68.8104019165039,
      "step": 379
    },
    {
      "epoch": 0.5580029368575624,
      "fcm_dpo/beta": 0.014686446636915207,
      "fcm_dpo/delta": -0.03623585402965546,
      "fcm_dpo/margin": 60.12630081176758,
      "fcm_dpo/q_t": 0.3278425931930542,
      "grad_norm": 14.065608024597168,
      "learning_rate": 2.4486703937790243e-07,
      "logits/chosen": -1.41129732131958,
      "logits/rejected": -1.3641774654388428,
      "logps/chosen": -190.62673950195312,
      "logps/ref_chosen": -79.83321380615234,
      "logps/ref_rejected": -110.7447509765625,
      "logps/rejected": -281.6645812988281,
      "loss": 0.8771,
      "margin_dpo/margin_mean": 60.12629699707031,
      "margin_dpo/margin_std": 68.94208526611328,
      "step": 380
    },
    {
      "epoch": 0.5594713656387665,
      "fcm_dpo/beta": 0.014767487533390522,
      "fcm_dpo/delta": 0.05422665923833847,
      "fcm_dpo/margin": 54.18187713623047,
      "fcm_dpo/q_t": 0.3340100944042206,
      "grad_norm": 14.278923034667969,
      "learning_rate": 2.435840528363426e-07,
      "logits/chosen": -1.7580182552337646,
      "logits/rejected": -1.2122209072113037,
      "logps/chosen": -183.7044677734375,
      "logps/ref_chosen": -86.56683349609375,
      "logps/ref_rejected": -84.416748046875,
      "logps/rejected": -235.73626708984375,
      "loss": 0.887,
      "margin_dpo/margin_mean": 54.18187713623047,
      "margin_dpo/margin_std": 57.312503814697266,
      "step": 381
    },
    {
      "epoch": 0.5609397944199707,
      "fcm_dpo/beta": 0.01462157815694809,
      "fcm_dpo/delta": -0.017210952937602997,
      "fcm_dpo/margin": 59.04779052734375,
      "fcm_dpo/q_t": 0.3251911401748657,
      "grad_norm": 12.774848937988281,
      "learning_rate": 2.4230123536095745e-07,
      "logits/chosen": -1.765809178352356,
      "logits/rejected": -1.4844367504119873,
      "logps/chosen": -201.82684326171875,
      "logps/ref_chosen": -104.46180725097656,
      "logps/ref_rejected": -114.60223388671875,
      "logps/rejected": -271.0150451660156,
      "loss": 0.8592,
      "margin_dpo/margin_mean": 59.047786712646484,
      "margin_dpo/margin_std": 59.06350326538086,
      "step": 382
    },
    {
      "epoch": 0.5624082232011748,
      "fcm_dpo/beta": 0.014921590685844421,
      "fcm_dpo/delta": 0.09733906388282776,
      "fcm_dpo/margin": 50.940895080566406,
      "fcm_dpo/q_t": 0.34485840797424316,
      "grad_norm": 13.45592212677002,
      "learning_rate": 2.4101862075518037e-07,
      "logits/chosen": -1.6310830116271973,
      "logits/rejected": -1.5183734893798828,
      "logps/chosen": -186.19158935546875,
      "logps/ref_chosen": -82.11607360839844,
      "logps/ref_rejected": -98.77716064453125,
      "logps/rejected": -253.79356384277344,
      "loss": 0.9232,
      "margin_dpo/margin_mean": 50.940895080566406,
      "margin_dpo/margin_std": 57.292198181152344,
      "step": 383
    },
    {
      "epoch": 0.5638766519823789,
      "fcm_dpo/beta": 0.01533111184835434,
      "fcm_dpo/delta": 0.04174236208200455,
      "fcm_dpo/margin": 52.73126983642578,
      "fcm_dpo/q_t": 0.33828550577163696,
      "grad_norm": 15.4530029296875,
      "learning_rate": 2.397362428170992e-07,
      "logits/chosen": -1.7482203245162964,
      "logits/rejected": -1.5194519758224487,
      "logps/chosen": -191.6592254638672,
      "logps/ref_chosen": -80.58525085449219,
      "logps/ref_rejected": -91.44789123535156,
      "logps/rejected": -255.2531280517578,
      "loss": 0.9616,
      "margin_dpo/margin_mean": 52.73126983642578,
      "margin_dpo/margin_std": 66.3970718383789,
      "step": 384
    },
    {
      "epoch": 0.5653450807635829,
      "fcm_dpo/beta": 0.014804725535213947,
      "fcm_dpo/delta": -0.0903623104095459,
      "fcm_dpo/margin": 62.7065544128418,
      "fcm_dpo/q_t": 0.31812626123428345,
      "grad_norm": 13.666119575500488,
      "learning_rate": 2.3845413533856514e-07,
      "logits/chosen": -2.33363676071167,
      "logits/rejected": -1.75101637840271,
      "logps/chosen": -186.3187255859375,
      "logps/ref_chosen": -92.76016235351562,
      "logps/ref_rejected": -84.45997619628906,
      "logps/rejected": -240.72511291503906,
      "loss": 0.858,
      "margin_dpo/margin_mean": 62.70655822753906,
      "margin_dpo/margin_std": 65.78327941894531,
      "step": 385
    },
    {
      "epoch": 0.566813509544787,
      "fcm_dpo/beta": 0.014840572141110897,
      "fcm_dpo/delta": -0.0528637170791626,
      "fcm_dpo/margin": 60.51496505737305,
      "fcm_dpo/q_t": 0.3133012652397156,
      "grad_norm": 11.866787910461426,
      "learning_rate": 2.3717233210430254e-07,
      "logits/chosen": -1.5037336349487305,
      "logits/rejected": -1.2165539264678955,
      "logps/chosen": -182.65261840820312,
      "logps/ref_chosen": -85.35896301269531,
      "logps/ref_rejected": -96.99065399169922,
      "logps/rejected": -254.79925537109375,
      "loss": 0.8583,
      "margin_dpo/margin_mean": 60.51496124267578,
      "margin_dpo/margin_std": 61.9644775390625,
      "step": 386
    },
    {
      "epoch": 0.5682819383259912,
      "fcm_dpo/beta": 0.014781072735786438,
      "fcm_dpo/delta": -0.010366106405854225,
      "fcm_dpo/margin": 58.13372039794922,
      "fcm_dpo/q_t": 0.3218954801559448,
      "grad_norm": 12.85347843170166,
      "learning_rate": 2.3589086689101889e-07,
      "logits/chosen": -2.2964346408843994,
      "logits/rejected": -1.622833490371704,
      "logps/chosen": -212.4945068359375,
      "logps/ref_chosen": -102.68814086914062,
      "logps/ref_rejected": -99.42858123779297,
      "logps/rejected": -267.3686828613281,
      "loss": 0.8492,
      "margin_dpo/margin_mean": 58.13372039794922,
      "margin_dpo/margin_std": 56.552650451660156,
      "step": 387
    },
    {
      "epoch": 0.5697503671071953,
      "fcm_dpo/beta": 0.01442895270884037,
      "fcm_dpo/delta": -0.12326682358980179,
      "fcm_dpo/margin": 66.6114501953125,
      "fcm_dpo/q_t": 0.3023766279220581,
      "grad_norm": 12.544069290161133,
      "learning_rate": 2.3460977346651428e-07,
      "logits/chosen": -1.5236001014709473,
      "logits/rejected": -1.368851900100708,
      "logps/chosen": -191.8466796875,
      "logps/ref_chosen": -83.58523559570312,
      "logps/ref_rejected": -110.10037231445312,
      "logps/rejected": -284.9732666015625,
      "loss": 0.7869,
      "margin_dpo/margin_mean": 66.6114501953125,
      "margin_dpo/margin_std": 58.575645446777344,
      "step": 388
    },
    {
      "epoch": 0.5712187958883994,
      "fcm_dpo/beta": 0.014386516064405441,
      "fcm_dpo/delta": 0.0063179126009345055,
      "fcm_dpo/margin": 58.680145263671875,
      "fcm_dpo/q_t": 0.33595648407936096,
      "grad_norm": 13.45885944366455,
      "learning_rate": 2.3332908558879177e-07,
      "logits/chosen": -2.3460850715637207,
      "logits/rejected": -1.9088528156280518,
      "logps/chosen": -203.8497314453125,
      "logps/ref_chosen": -89.822265625,
      "logps/ref_rejected": -84.85376739501953,
      "logps/rejected": -257.5613708496094,
      "loss": 0.9254,
      "margin_dpo/margin_mean": 58.680145263671875,
      "margin_dpo/margin_std": 72.35084533691406,
      "step": 389
    },
    {
      "epoch": 0.5726872246696035,
      "fcm_dpo/beta": 0.014441296458244324,
      "fcm_dpo/delta": 0.09785392880439758,
      "fcm_dpo/margin": 52.516563415527344,
      "fcm_dpo/q_t": 0.3560144901275635,
      "grad_norm": 13.203153610229492,
      "learning_rate": 2.320488370051681e-07,
      "logits/chosen": -1.580621361732483,
      "logits/rejected": -1.3598695993423462,
      "logps/chosen": -193.17552185058594,
      "logps/ref_chosen": -77.53244018554688,
      "logps/ref_rejected": -92.58392333984375,
      "logps/rejected": -260.7435607910156,
      "loss": 0.9781,
      "margin_dpo/margin_mean": 52.51655960083008,
      "margin_dpo/margin_std": 71.42833709716797,
      "step": 390
    },
    {
      "epoch": 0.5741556534508077,
      "fcm_dpo/beta": 0.01456206664443016,
      "fcm_dpo/delta": -0.014334239065647125,
      "fcm_dpo/margin": 59.23310852050781,
      "fcm_dpo/q_t": 0.3304310441017151,
      "grad_norm": 13.462164878845215,
      "learning_rate": 2.3076906145138405e-07,
      "logits/chosen": -1.8457610607147217,
      "logits/rejected": -1.3921658992767334,
      "logps/chosen": -193.5087890625,
      "logps/ref_chosen": -88.2329330444336,
      "logps/ref_rejected": -93.00756072998047,
      "logps/rejected": -257.51654052734375,
      "loss": 0.8957,
      "margin_dpo/margin_mean": 59.23310852050781,
      "margin_dpo/margin_std": 68.13093566894531,
      "step": 391
    },
    {
      "epoch": 0.5756240822320118,
      "fcm_dpo/beta": 0.014434915035963058,
      "fcm_dpo/delta": -0.12481804192066193,
      "fcm_dpo/margin": 66.70355224609375,
      "fcm_dpo/q_t": 0.30818355083465576,
      "grad_norm": 14.184881210327148,
      "learning_rate": 2.294897926507156e-07,
      "logits/chosen": -1.6148556470870972,
      "logits/rejected": -1.4798262119293213,
      "logps/chosen": -193.44656372070312,
      "logps/ref_chosen": -87.74442291259766,
      "logps/ref_rejected": -108.57071685791016,
      "logps/rejected": -280.9764404296875,
      "loss": 0.8476,
      "margin_dpo/margin_mean": 66.70355224609375,
      "margin_dpo/margin_std": 70.49634552001953,
      "step": 392
    },
    {
      "epoch": 0.5770925110132159,
      "fcm_dpo/beta": 0.014169027097523212,
      "fcm_dpo/delta": -0.032773129642009735,
      "fcm_dpo/margin": 62.09144592285156,
      "fcm_dpo/q_t": 0.32115113735198975,
      "grad_norm": 11.957456588745117,
      "learning_rate": 2.2821106431308543e-07,
      "logits/chosen": -1.8445684909820557,
      "logits/rejected": -1.5407391786575317,
      "logps/chosen": -170.1377410888672,
      "logps/ref_chosen": -76.56381225585938,
      "logps/ref_rejected": -79.72618103027344,
      "logps/rejected": -235.39157104492188,
      "loss": 0.8532,
      "margin_dpo/margin_mean": 62.09144592285156,
      "margin_dpo/margin_std": 63.155155181884766,
      "step": 393
    },
    {
      "epoch": 0.57856093979442,
      "fcm_dpo/beta": 0.013979214243590832,
      "fcm_dpo/delta": -0.07188640534877777,
      "fcm_dpo/margin": 65.45079040527344,
      "fcm_dpo/q_t": 0.3089185357093811,
      "grad_norm": 14.39035701751709,
      "learning_rate": 2.2693291013417452e-07,
      "logits/chosen": -1.5587331056594849,
      "logits/rejected": -1.3254432678222656,
      "logps/chosen": -182.34947204589844,
      "logps/ref_chosen": -84.69866180419922,
      "logps/ref_rejected": -97.457763671875,
      "logps/rejected": -260.55938720703125,
      "loss": 0.802,
      "margin_dpo/margin_mean": 65.45079040527344,
      "margin_dpo/margin_std": 57.37220764160156,
      "step": 394
    },
    {
      "epoch": 0.580029368575624,
      "fcm_dpo/beta": 0.014071676880121231,
      "fcm_dpo/delta": 0.0893394872546196,
      "fcm_dpo/margin": 54.5306282043457,
      "fcm_dpo/q_t": 0.3583162724971771,
      "grad_norm": 14.767304420471191,
      "learning_rate": 2.2565536379453404e-07,
      "logits/chosen": -2.528858184814453,
      "logits/rejected": -2.1432571411132812,
      "logps/chosen": -189.65113830566406,
      "logps/ref_chosen": -88.88763427734375,
      "logps/ref_rejected": -91.0455322265625,
      "logps/rejected": -246.33966064453125,
      "loss": 1.0618,
      "margin_dpo/margin_mean": 54.53062438964844,
      "margin_dpo/margin_std": 86.91376495361328,
      "step": 395
    },
    {
      "epoch": 0.5814977973568282,
      "fcm_dpo/beta": 0.014645865187048912,
      "fcm_dpo/delta": 0.19641628861427307,
      "fcm_dpo/margin": 45.51504135131836,
      "fcm_dpo/q_t": 0.3645893633365631,
      "grad_norm": 17.304218292236328,
      "learning_rate": 2.2437845895869825e-07,
      "logits/chosen": -1.776867151260376,
      "logits/rejected": -1.2883884906768799,
      "logps/chosen": -211.24041748046875,
      "logps/ref_chosen": -102.47132873535156,
      "logps/ref_rejected": -93.23478698730469,
      "logps/rejected": -247.51890563964844,
      "loss": 1.0063,
      "margin_dpo/margin_mean": 45.515037536621094,
      "margin_dpo/margin_std": 62.76597595214844,
      "step": 396
    },
    {
      "epoch": 0.5829662261380323,
      "fcm_dpo/beta": 0.013984641060233116,
      "fcm_dpo/delta": -0.33509910106658936,
      "fcm_dpo/margin": 81.79728698730469,
      "fcm_dpo/q_t": 0.2831918001174927,
      "grad_norm": 13.865079879760742,
      "learning_rate": 2.2310222927429716e-07,
      "logits/chosen": -2.201831817626953,
      "logits/rejected": -1.599250078201294,
      "logps/chosen": -198.48558044433594,
      "logps/ref_chosen": -101.04441833496094,
      "logps/ref_rejected": -108.8511734008789,
      "logps/rejected": -288.0896301269531,
      "loss": 0.7855,
      "margin_dpo/margin_mean": 81.79728698730469,
      "margin_dpo/margin_std": 80.89474487304688,
      "step": 397
    },
    {
      "epoch": 0.5844346549192364,
      "fcm_dpo/beta": 0.013811508193612099,
      "fcm_dpo/delta": 0.005452306941151619,
      "fcm_dpo/margin": 61.17559814453125,
      "fcm_dpo/q_t": 0.3352923095226288,
      "grad_norm": 14.45501708984375,
      "learning_rate": 2.2182670837116972e-07,
      "logits/chosen": -1.9161015748977661,
      "logits/rejected": -1.6111736297607422,
      "logps/chosen": -197.73312377929688,
      "logps/ref_chosen": -95.04922485351562,
      "logps/ref_rejected": -114.00563049316406,
      "logps/rejected": -277.8651123046875,
      "loss": 0.9266,
      "margin_dpo/margin_mean": 61.175594329833984,
      "margin_dpo/margin_std": 75.55554962158203,
      "step": 398
    },
    {
      "epoch": 0.5859030837004405,
      "fcm_dpo/beta": 0.013482306152582169,
      "fcm_dpo/delta": -0.218872532248497,
      "fcm_dpo/margin": 77.61836242675781,
      "fcm_dpo/q_t": 0.284706175327301,
      "grad_norm": 12.366701126098633,
      "learning_rate": 2.2055192986047804e-07,
      "logits/chosen": -2.152554512023926,
      "logits/rejected": -1.3891927003860474,
      "logps/chosen": -189.8502960205078,
      "logps/ref_chosen": -96.63854217529297,
      "logps/ref_rejected": -83.53645324707031,
      "logps/rejected": -254.3665771484375,
      "loss": 0.7319,
      "margin_dpo/margin_mean": 77.61836242675781,
      "margin_dpo/margin_std": 61.290470123291016,
      "step": 399
    },
    {
      "epoch": 0.5873715124816447,
      "fcm_dpo/beta": 0.012850401923060417,
      "fcm_dpo/delta": -0.1984182745218277,
      "fcm_dpo/margin": 79.97297668457031,
      "fcm_dpo/q_t": 0.29666027426719666,
      "grad_norm": 14.71058464050293,
      "learning_rate": 2.192779273338215e-07,
      "logits/chosen": -2.1020350456237793,
      "logits/rejected": -1.615342140197754,
      "logps/chosen": -198.56103515625,
      "logps/ref_chosen": -105.30314636230469,
      "logps/ref_rejected": -108.6555404663086,
      "logps/rejected": -281.88641357421875,
      "loss": 0.8025,
      "margin_dpo/margin_mean": 79.97298431396484,
      "margin_dpo/margin_std": 77.14982604980469,
      "step": 400
    },
    {
      "epoch": 0.5873715124816447,
      "eval_fcm_dpo/beta": 0.012737083248794079,
      "eval_logits/chosen": -1.7851052284240723,
      "eval_logits/rejected": -1.422616958618164,
      "eval_logps/chosen": -216.16845703125,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -261.1965026855469,
      "eval_loss": 0.49569863080978394,
      "eval_margin_dpo/margin_mean": 51.453895568847656,
      "eval_margin_dpo/margin_std": 65.75733184814453,
      "eval_runtime": 43.747,
      "eval_samples_per_second": 53.466,
      "eval_steps_per_second": 1.692,
      "step": 400
    },
    {
      "epoch": 0.5888399412628488,
      "fcm_dpo/beta": 0.013059141114354134,
      "fcm_dpo/delta": 0.20135712623596191,
      "fcm_dpo/margin": 50.72594451904297,
      "fcm_dpo/q_t": 0.3744737505912781,
      "grad_norm": 15.875528335571289,
      "learning_rate": 2.1800473436235136e-07,
      "logits/chosen": -1.5951911211013794,
      "logits/rejected": -1.3578665256500244,
      "logps/chosen": -190.99041748046875,
      "logps/ref_chosen": -84.28916931152344,
      "logps/ref_rejected": -90.943115234375,
      "logps/rejected": -248.3703155517578,
      "loss": 1.0408,
      "margin_dpo/margin_mean": 50.72594451904297,
      "margin_dpo/margin_std": 78.53496551513672,
      "step": 401
    },
    {
      "epoch": 0.5903083700440529,
      "fcm_dpo/beta": 0.012583300471305847,
      "fcm_dpo/delta": -0.35893315076828003,
      "fcm_dpo/margin": 92.74884033203125,
      "fcm_dpo/q_t": 0.2584311366081238,
      "grad_norm": 12.396269798278809,
      "learning_rate": 2.1673238449588665e-07,
      "logits/chosen": -2.003387451171875,
      "logits/rejected": -1.5525648593902588,
      "logps/chosen": -179.36959838867188,
      "logps/ref_chosen": -83.59312438964844,
      "logps/ref_rejected": -87.81027221679688,
      "logps/rejected": -276.3355712890625,
      "loss": 0.6644,
      "margin_dpo/margin_mean": 92.74884033203125,
      "margin_dpo/margin_std": 64.43993377685547,
      "step": 402
    },
    {
      "epoch": 0.591776798825257,
      "fcm_dpo/beta": 0.012310395017266273,
      "fcm_dpo/delta": 0.009639285504817963,
      "fcm_dpo/margin": 68.3082046508789,
      "fcm_dpo/q_t": 0.3275585174560547,
      "grad_norm": 11.257796287536621,
      "learning_rate": 2.154609112620295e-07,
      "logits/chosen": -2.2196717262268066,
      "logits/rejected": -1.8389365673065186,
      "logps/chosen": -173.17076110839844,
      "logps/ref_chosen": -73.75308227539062,
      "logps/ref_rejected": -83.92012786865234,
      "logps/rejected": -251.64601135253906,
      "loss": 0.8697,
      "margin_dpo/margin_mean": 68.3082046508789,
      "margin_dpo/margin_std": 70.12299346923828,
      "step": 403
    },
    {
      "epoch": 0.593245227606461,
      "fcm_dpo/beta": 0.012473889626562595,
      "fcm_dpo/delta": 0.08444693684577942,
      "fcm_dpo/margin": 61.89647674560547,
      "fcm_dpo/q_t": 0.34659212827682495,
      "grad_norm": 16.581104278564453,
      "learning_rate": 2.1419034816528218e-07,
      "logits/chosen": -2.2287228107452393,
      "logits/rejected": -1.9110980033874512,
      "logps/chosen": -196.9864044189453,
      "logps/ref_chosen": -79.67617797851562,
      "logps/ref_rejected": -84.280517578125,
      "logps/rejected": -263.4872131347656,
      "loss": 0.9672,
      "margin_dpo/margin_mean": 61.89647674560547,
      "margin_dpo/margin_std": 81.55596923828125,
      "step": 404
    },
    {
      "epoch": 0.5947136563876652,
      "fcm_dpo/beta": 0.012832706794142723,
      "fcm_dpo/delta": 0.08121472597122192,
      "fcm_dpo/margin": 60.25043487548828,
      "fcm_dpo/q_t": 0.3399437665939331,
      "grad_norm": 11.338379859924316,
      "learning_rate": 2.129207286861638e-07,
      "logits/chosen": -1.6208863258361816,
      "logits/rejected": -1.1806436777114868,
      "logps/chosen": -217.16705322265625,
      "logps/ref_chosen": -96.46195220947266,
      "logps/ref_rejected": -92.87071228027344,
      "logps/rejected": -273.82623291015625,
      "loss": 0.8836,
      "margin_dpo/margin_mean": 60.25043487548828,
      "margin_dpo/margin_std": 60.95580291748047,
      "step": 405
    },
    {
      "epoch": 0.5961820851688693,
      "fcm_dpo/beta": 0.01305477600544691,
      "fcm_dpo/delta": 0.13485847413539886,
      "fcm_dpo/margin": 55.51643371582031,
      "fcm_dpo/q_t": 0.3540048599243164,
      "grad_norm": 16.996318817138672,
      "learning_rate": 2.1165208628032861e-07,
      "logits/chosen": -1.942261815071106,
      "logits/rejected": -1.7632594108581543,
      "logps/chosen": -192.12899780273438,
      "logps/ref_chosen": -78.13396453857422,
      "logps/ref_rejected": -98.28359985351562,
      "logps/rejected": -267.7950744628906,
      "loss": 0.9845,
      "margin_dpo/margin_mean": 55.51643371582031,
      "margin_dpo/margin_std": 73.61194610595703,
      "step": 406
    },
    {
      "epoch": 0.5976505139500734,
      "fcm_dpo/beta": 0.013465587049722672,
      "fcm_dpo/delta": 0.17369112372398376,
      "fcm_dpo/margin": 51.12046432495117,
      "fcm_dpo/q_t": 0.36146247386932373,
      "grad_norm": 15.288115501403809,
      "learning_rate": 2.1038445437768375e-07,
      "logits/chosen": -2.192194700241089,
      "logits/rejected": -1.68497633934021,
      "logps/chosen": -205.4015350341797,
      "logps/ref_chosen": -84.01283264160156,
      "logps/ref_rejected": -82.78103637695312,
      "logps/rejected": -255.29019165039062,
      "loss": 0.9881,
      "margin_dpo/margin_mean": 51.120460510253906,
      "margin_dpo/margin_std": 69.12421417236328,
      "step": 407
    },
    {
      "epoch": 0.5991189427312775,
      "fcm_dpo/beta": 0.013818442821502686,
      "fcm_dpo/delta": 0.13276906311511993,
      "fcm_dpo/margin": 52.61656188964844,
      "fcm_dpo/q_t": 0.3611217141151428,
      "grad_norm": 15.027425765991211,
      "learning_rate": 2.0911786638150872e-07,
      "logits/chosen": -2.2886483669281006,
      "logits/rejected": -1.7039839029312134,
      "logps/chosen": -217.49435424804688,
      "logps/ref_chosen": -104.46174621582031,
      "logps/ref_rejected": -96.37217712402344,
      "logps/rejected": -262.0213317871094,
      "loss": 1.0044,
      "margin_dpo/margin_mean": 52.61656188964844,
      "margin_dpo/margin_std": 76.11251831054688,
      "step": 408
    },
    {
      "epoch": 0.6005873715124816,
      "fcm_dpo/beta": 0.014177883975207806,
      "fcm_dpo/delta": 0.11815360188484192,
      "fcm_dpo/margin": 52.24652862548828,
      "fcm_dpo/q_t": 0.34808868169784546,
      "grad_norm": 14.006386756896973,
      "learning_rate": 2.0785235566757517e-07,
      "logits/chosen": -1.5552918910980225,
      "logits/rejected": -1.1278228759765625,
      "logps/chosen": -222.04092407226562,
      "logps/ref_chosen": -97.66830444335938,
      "logps/ref_rejected": -90.04583740234375,
      "logps/rejected": -266.66497802734375,
      "loss": 1.0119,
      "margin_dpo/margin_mean": 52.24652862548828,
      "margin_dpo/margin_std": 73.68948364257812,
      "step": 409
    },
    {
      "epoch": 0.6020558002936858,
      "fcm_dpo/beta": 0.014412937685847282,
      "fcm_dpo/delta": 0.04549999535083771,
      "fcm_dpo/margin": 56.066070556640625,
      "fcm_dpo/q_t": 0.33421647548675537,
      "grad_norm": 14.076881408691406,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -1.8263776302337646,
      "logits/rejected": -1.6934200525283813,
      "logps/chosen": -198.36102294921875,
      "logps/ref_chosen": -76.46923828125,
      "logps/ref_rejected": -88.64064025878906,
      "logps/rejected": -266.5985107421875,
      "loss": 0.9166,
      "margin_dpo/margin_mean": 56.066070556640625,
      "margin_dpo/margin_std": 65.75325775146484,
      "step": 410
    },
    {
      "epoch": 0.6035242290748899,
      "fcm_dpo/beta": 0.014379952102899551,
      "fcm_dpo/delta": 0.015824299305677414,
      "fcm_dpo/margin": 58.06561279296875,
      "fcm_dpo/q_t": 0.3332013487815857,
      "grad_norm": 13.497848510742188,
      "learning_rate": 2.0532469944670343e-07,
      "logits/chosen": -2.267174243927002,
      "logits/rejected": -1.8730146884918213,
      "logps/chosen": -215.80242919921875,
      "logps/ref_chosen": -87.16630554199219,
      "logps/ref_rejected": -87.09603118896484,
      "logps/rejected": -273.79779052734375,
      "loss": 0.884,
      "margin_dpo/margin_mean": 58.06561279296875,
      "margin_dpo/margin_std": 62.760345458984375,
      "step": 411
    },
    {
      "epoch": 0.604992657856094,
      "fcm_dpo/beta": 0.014468617737293243,
      "fcm_dpo/delta": 0.05017668381333351,
      "fcm_dpo/margin": 55.50398635864258,
      "fcm_dpo/q_t": 0.34975796937942505,
      "grad_norm": 15.654382705688477,
      "learning_rate": 2.0406262054585738e-07,
      "logits/chosen": -2.643434524536133,
      "logits/rejected": -2.4237518310546875,
      "logps/chosen": -199.4673309326172,
      "logps/ref_chosen": -78.94734191894531,
      "logps/ref_rejected": -106.10554504394531,
      "logps/rejected": -282.1295166015625,
      "loss": 0.9861,
      "margin_dpo/margin_mean": 55.503990173339844,
      "margin_dpo/margin_std": 77.02133178710938,
      "step": 412
    },
    {
      "epoch": 0.6064610866372981,
      "fcm_dpo/beta": 0.014438673853874207,
      "fcm_dpo/delta": -0.0511392205953598,
      "fcm_dpo/margin": 62.048255920410156,
      "fcm_dpo/q_t": 0.3228393793106079,
      "grad_norm": 14.533951759338379,
      "learning_rate": 2.0280175213768205e-07,
      "logits/chosen": -1.3055384159088135,
      "logits/rejected": -1.0555856227874756,
      "logps/chosen": -225.90713500976562,
      "logps/ref_chosen": -95.69470977783203,
      "logps/ref_rejected": -107.96085357666016,
      "logps/rejected": -300.2215270996094,
      "loss": 0.9045,
      "margin_dpo/margin_mean": 62.048255920410156,
      "margin_dpo/margin_std": 75.0443115234375,
      "step": 413
    },
    {
      "epoch": 0.6079295154185022,
      "fcm_dpo/beta": 0.01450310554355383,
      "fcm_dpo/delta": 0.014196475967764854,
      "fcm_dpo/margin": 57.7109260559082,
      "fcm_dpo/q_t": 0.34177166223526,
      "grad_norm": 19.222719192504883,
      "learning_rate": 2.0154212744723247e-07,
      "logits/chosen": -1.5118329524993896,
      "logits/rejected": -1.1614055633544922,
      "logps/chosen": -208.80809020996094,
      "logps/ref_chosen": -88.27667236328125,
      "logps/ref_rejected": -92.87004089355469,
      "logps/rejected": -271.11236572265625,
      "loss": 0.9818,
      "margin_dpo/margin_mean": 57.71092224121094,
      "margin_dpo/margin_std": 80.63750457763672,
      "step": 414
    },
    {
      "epoch": 0.6093979441997063,
      "fcm_dpo/beta": 0.014715914614498615,
      "fcm_dpo/delta": 0.07649455219507217,
      "fcm_dpo/margin": 52.944557189941406,
      "fcm_dpo/q_t": 0.3491487503051758,
      "grad_norm": 15.72915267944336,
      "learning_rate": 2.002837796667909e-07,
      "logits/chosen": -1.5652879476547241,
      "logits/rejected": -1.1960127353668213,
      "logps/chosen": -235.4866180419922,
      "logps/ref_chosen": -108.91590118408203,
      "logps/ref_rejected": -107.47135925292969,
      "logps/rejected": -286.98663330078125,
      "loss": 0.9785,
      "margin_dpo/margin_mean": 52.944557189941406,
      "margin_dpo/margin_std": 71.51063537597656,
      "step": 415
    },
    {
      "epoch": 0.6108663729809104,
      "fcm_dpo/beta": 0.014534441754221916,
      "fcm_dpo/delta": -0.19304054975509644,
      "fcm_dpo/margin": 70.30528259277344,
      "fcm_dpo/q_t": 0.3036141097545624,
      "grad_norm": 13.811197280883789,
      "learning_rate": 1.990267419549914e-07,
      "logits/chosen": -1.9736833572387695,
      "logits/rejected": -1.4895861148834229,
      "logps/chosen": -219.60324096679688,
      "logps/ref_chosen": -93.39888000488281,
      "logps/ref_rejected": -97.6729736328125,
      "logps/rejected": -294.1826171875,
      "loss": 0.8409,
      "margin_dpo/margin_mean": 70.30529022216797,
      "margin_dpo/margin_std": 73.31067657470703,
      "step": 416
    },
    {
      "epoch": 0.6123348017621145,
      "fcm_dpo/beta": 0.01407422311604023,
      "fcm_dpo/delta": -0.07601834833621979,
      "fcm_dpo/margin": 65.29705810546875,
      "fcm_dpo/q_t": 0.3138529360294342,
      "grad_norm": 13.441536903381348,
      "learning_rate": 1.9777104743594686e-07,
      "logits/chosen": -2.2062695026397705,
      "logits/rejected": -1.3112609386444092,
      "logps/chosen": -203.29037475585938,
      "logps/ref_chosen": -83.53533172607422,
      "logps/ref_rejected": -74.44184875488281,
      "logps/rejected": -259.49395751953125,
      "loss": 0.8486,
      "margin_dpo/margin_mean": 65.29705810546875,
      "margin_dpo/margin_std": 66.7147216796875,
      "step": 417
    },
    {
      "epoch": 0.6138032305433186,
      "fcm_dpo/beta": 0.013649359345436096,
      "fcm_dpo/delta": -0.2670527398586273,
      "fcm_dpo/margin": 79.63304138183594,
      "fcm_dpo/q_t": 0.2834809422492981,
      "grad_norm": 15.17289924621582,
      "learning_rate": 1.965167291983757e-07,
      "logits/chosen": -2.136475086212158,
      "logits/rejected": -1.491228461265564,
      "logps/chosen": -221.35855102539062,
      "logps/ref_chosen": -108.22152709960938,
      "logps/ref_rejected": -111.8646469116211,
      "logps/rejected": -304.63470458984375,
      "loss": 0.7466,
      "margin_dpo/margin_mean": 79.63304138183594,
      "margin_dpo/margin_std": 66.44700622558594,
      "step": 418
    },
    {
      "epoch": 0.6152716593245228,
      "fcm_dpo/beta": 0.013165738433599472,
      "fcm_dpo/delta": -0.06787989288568497,
      "fcm_dpo/margin": 69.23770141601562,
      "fcm_dpo/q_t": 0.3166205883026123,
      "grad_norm": 12.66903305053711,
      "learning_rate": 1.9526382029472988e-07,
      "logits/chosen": -1.1017417907714844,
      "logits/rejected": -0.6296178698539734,
      "logps/chosen": -218.44863891601562,
      "logps/ref_chosen": -97.18328094482422,
      "logps/ref_rejected": -98.18531799316406,
      "logps/rejected": -288.6883544921875,
      "loss": 0.8503,
      "margin_dpo/margin_mean": 69.2376937866211,
      "margin_dpo/margin_std": 71.49453735351562,
      "step": 419
    },
    {
      "epoch": 0.6167400881057269,
      "fcm_dpo/beta": 0.013205429539084435,
      "fcm_dpo/delta": 0.017094507813453674,
      "fcm_dpo/margin": 63.06053924560547,
      "fcm_dpo/q_t": 0.3347446918487549,
      "grad_norm": 15.853597640991211,
      "learning_rate": 1.9401235374032425e-07,
      "logits/chosen": -2.1705939769744873,
      "logits/rejected": -1.1035213470458984,
      "logps/chosen": -238.99166870117188,
      "logps/ref_chosen": -114.30848693847656,
      "logps/ref_rejected": -75.68355560302734,
      "logps/rejected": -263.42730712890625,
      "loss": 0.9019,
      "margin_dpo/margin_mean": 63.060543060302734,
      "margin_dpo/margin_std": 71.72982788085938,
      "step": 420
    },
    {
      "epoch": 0.618208516886931,
      "fcm_dpo/beta": 0.012998622842133045,
      "fcm_dpo/delta": -0.0270537156611681,
      "fcm_dpo/margin": 67.26036834716797,
      "fcm_dpo/q_t": 0.32487133145332336,
      "grad_norm": 12.69776439666748,
      "learning_rate": 1.9276236251246653e-07,
      "logits/chosen": -1.688891887664795,
      "logits/rejected": -1.3223713636398315,
      "logps/chosen": -209.80950927734375,
      "logps/ref_chosen": -85.87985229492188,
      "logps/ref_rejected": -96.33648681640625,
      "logps/rejected": -287.5264892578125,
      "loss": 0.8675,
      "margin_dpo/margin_mean": 67.26036834716797,
      "margin_dpo/margin_std": 70.79811096191406,
      "step": 421
    },
    {
      "epoch": 0.6196769456681351,
      "fcm_dpo/beta": 0.013068338856101036,
      "fcm_dpo/delta": -0.010718483477830887,
      "fcm_dpo/margin": 65.76750183105469,
      "fcm_dpo/q_t": 0.3269606828689575,
      "grad_norm": 12.487133979797363,
      "learning_rate": 1.9151387954958792e-07,
      "logits/chosen": -2.3033037185668945,
      "logits/rejected": -1.9288625717163086,
      "logps/chosen": -220.82708740234375,
      "logps/ref_chosen": -100.48060607910156,
      "logps/ref_rejected": -94.40821838378906,
      "logps/rejected": -280.5221862792969,
      "loss": 0.8611,
      "margin_dpo/margin_mean": 65.76750183105469,
      "margin_dpo/margin_std": 69.10296630859375,
      "step": 422
    },
    {
      "epoch": 0.6211453744493393,
      "fcm_dpo/beta": 0.013151784427464008,
      "fcm_dpo/delta": 0.04228432476520538,
      "fcm_dpo/margin": 61.62306213378906,
      "fcm_dpo/q_t": 0.3433518409729004,
      "grad_norm": 13.384033203125,
      "learning_rate": 1.902669377503756e-07,
      "logits/chosen": -1.8323371410369873,
      "logits/rejected": -1.7185084819793701,
      "logps/chosen": -195.48995971679688,
      "logps/ref_chosen": -78.44993591308594,
      "logps/ref_rejected": -92.04652404785156,
      "logps/rejected": -270.7095947265625,
      "loss": 0.943,
      "margin_dpo/margin_mean": 61.6230583190918,
      "margin_dpo/margin_std": 77.90271759033203,
      "step": 423
    },
    {
      "epoch": 0.6226138032305433,
      "fcm_dpo/beta": 0.013109794817864895,
      "fcm_dpo/delta": -0.008033724501729012,
      "fcm_dpo/margin": 65.39669799804688,
      "fcm_dpo/q_t": 0.3260476589202881,
      "grad_norm": 12.125853538513184,
      "learning_rate": 1.890215699729057e-07,
      "logits/chosen": -1.810041904449463,
      "logits/rejected": -1.2230205535888672,
      "logps/chosen": -215.9375457763672,
      "logps/ref_chosen": -87.64236450195312,
      "logps/ref_rejected": -72.36566162109375,
      "logps/rejected": -266.05755615234375,
      "loss": 0.8727,
      "margin_dpo/margin_mean": 65.39669799804688,
      "margin_dpo/margin_std": 69.99500274658203,
      "step": 424
    },
    {
      "epoch": 0.6240822320117474,
      "fcm_dpo/beta": 0.013596277683973312,
      "fcm_dpo/delta": 0.27160927653312683,
      "fcm_dpo/margin": 43.7700309753418,
      "fcm_dpo/q_t": 0.3926636874675751,
      "grad_norm": 15.634125709533691,
      "learning_rate": 1.8777780903377732e-07,
      "logits/chosen": -1.4916131496429443,
      "logits/rejected": -1.4072489738464355,
      "logps/chosen": -212.03672790527344,
      "logps/ref_chosen": -78.51979064941406,
      "logps/ref_rejected": -102.74864196777344,
      "logps/rejected": -280.0356140136719,
      "loss": 1.136,
      "margin_dpo/margin_mean": 43.77003479003906,
      "margin_dpo/margin_std": 82.93934631347656,
      "step": 425
    },
    {
      "epoch": 0.6255506607929515,
      "fcm_dpo/beta": 0.013757694512605667,
      "fcm_dpo/delta": -0.045442745089530945,
      "fcm_dpo/margin": 64.79296875,
      "fcm_dpo/q_t": 0.3165437877178192,
      "grad_norm": 15.151692390441895,
      "learning_rate": 1.8653568770724803e-07,
      "logits/chosen": -1.5372415781021118,
      "logits/rejected": -0.826823353767395,
      "logps/chosen": -223.7195587158203,
      "logps/ref_chosen": -108.50582885742188,
      "logps/ref_rejected": -88.300048828125,
      "logps/rejected": -268.3067321777344,
      "loss": 0.8451,
      "margin_dpo/margin_mean": 64.79296875,
      "margin_dpo/margin_std": 64.40265655517578,
      "step": 426
    },
    {
      "epoch": 0.6270190895741556,
      "fcm_dpo/beta": 0.01376783475279808,
      "fcm_dpo/delta": 0.018252946436405182,
      "fcm_dpo/margin": 60.51248550415039,
      "fcm_dpo/q_t": 0.32682928442955017,
      "grad_norm": 13.044880867004395,
      "learning_rate": 1.8529523872436977e-07,
      "logits/chosen": -2.440342903137207,
      "logits/rejected": -1.8359763622283936,
      "logps/chosen": -198.5206298828125,
      "logps/ref_chosen": -99.12046813964844,
      "logps/ref_rejected": -85.724609375,
      "logps/rejected": -245.63723754882812,
      "loss": 0.8555,
      "margin_dpo/margin_mean": 60.51248550415039,
      "margin_dpo/margin_std": 59.35211181640625,
      "step": 427
    },
    {
      "epoch": 0.6284875183553598,
      "fcm_dpo/beta": 0.01399192400276661,
      "fcm_dpo/delta": 0.18102213740348816,
      "fcm_dpo/margin": 48.68280029296875,
      "fcm_dpo/q_t": 0.36542826890945435,
      "grad_norm": 14.462920188903809,
      "learning_rate": 1.8405649477212697e-07,
      "logits/chosen": -1.4676676988601685,
      "logits/rejected": -1.1950892210006714,
      "logps/chosen": -239.3568115234375,
      "logps/ref_chosen": -105.96925354003906,
      "logps/ref_rejected": -109.10218048095703,
      "logps/rejected": -291.17254638671875,
      "loss": 1.0476,
      "margin_dpo/margin_mean": 48.68279266357422,
      "margin_dpo/margin_std": 73.93551635742188,
      "step": 428
    },
    {
      "epoch": 0.6299559471365639,
      "fcm_dpo/beta": 0.014181070029735565,
      "fcm_dpo/delta": -0.12901651859283447,
      "fcm_dpo/margin": 68.02422332763672,
      "fcm_dpo/q_t": 0.3178831934928894,
      "grad_norm": 14.960230827331543,
      "learning_rate": 1.828194884925749e-07,
      "logits/chosen": -1.7683967351913452,
      "logits/rejected": -1.0629897117614746,
      "logps/chosen": -233.390869140625,
      "logps/ref_chosen": -113.54487609863281,
      "logps/ref_rejected": -98.24201965332031,
      "logps/rejected": -286.11224365234375,
      "loss": 0.8717,
      "margin_dpo/margin_mean": 68.02421569824219,
      "margin_dpo/margin_std": 75.7259292602539,
      "step": 429
    },
    {
      "epoch": 0.631424375917768,
      "fcm_dpo/beta": 0.014241167344152927,
      "fcm_dpo/delta": 0.14151397347450256,
      "fcm_dpo/margin": 50.407920837402344,
      "fcm_dpo/q_t": 0.35696834325790405,
      "grad_norm": 16.362232208251953,
      "learning_rate": 1.8158425248197928e-07,
      "logits/chosen": -1.3190555572509766,
      "logits/rejected": -1.270359992980957,
      "logps/chosen": -212.43923950195312,
      "logps/ref_chosen": -91.31936645507812,
      "logps/ref_rejected": -110.1096420288086,
      "logps/rejected": -281.637451171875,
      "loss": 0.9831,
      "margin_dpo/margin_mean": 50.407920837402344,
      "margin_dpo/margin_std": 67.11074829101562,
      "step": 430
    },
    {
      "epoch": 0.6328928046989721,
      "fcm_dpo/beta": 0.014219909906387329,
      "fcm_dpo/delta": -0.0074679069221019745,
      "fcm_dpo/margin": 60.23283386230469,
      "fcm_dpo/q_t": 0.33199840784072876,
      "grad_norm": 14.520343780517578,
      "learning_rate": 1.8035081928995788e-07,
      "logits/chosen": -1.8971822261810303,
      "logits/rejected": -1.5131553411483765,
      "logps/chosen": -212.03729248046875,
      "logps/ref_chosen": -93.18122100830078,
      "logps/ref_rejected": -98.13226318359375,
      "logps/rejected": -277.22119140625,
      "loss": 0.9109,
      "margin_dpo/margin_mean": 60.23283386230469,
      "margin_dpo/margin_std": 71.3617935180664,
      "step": 431
    },
    {
      "epoch": 0.6343612334801763,
      "fcm_dpo/beta": 0.01398632675409317,
      "fcm_dpo/delta": -0.17663273215293884,
      "fcm_dpo/margin": 72.14842987060547,
      "fcm_dpo/q_t": 0.30276134610176086,
      "grad_norm": 13.15101432800293,
      "learning_rate": 1.791192214186223e-07,
      "logits/chosen": -1.2233517169952393,
      "logits/rejected": -0.6508141756057739,
      "logps/chosen": -211.9017333984375,
      "logps/ref_chosen": -104.43478393554688,
      "logps/ref_rejected": -105.08955383300781,
      "logps/rejected": -284.7049560546875,
      "loss": 0.8083,
      "margin_dpo/margin_mean": 72.14842987060547,
      "margin_dpo/margin_std": 70.37438201904297,
      "step": 432
    },
    {
      "epoch": 0.6358296622613803,
      "fcm_dpo/beta": 0.014024633914232254,
      "fcm_dpo/delta": 0.04999150335788727,
      "fcm_dpo/margin": 57.14216613769531,
      "fcm_dpo/q_t": 0.34285709261894226,
      "grad_norm": 14.23776912689209,
      "learning_rate": 1.7788949132172193e-07,
      "logits/chosen": -1.6155874729156494,
      "logits/rejected": -1.2833582162857056,
      "logps/chosen": -216.49232482910156,
      "logps/ref_chosen": -89.84322357177734,
      "logps/ref_rejected": -101.73345947265625,
      "logps/rejected": -285.52471923828125,
      "loss": 0.9633,
      "margin_dpo/margin_mean": 57.142173767089844,
      "margin_dpo/margin_std": 73.58992004394531,
      "step": 433
    },
    {
      "epoch": 0.6372980910425844,
      "fcm_dpo/beta": 0.013989130035042763,
      "fcm_dpo/delta": 0.024751663208007812,
      "fcm_dpo/margin": 59.13129806518555,
      "fcm_dpo/q_t": 0.342306524515152,
      "grad_norm": 13.736560821533203,
      "learning_rate": 1.7666166140378853e-07,
      "logits/chosen": -2.4307074546813965,
      "logits/rejected": -1.8070075511932373,
      "logps/chosen": -215.8836212158203,
      "logps/ref_chosen": -97.6925277709961,
      "logps/ref_rejected": -84.09129333496094,
      "logps/rejected": -261.4136962890625,
      "loss": 0.9414,
      "margin_dpo/margin_mean": 59.13129806518555,
      "margin_dpo/margin_std": 77.16365814208984,
      "step": 434
    },
    {
      "epoch": 0.6387665198237885,
      "fcm_dpo/beta": 0.01387035846710205,
      "fcm_dpo/delta": -0.06785215437412262,
      "fcm_dpo/margin": 65.7310791015625,
      "fcm_dpo/q_t": 0.3216383159160614,
      "grad_norm": 14.38983154296875,
      "learning_rate": 1.7543576401928218e-07,
      "logits/chosen": -2.151081085205078,
      "logits/rejected": -1.6833906173706055,
      "logps/chosen": -198.29730224609375,
      "logps/ref_chosen": -86.17192077636719,
      "logps/ref_rejected": -93.751708984375,
      "logps/rejected": -271.608154296875,
      "loss": 0.8643,
      "margin_dpo/margin_mean": 65.7310791015625,
      "margin_dpo/margin_std": 72.66617584228516,
      "step": 435
    },
    {
      "epoch": 0.6402349486049926,
      "fcm_dpo/beta": 0.01344333216547966,
      "fcm_dpo/delta": -0.18730014562606812,
      "fcm_dpo/margin": 75.75979614257812,
      "fcm_dpo/q_t": 0.289669394493103,
      "grad_norm": 12.1524076461792,
      "learning_rate": 1.742118314717391e-07,
      "logits/chosen": -1.5553090572357178,
      "logits/rejected": -0.6732375025749207,
      "logps/chosen": -197.84014892578125,
      "logps/ref_chosen": -105.78710174560547,
      "logps/ref_rejected": -88.62471008300781,
      "logps/rejected": -256.43756103515625,
      "loss": 0.7541,
      "margin_dpo/margin_mean": 75.75979614257812,
      "margin_dpo/margin_std": 62.77748107910156,
      "step": 436
    },
    {
      "epoch": 0.6417033773861968,
      "fcm_dpo/beta": 0.013649407774209976,
      "fcm_dpo/delta": 0.1684824675321579,
      "fcm_dpo/margin": 50.75785446166992,
      "fcm_dpo/q_t": 0.3533422648906708,
      "grad_norm": 14.654732704162598,
      "learning_rate": 1.7298989601292036e-07,
      "logits/chosen": -2.0003044605255127,
      "logits/rejected": -1.368834376335144,
      "logps/chosen": -206.8590087890625,
      "logps/ref_chosen": -96.06204223632812,
      "logps/ref_rejected": -89.01220703125,
      "logps/rejected": -250.5670166015625,
      "loss": 0.954,
      "margin_dpo/margin_mean": 50.757850646972656,
      "margin_dpo/margin_std": 59.297645568847656,
      "step": 437
    },
    {
      "epoch": 0.6431718061674009,
      "fcm_dpo/beta": 0.013722064904868603,
      "fcm_dpo/delta": 0.016998998820781708,
      "fcm_dpo/margin": 60.7908935546875,
      "fcm_dpo/q_t": 0.32923266291618347,
      "grad_norm": 13.463798522949219,
      "learning_rate": 1.7176998984196144e-07,
      "logits/chosen": -2.1621651649475098,
      "logits/rejected": -1.3619840145111084,
      "logps/chosen": -206.91421508789062,
      "logps/ref_chosen": -101.85537719726562,
      "logps/ref_rejected": -89.44766235351562,
      "logps/rejected": -255.2974090576172,
      "loss": 0.8886,
      "margin_dpo/margin_mean": 60.790897369384766,
      "margin_dpo/margin_std": 65.22203063964844,
      "step": 438
    },
    {
      "epoch": 0.644640234948605,
      "fcm_dpo/beta": 0.013775939121842384,
      "fcm_dpo/delta": 0.07153499871492386,
      "fcm_dpo/margin": 56.78171920776367,
      "fcm_dpo/q_t": 0.3450607657432556,
      "grad_norm": 14.867757797241211,
      "learning_rate": 1.7055214510452458e-07,
      "logits/chosen": -1.614975929260254,
      "logits/rejected": -1.3566641807556152,
      "logps/chosen": -200.1914520263672,
      "logps/ref_chosen": -81.75563049316406,
      "logps/ref_rejected": -90.58636474609375,
      "logps/rejected": -265.80389404296875,
      "loss": 0.9414,
      "margin_dpo/margin_mean": 56.78171920776367,
      "margin_dpo/margin_std": 68.44754028320312,
      "step": 439
    },
    {
      "epoch": 0.6461086637298091,
      "fcm_dpo/beta": 0.013727106153964996,
      "fcm_dpo/delta": -0.06820373237133026,
      "fcm_dpo/margin": 66.29185485839844,
      "fcm_dpo/q_t": 0.3204452395439148,
      "grad_norm": 12.376969337463379,
      "learning_rate": 1.6933639389195134e-07,
      "logits/chosen": -2.5066826343536377,
      "logits/rejected": -2.0377886295318604,
      "logps/chosen": -211.48703002929688,
      "logps/ref_chosen": -105.64108276367188,
      "logps/ref_rejected": -103.4010009765625,
      "logps/rejected": -275.538818359375,
      "loss": 0.8662,
      "margin_dpo/margin_mean": 66.29185485839844,
      "margin_dpo/margin_std": 71.9673080444336,
      "step": 440
    },
    {
      "epoch": 0.6475770925110133,
      "fcm_dpo/beta": 0.013530462980270386,
      "fcm_dpo/delta": -0.10768324136734009,
      "fcm_dpo/margin": 69.98019409179688,
      "fcm_dpo/q_t": 0.30917853116989136,
      "grad_norm": 11.863908767700195,
      "learning_rate": 1.681227682404166e-07,
      "logits/chosen": -1.7193225622177124,
      "logits/rejected": -1.3498790264129639,
      "logps/chosen": -214.11981201171875,
      "logps/ref_chosen": -91.529541015625,
      "logps/ref_rejected": -103.619384765625,
      "logps/rejected": -296.18988037109375,
      "loss": 0.787,
      "margin_dpo/margin_mean": 69.98018646240234,
      "margin_dpo/margin_std": 65.251708984375,
      "step": 441
    },
    {
      "epoch": 0.6490455212922174,
      "fcm_dpo/beta": 0.013135121203958988,
      "fcm_dpo/delta": -0.1771937906742096,
      "fcm_dpo/margin": 76.81739044189453,
      "fcm_dpo/q_t": 0.29648613929748535,
      "grad_norm": 12.602315902709961,
      "learning_rate": 1.669113001300851e-07,
      "logits/chosen": -1.4338107109069824,
      "logits/rejected": -0.9611091613769531,
      "logps/chosen": -208.39720153808594,
      "logps/ref_chosen": -84.77755737304688,
      "logps/ref_rejected": -83.82415771484375,
      "logps/rejected": -284.26116943359375,
      "loss": 0.7703,
      "margin_dpo/margin_mean": 76.8173828125,
      "margin_dpo/margin_std": 68.45848083496094,
      "step": 442
    },
    {
      "epoch": 0.6505139500734214,
      "fcm_dpo/beta": 0.013111368753015995,
      "fcm_dpo/delta": 0.11141438782215118,
      "fcm_dpo/margin": 56.881134033203125,
      "fcm_dpo/q_t": 0.34306538105010986,
      "grad_norm": 13.224976539611816,
      "learning_rate": 1.6570202148426815e-07,
      "logits/chosen": -1.735137701034546,
      "logits/rejected": -1.1993635892868042,
      "logps/chosen": -220.87890625,
      "logps/ref_chosen": -102.64927673339844,
      "logps/ref_rejected": -93.03807067871094,
      "logps/rejected": -268.14886474609375,
      "loss": 0.9271,
      "margin_dpo/margin_mean": 56.881134033203125,
      "margin_dpo/margin_std": 61.88277816772461,
      "step": 443
    },
    {
      "epoch": 0.6519823788546255,
      "fcm_dpo/beta": 0.0135607048869133,
      "fcm_dpo/delta": 0.10103952884674072,
      "fcm_dpo/margin": 55.769325256347656,
      "fcm_dpo/q_t": 0.35067448019981384,
      "grad_norm": 14.165532112121582,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": -1.592213749885559,
      "logits/rejected": -1.420748233795166,
      "logps/chosen": -214.36148071289062,
      "logps/ref_chosen": -87.91971588134766,
      "logps/ref_rejected": -103.32345581054688,
      "logps/rejected": -285.5345458984375,
      "loss": 0.9687,
      "margin_dpo/margin_mean": 55.769325256347656,
      "margin_dpo/margin_std": 72.71159362792969,
      "step": 444
    },
    {
      "epoch": 0.6534508076358296,
      "fcm_dpo/beta": 0.01349145732820034,
      "fcm_dpo/delta": -0.027875246480107307,
      "fcm_dpo/margin": 64.85955810546875,
      "fcm_dpo/q_t": 0.32111674547195435,
      "grad_norm": 12.245442390441895,
      "learning_rate": 1.6329015999011182e-07,
      "logits/chosen": -1.9300236701965332,
      "logits/rejected": -1.5292446613311768,
      "logps/chosen": -211.0179443359375,
      "logps/ref_chosen": -101.40086364746094,
      "logps/ref_rejected": -99.03790283203125,
      "logps/rejected": -273.5145263671875,
      "loss": 0.8467,
      "margin_dpo/margin_mean": 64.85956573486328,
      "margin_dpo/margin_std": 64.7419662475586,
      "step": 445
    },
    {
      "epoch": 0.6549192364170338,
      "fcm_dpo/beta": 0.013596500270068645,
      "fcm_dpo/delta": 0.05426015704870224,
      "fcm_dpo/margin": 58.844242095947266,
      "fcm_dpo/q_t": 0.33681419491767883,
      "grad_norm": 14.447946548461914,
      "learning_rate": 1.6208764069656578e-07,
      "logits/chosen": -1.9224543571472168,
      "logits/rejected": -1.767820119857788,
      "logps/chosen": -194.24234008789062,
      "logps/ref_chosen": -87.4223403930664,
      "logps/ref_rejected": -106.70075988769531,
      "logps/rejected": -272.364990234375,
      "loss": 0.9015,
      "margin_dpo/margin_mean": 58.84424591064453,
      "margin_dpo/margin_std": 65.06867980957031,
      "step": 446
    },
    {
      "epoch": 0.6563876651982379,
      "fcm_dpo/beta": 0.01339960377663374,
      "fcm_dpo/delta": -0.11742748320102692,
      "fcm_dpo/margin": 71.3275375366211,
      "fcm_dpo/q_t": 0.31018170714378357,
      "grad_norm": 13.609336853027344,
      "learning_rate": 1.608874379754465e-07,
      "logits/chosen": -2.158440113067627,
      "logits/rejected": -2.107996940612793,
      "logps/chosen": -191.38192749023438,
      "logps/ref_chosen": -83.6152572631836,
      "logps/ref_rejected": -104.91239929199219,
      "logps/rejected": -284.006591796875,
      "loss": 0.8392,
      "margin_dpo/margin_mean": 71.3275375366211,
      "margin_dpo/margin_std": 73.93170166015625,
      "step": 447
    },
    {
      "epoch": 0.657856093979442,
      "fcm_dpo/beta": 0.013280214741826057,
      "fcm_dpo/delta": -0.09408734738826752,
      "fcm_dpo/margin": 70.38318634033203,
      "fcm_dpo/q_t": 0.32261860370635986,
      "grad_norm": 14.318117141723633,
      "learning_rate": 1.5968958345321177e-07,
      "logits/chosen": -1.5758308172225952,
      "logits/rejected": -1.3030452728271484,
      "logps/chosen": -207.17013549804688,
      "logps/ref_chosen": -92.5757827758789,
      "logps/ref_rejected": -107.68977355957031,
      "logps/rejected": -292.6673278808594,
      "loss": 0.9036,
      "margin_dpo/margin_mean": 70.38318634033203,
      "margin_dpo/margin_std": 83.00077819824219,
      "step": 448
    },
    {
      "epoch": 0.6593245227606461,
      "fcm_dpo/beta": 0.012970153242349625,
      "fcm_dpo/delta": -0.09647231549024582,
      "fcm_dpo/margin": 72.28010559082031,
      "fcm_dpo/q_t": 0.3125626742839813,
      "grad_norm": 12.015176773071289,
      "learning_rate": 1.584941086944423e-07,
      "logits/chosen": -1.4746041297912598,
      "logits/rejected": -1.0464733839035034,
      "logps/chosen": -215.36099243164062,
      "logps/ref_chosen": -102.39893341064453,
      "logps/ref_rejected": -95.14886474609375,
      "logps/rejected": -280.3910217285156,
      "loss": 0.8473,
      "margin_dpo/margin_mean": 72.28010559082031,
      "margin_dpo/margin_std": 75.04696655273438,
      "step": 449
    },
    {
      "epoch": 0.6607929515418502,
      "fcm_dpo/beta": 0.012733325362205505,
      "fcm_dpo/delta": -0.0715692788362503,
      "fcm_dpo/margin": 71.86397552490234,
      "fcm_dpo/q_t": 0.3133106827735901,
      "grad_norm": 12.985596656799316,
      "learning_rate": 1.573010452010098e-07,
      "logits/chosen": -2.383078098297119,
      "logits/rejected": -2.1813578605651855,
      "logps/chosen": -186.05169677734375,
      "logps/ref_chosen": -86.99285888671875,
      "logps/ref_rejected": -108.53203582763672,
      "logps/rejected": -279.4548645019531,
      "loss": 0.8488,
      "margin_dpo/margin_mean": 71.86397552490234,
      "margin_dpo/margin_std": 73.41386413574219,
      "step": 450
    },
    {
      "epoch": 0.6622613803230544,
      "fcm_dpo/beta": 0.012889644131064415,
      "fcm_dpo/delta": 0.07636302709579468,
      "fcm_dpo/margin": 60.40147399902344,
      "fcm_dpo/q_t": 0.3498266339302063,
      "grad_norm": 18.933948516845703,
      "learning_rate": 1.5611042441124687e-07,
      "logits/chosen": -2.3939194679260254,
      "logits/rejected": -1.90476655960083,
      "logps/chosen": -199.50680541992188,
      "logps/ref_chosen": -86.811279296875,
      "logps/ref_rejected": -79.8555908203125,
      "logps/rejected": -252.95257568359375,
      "loss": 0.9826,
      "margin_dpo/margin_mean": 60.40147399902344,
      "margin_dpo/margin_std": 81.97018432617188,
      "step": 451
    },
    {
      "epoch": 0.6637298091042585,
      "fcm_dpo/beta": 0.012789755128324032,
      "fcm_dpo/delta": 0.04102400690317154,
      "fcm_dpo/margin": 63.32386779785156,
      "fcm_dpo/q_t": 0.3287479281425476,
      "grad_norm": 12.005148887634277,
      "learning_rate": 1.549222776991186e-07,
      "logits/chosen": -1.8443999290466309,
      "logits/rejected": -1.7731636762619019,
      "logps/chosen": -179.27359008789062,
      "logps/ref_chosen": -79.379638671875,
      "logps/ref_rejected": -103.71539306640625,
      "logps/rejected": -266.9331970214844,
      "loss": 0.8507,
      "margin_dpo/margin_mean": 63.32386779785156,
      "margin_dpo/margin_std": 55.35037612915039,
      "step": 452
    },
    {
      "epoch": 0.6651982378854625,
      "fcm_dpo/beta": 0.012718062847852707,
      "fcm_dpo/delta": -0.1492374986410141,
      "fcm_dpo/margin": 77.42842102050781,
      "fcm_dpo/q_t": 0.30019137263298035,
      "grad_norm": 12.277626037597656,
      "learning_rate": 1.5373663637339584e-07,
      "logits/chosen": -1.9240939617156982,
      "logits/rejected": -1.4621145725250244,
      "logps/chosen": -204.21820068359375,
      "logps/ref_chosen": -87.6951904296875,
      "logps/ref_rejected": -90.0582275390625,
      "logps/rejected": -284.0096435546875,
      "loss": 0.7741,
      "margin_dpo/margin_mean": 77.42842102050781,
      "margin_dpo/margin_std": 67.35591125488281,
      "step": 453
    },
    {
      "epoch": 0.6666666666666666,
      "fcm_dpo/beta": 0.012760400772094727,
      "fcm_dpo/delta": 0.06223556026816368,
      "fcm_dpo/margin": 62.07777786254883,
      "fcm_dpo/q_t": 0.3461794853210449,
      "grad_norm": 14.331741333007812,
      "learning_rate": 1.5255353167683017e-07,
      "logits/chosen": -1.8063360452651978,
      "logits/rejected": -1.3538455963134766,
      "logps/chosen": -215.82162475585938,
      "logps/ref_chosen": -89.56623840332031,
      "logps/ref_rejected": -92.92105102539062,
      "logps/rejected": -281.25421142578125,
      "loss": 0.9868,
      "margin_dpo/margin_mean": 62.07777786254883,
      "margin_dpo/margin_std": 85.04985809326172,
      "step": 454
    },
    {
      "epoch": 0.6681350954478708,
      "fcm_dpo/beta": 0.012872504070401192,
      "fcm_dpo/delta": 0.028366755694150925,
      "fcm_dpo/margin": 63.95558547973633,
      "fcm_dpo/q_t": 0.3338191509246826,
      "grad_norm": 13.07781982421875,
      "learning_rate": 1.5137299478533064e-07,
      "logits/chosen": -1.8931918144226074,
      "logits/rejected": -1.8260715007781982,
      "logps/chosen": -190.75283813476562,
      "logps/ref_chosen": -77.62998962402344,
      "logps/ref_rejected": -118.97795104980469,
      "logps/rejected": -296.0563659667969,
      "loss": 0.9057,
      "margin_dpo/margin_mean": 63.95558166503906,
      "margin_dpo/margin_std": 71.64030456542969,
      "step": 455
    },
    {
      "epoch": 0.6696035242290749,
      "fcm_dpo/beta": 0.013063902035355568,
      "fcm_dpo/delta": 0.09057056158781052,
      "fcm_dpo/margin": 58.58759307861328,
      "fcm_dpo/q_t": 0.3430330455303192,
      "grad_norm": 15.235799789428711,
      "learning_rate": 1.5019505680714232e-07,
      "logits/chosen": -2.0362932682037354,
      "logits/rejected": -1.9036009311676025,
      "logps/chosen": -206.73255920410156,
      "logps/ref_chosen": -89.61686706542969,
      "logps/ref_rejected": -109.5597152709961,
      "logps/rejected": -285.26300048828125,
      "loss": 0.9192,
      "margin_dpo/margin_mean": 58.58759307861328,
      "margin_dpo/margin_std": 65.98350524902344,
      "step": 456
    },
    {
      "epoch": 0.671071953010279,
      "fcm_dpo/beta": 0.012593207880854607,
      "fcm_dpo/delta": -0.1489265263080597,
      "fcm_dpo/margin": 77.71039581298828,
      "fcm_dpo/q_t": 0.2934274673461914,
      "grad_norm": 12.465642929077148,
      "learning_rate": 1.4901974878202627e-07,
      "logits/chosen": -1.9858686923980713,
      "logits/rejected": -1.677903413772583,
      "logps/chosen": -187.23529052734375,
      "logps/ref_chosen": -87.32168579101562,
      "logps/ref_rejected": -90.76660919189453,
      "logps/rejected": -268.390625,
      "loss": 0.753,
      "margin_dpo/margin_mean": 77.71039581298828,
      "margin_dpo/margin_std": 58.36017608642578,
      "step": 457
    },
    {
      "epoch": 0.6725403817914831,
      "fcm_dpo/beta": 0.012880630791187286,
      "fcm_dpo/delta": 0.09830697625875473,
      "fcm_dpo/margin": 58.92164611816406,
      "fcm_dpo/q_t": 0.3527994155883789,
      "grad_norm": 14.763307571411133,
      "learning_rate": 1.4784710168044212e-07,
      "logits/chosen": -1.6106715202331543,
      "logits/rejected": -1.3593188524246216,
      "logps/chosen": -205.40960693359375,
      "logps/ref_chosen": -93.52044677734375,
      "logps/ref_rejected": -103.36898803710938,
      "logps/rejected": -274.1798095703125,
      "loss": 0.9732,
      "margin_dpo/margin_mean": 58.92164611816406,
      "margin_dpo/margin_std": 79.73458099365234,
      "step": 458
    },
    {
      "epoch": 0.6740088105726872,
      "fcm_dpo/beta": 0.01270090602338314,
      "fcm_dpo/delta": -0.07352010160684586,
      "fcm_dpo/margin": 72.08202362060547,
      "fcm_dpo/q_t": 0.3198687434196472,
      "grad_norm": 10.96422290802002,
      "learning_rate": 1.466771464027316e-07,
      "logits/chosen": -1.7052292823791504,
      "logits/rejected": -1.4467270374298096,
      "logps/chosen": -184.02252197265625,
      "logps/ref_chosen": -75.68820190429688,
      "logps/ref_rejected": -92.17048645019531,
      "logps/rejected": -272.5868225097656,
      "loss": 0.8545,
      "margin_dpo/margin_mean": 72.08201599121094,
      "margin_dpo/margin_std": 76.00017547607422,
      "step": 459
    },
    {
      "epoch": 0.6754772393538914,
      "fcm_dpo/beta": 0.012723826803267002,
      "fcm_dpo/delta": 0.032767970114946365,
      "fcm_dpo/margin": 64.40177917480469,
      "fcm_dpo/q_t": 0.3371509313583374,
      "grad_norm": 14.807209014892578,
      "learning_rate": 1.4550991377830423e-07,
      "logits/chosen": -1.999427318572998,
      "logits/rejected": -1.9956591129302979,
      "logps/chosen": -194.11785888671875,
      "logps/ref_chosen": -81.11788940429688,
      "logps/ref_rejected": -110.31238555908203,
      "logps/rejected": -287.714111328125,
      "loss": 0.9115,
      "margin_dpo/margin_mean": 64.40177917480469,
      "margin_dpo/margin_std": 75.08444213867188,
      "step": 460
    },
    {
      "epoch": 0.6769456681350955,
      "fcm_dpo/beta": 0.013109425082802773,
      "fcm_dpo/delta": 0.17610467970371246,
      "fcm_dpo/margin": 52.35367965698242,
      "fcm_dpo/q_t": 0.36317408084869385,
      "grad_norm": 14.377598762512207,
      "learning_rate": 1.4434543456482518e-07,
      "logits/chosen": -1.51175057888031,
      "logits/rejected": -1.312682867050171,
      "logps/chosen": -216.15902709960938,
      "logps/ref_chosen": -81.58352661132812,
      "logps/ref_rejected": -93.87710571289062,
      "logps/rejected": -280.8062744140625,
      "loss": 0.9802,
      "margin_dpo/margin_mean": 52.35367965698242,
      "margin_dpo/margin_std": 67.33370971679688,
      "step": 461
    },
    {
      "epoch": 0.6784140969162996,
      "fcm_dpo/beta": 0.01316889002919197,
      "fcm_dpo/delta": -0.08168931305408478,
      "fcm_dpo/margin": 70.1772232055664,
      "fcm_dpo/q_t": 0.3104817271232605,
      "grad_norm": 12.93285846710205,
      "learning_rate": 1.4318373944740484e-07,
      "logits/chosen": -2.101900100708008,
      "logits/rejected": -1.5979928970336914,
      "logps/chosen": -198.15518188476562,
      "logps/ref_chosen": -94.19854736328125,
      "logps/ref_rejected": -85.63162994384766,
      "logps/rejected": -259.7654724121094,
      "loss": 0.8176,
      "margin_dpo/margin_mean": 70.17721557617188,
      "margin_dpo/margin_std": 65.97860717773438,
      "step": 462
    },
    {
      "epoch": 0.6798825256975036,
      "fcm_dpo/beta": 0.012822076678276062,
      "fcm_dpo/delta": -0.21826551854610443,
      "fcm_dpo/margin": 81.48209381103516,
      "fcm_dpo/q_t": 0.2897373139858246,
      "grad_norm": 13.616190910339355,
      "learning_rate": 1.4202485903778976e-07,
      "logits/chosen": -1.8920228481292725,
      "logits/rejected": -1.4664556980133057,
      "logps/chosen": -187.51708984375,
      "logps/ref_chosen": -85.92474365234375,
      "logps/ref_rejected": -96.90184020996094,
      "logps/rejected": -279.97625732421875,
      "loss": 0.8029,
      "margin_dpo/margin_mean": 81.48209381103516,
      "margin_dpo/margin_std": 75.05322265625,
      "step": 463
    },
    {
      "epoch": 0.6813509544787077,
      "fcm_dpo/beta": 0.012513959780335426,
      "fcm_dpo/delta": 0.027589473873376846,
      "fcm_dpo/margin": 65.88970947265625,
      "fcm_dpo/q_t": 0.3308594822883606,
      "grad_norm": 17.59402847290039,
      "learning_rate": 1.4086882387355658e-07,
      "logits/chosen": -1.7415244579315186,
      "logits/rejected": -1.8106495141983032,
      "logps/chosen": -202.68325805664062,
      "logps/ref_chosen": -79.68919372558594,
      "logps/ref_rejected": -107.29232025146484,
      "logps/rejected": -296.1761169433594,
      "loss": 0.8853,
      "margin_dpo/margin_mean": 65.88970947265625,
      "margin_dpo/margin_std": 71.70936584472656,
      "step": 464
    },
    {
      "epoch": 0.6828193832599119,
      "fcm_dpo/beta": 0.012278483249247074,
      "fcm_dpo/delta": -0.1325981318950653,
      "fcm_dpo/margin": 78.90789794921875,
      "fcm_dpo/q_t": 0.3039529621601105,
      "grad_norm": 13.62932014465332,
      "learning_rate": 1.3971566441730714e-07,
      "logits/chosen": -1.6111962795257568,
      "logits/rejected": -1.3117473125457764,
      "logps/chosen": -204.054443359375,
      "logps/ref_chosen": -91.8602294921875,
      "logps/ref_rejected": -118.71001434326172,
      "logps/rejected": -309.8121337890625,
      "loss": 0.7876,
      "margin_dpo/margin_mean": 78.90789794921875,
      "margin_dpo/margin_std": 71.70303344726562,
      "step": 465
    },
    {
      "epoch": 0.684287812041116,
      "fcm_dpo/beta": 0.011934199370443821,
      "fcm_dpo/delta": -0.1437564343214035,
      "fcm_dpo/margin": 82.00619506835938,
      "fcm_dpo/q_t": 0.30160197615623474,
      "grad_norm": 10.586161613464355,
      "learning_rate": 1.3856541105586545e-07,
      "logits/chosen": -1.4708101749420166,
      "logits/rejected": -1.1374900341033936,
      "logps/chosen": -186.422607421875,
      "logps/ref_chosen": -84.70140838623047,
      "logps/ref_rejected": -96.05084228515625,
      "logps/rejected": -279.77825927734375,
      "loss": 0.7936,
      "margin_dpo/margin_mean": 82.00619506835938,
      "margin_dpo/margin_std": 75.4998779296875,
      "step": 466
    },
    {
      "epoch": 0.6857562408223201,
      "fcm_dpo/beta": 0.011811432428658009,
      "fcm_dpo/delta": -0.001256793737411499,
      "fcm_dpo/margin": 71.97554016113281,
      "fcm_dpo/q_t": 0.3301178514957428,
      "grad_norm": 13.264272689819336,
      "learning_rate": 1.3741809409947729e-07,
      "logits/chosen": -2.3040666580200195,
      "logits/rejected": -1.7428507804870605,
      "logps/chosen": -234.81190490722656,
      "logps/ref_chosen": -109.29832458496094,
      "logps/ref_rejected": -108.8436508178711,
      "logps/rejected": -306.332763671875,
      "loss": 0.9072,
      "margin_dpo/margin_mean": 71.97554016113281,
      "margin_dpo/margin_std": 82.74515533447266,
      "step": 467
    },
    {
      "epoch": 0.6872246696035242,
      "fcm_dpo/beta": 0.012133453041315079,
      "fcm_dpo/delta": 0.14525851607322693,
      "fcm_dpo/margin": 58.95384979248047,
      "fcm_dpo/q_t": 0.3619929850101471,
      "grad_norm": 13.532750129699707,
      "learning_rate": 1.362737437810114e-07,
      "logits/chosen": -1.8041236400604248,
      "logits/rejected": -1.618350625038147,
      "logps/chosen": -202.35598754882812,
      "logps/ref_chosen": -98.32164764404297,
      "logps/ref_rejected": -106.68048095703125,
      "logps/rejected": -269.6686706542969,
      "loss": 0.9891,
      "margin_dpo/margin_mean": 58.953853607177734,
      "margin_dpo/margin_std": 81.2957992553711,
      "step": 468
    },
    {
      "epoch": 0.6886930983847284,
      "fcm_dpo/beta": 0.012059122323989868,
      "fcm_dpo/delta": -0.059398531913757324,
      "fcm_dpo/margin": 74.865966796875,
      "fcm_dpo/q_t": 0.3162824809551239,
      "grad_norm": 15.22911548614502,
      "learning_rate": 1.351323902551631e-07,
      "logits/chosen": -2.2128915786743164,
      "logits/rejected": -1.7914154529571533,
      "logps/chosen": -218.97763061523438,
      "logps/ref_chosen": -96.76420593261719,
      "logps/ref_rejected": -109.59500885009766,
      "logps/rejected": -306.6744079589844,
      "loss": 0.8504,
      "margin_dpo/margin_mean": 74.86597442626953,
      "margin_dpo/margin_std": 76.03158569335938,
      "step": 469
    },
    {
      "epoch": 0.6901615271659325,
      "fcm_dpo/beta": 0.012082107365131378,
      "fcm_dpo/delta": -0.016534948721528053,
      "fcm_dpo/margin": 71.60215759277344,
      "fcm_dpo/q_t": 0.3221762180328369,
      "grad_norm": 12.687642097473145,
      "learning_rate": 1.339940635976592e-07,
      "logits/chosen": -1.6372177600860596,
      "logits/rejected": -1.2168526649475098,
      "logps/chosen": -199.10205078125,
      "logps/ref_chosen": -83.49665832519531,
      "logps/ref_rejected": -88.48578643798828,
      "logps/rejected": -275.6933288574219,
      "loss": 0.8488,
      "margin_dpo/margin_mean": 71.60215759277344,
      "margin_dpo/margin_std": 71.88697052001953,
      "step": 470
    },
    {
      "epoch": 0.6916299559471366,
      "fcm_dpo/beta": 0.01205938495695591,
      "fcm_dpo/delta": 0.02074136957526207,
      "fcm_dpo/margin": 68.89513397216797,
      "fcm_dpo/q_t": 0.3252991735935211,
      "grad_norm": 12.872730255126953,
      "learning_rate": 1.3285879380446563e-07,
      "logits/chosen": -1.4987744092941284,
      "logits/rejected": -1.1069977283477783,
      "logps/chosen": -221.21546936035156,
      "logps/ref_chosen": -88.47430419921875,
      "logps/ref_rejected": -90.48171997070312,
      "logps/rejected": -292.1180419921875,
      "loss": 0.852,
      "margin_dpo/margin_mean": 68.89513397216797,
      "margin_dpo/margin_std": 65.54840087890625,
      "step": 471
    },
    {
      "epoch": 0.6930983847283406,
      "fcm_dpo/beta": 0.011771542951464653,
      "fcm_dpo/delta": -0.16034545004367828,
      "fcm_dpo/margin": 84.37684631347656,
      "fcm_dpo/q_t": 0.2922556400299072,
      "grad_norm": 10.080926895141602,
      "learning_rate": 1.317266107909975e-07,
      "logits/chosen": -2.158059597015381,
      "logits/rejected": -1.4828379154205322,
      "logps/chosen": -244.9671630859375,
      "logps/ref_chosen": -125.23370361328125,
      "logps/ref_rejected": -121.05349731445312,
      "logps/rejected": -325.163818359375,
      "loss": 0.7379,
      "margin_dpo/margin_mean": 84.37684631347656,
      "margin_dpo/margin_std": 64.8412857055664,
      "step": 472
    },
    {
      "epoch": 0.6945668135095447,
      "fcm_dpo/beta": 0.012129031121730804,
      "fcm_dpo/delta": 0.19991233944892883,
      "fcm_dpo/margin": 54.608848571777344,
      "fcm_dpo/q_t": 0.3692033588886261,
      "grad_norm": 14.75636100769043,
      "learning_rate": 1.3059754439133002e-07,
      "logits/chosen": -1.7300171852111816,
      "logits/rejected": -1.2311158180236816,
      "logps/chosen": -231.66799926757812,
      "logps/ref_chosen": -95.61137390136719,
      "logps/ref_rejected": -88.15115356445312,
      "logps/rejected": -278.816650390625,
      "loss": 1.0173,
      "margin_dpo/margin_mean": 54.608848571777344,
      "margin_dpo/margin_std": 77.77999114990234,
      "step": 473
    },
    {
      "epoch": 0.6960352422907489,
      "fcm_dpo/beta": 0.012801304459571838,
      "fcm_dpo/delta": 0.23721067607402802,
      "fcm_dpo/margin": 48.58485412597656,
      "fcm_dpo/q_t": 0.3742219805717468,
      "grad_norm": 16.70572853088379,
      "learning_rate": 1.2947162435741277e-07,
      "logits/chosen": -1.5113579034805298,
      "logits/rejected": -1.3194079399108887,
      "logps/chosen": -215.53375244140625,
      "logps/ref_chosen": -81.47975158691406,
      "logps/ref_rejected": -96.46562957763672,
      "logps/rejected": -279.1044921875,
      "loss": 1.0623,
      "margin_dpo/margin_mean": 48.58485412597656,
      "margin_dpo/margin_std": 72.38153076171875,
      "step": 474
    },
    {
      "epoch": 0.697503671071953,
      "fcm_dpo/beta": 0.012611234560608864,
      "fcm_dpo/delta": -0.10543165355920792,
      "fcm_dpo/margin": 74.97346496582031,
      "fcm_dpo/q_t": 0.3156423270702362,
      "grad_norm": 13.300975799560547,
      "learning_rate": 1.2834888035828596e-07,
      "logits/chosen": -2.169642448425293,
      "logits/rejected": -2.1165261268615723,
      "logps/chosen": -175.56845092773438,
      "logps/ref_chosen": -74.19598388671875,
      "logps/ref_rejected": -94.69242095947266,
      "logps/rejected": -271.038330078125,
      "loss": 0.8354,
      "margin_dpo/margin_mean": 74.97346496582031,
      "margin_dpo/margin_std": 78.37078857421875,
      "step": 475
    },
    {
      "epoch": 0.6989720998531571,
      "fcm_dpo/beta": 0.012492308393120766,
      "fcm_dpo/delta": 0.027760498225688934,
      "fcm_dpo/margin": 65.93070983886719,
      "fcm_dpo/q_t": 0.341197669506073,
      "grad_norm": 14.65475082397461,
      "learning_rate": 1.2722934197929802e-07,
      "logits/chosen": -2.071859836578369,
      "logits/rejected": -1.693770170211792,
      "logps/chosen": -192.81143188476562,
      "logps/ref_chosen": -71.97109985351562,
      "logps/ref_rejected": -80.26224517822266,
      "logps/rejected": -267.0332946777344,
      "loss": 0.9165,
      "margin_dpo/margin_mean": 65.93070983886719,
      "margin_dpo/margin_std": 79.12750244140625,
      "step": 476
    },
    {
      "epoch": 0.7004405286343612,
      "fcm_dpo/beta": 0.012532323598861694,
      "fcm_dpo/delta": -0.045053720474243164,
      "fcm_dpo/margin": 71.10099029541016,
      "fcm_dpo/q_t": 0.31960827112197876,
      "grad_norm": 13.954968452453613,
      "learning_rate": 1.2611303872132631e-07,
      "logits/chosen": -2.056334972381592,
      "logits/rejected": -1.1700620651245117,
      "logps/chosen": -223.75918579101562,
      "logps/ref_chosen": -105.00555419921875,
      "logps/ref_rejected": -81.87843322753906,
      "logps/rejected": -271.73309326171875,
      "loss": 0.8799,
      "margin_dpo/margin_mean": 71.10099029541016,
      "margin_dpo/margin_std": 78.1356201171875,
      "step": 477
    },
    {
      "epoch": 0.7019089574155654,
      "fcm_dpo/beta": 0.012363728135824203,
      "fcm_dpo/delta": -0.06362555921077728,
      "fcm_dpo/margin": 73.42304992675781,
      "fcm_dpo/q_t": 0.3152768015861511,
      "grad_norm": 12.457188606262207,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -1.6319268941879272,
      "logits/rejected": -1.4650232791900635,
      "logps/chosen": -189.54151916503906,
      "logps/ref_chosen": -76.7882080078125,
      "logps/ref_rejected": -90.43994140625,
      "logps/rejected": -276.6163024902344,
      "loss": 0.8413,
      "margin_dpo/margin_mean": 73.42304992675781,
      "margin_dpo/margin_std": 74.97042083740234,
      "step": 478
    },
    {
      "epoch": 0.7033773861967695,
      "fcm_dpo/beta": 0.01251962035894394,
      "fcm_dpo/delta": 0.05877025052905083,
      "fcm_dpo/margin": 63.48487091064453,
      "fcm_dpo/q_t": 0.33374714851379395,
      "grad_norm": 16.574216842651367,
      "learning_rate": 1.2389025514492456e-07,
      "logits/chosen": -1.4575786590576172,
      "logits/rejected": -1.3220562934875488,
      "logps/chosen": -202.30274963378906,
      "logps/ref_chosen": -81.3623046875,
      "logps/ref_rejected": -101.09114074707031,
      "logps/rejected": -285.5164489746094,
      "loss": 0.9021,
      "margin_dpo/margin_mean": 63.48487091064453,
      "margin_dpo/margin_std": 68.2608871459961,
      "step": 479
    },
    {
      "epoch": 0.7048458149779736,
      "fcm_dpo/beta": 0.012569671496748924,
      "fcm_dpo/delta": 0.05630197376012802,
      "fcm_dpo/margin": 63.504146575927734,
      "fcm_dpo/q_t": 0.3408883213996887,
      "grad_norm": 16.51081085205078,
      "learning_rate": 1.227838333989088e-07,
      "logits/chosen": -1.806662917137146,
      "logits/rejected": -1.1421974897384644,
      "logps/chosen": -230.1032257080078,
      "logps/ref_chosen": -96.7739028930664,
      "logps/ref_rejected": -86.40473937988281,
      "logps/rejected": -283.23822021484375,
      "loss": 0.9275,
      "margin_dpo/margin_mean": 63.504150390625,
      "margin_dpo/margin_std": 75.69766235351562,
      "step": 480
    },
    {
      "epoch": 0.7063142437591777,
      "fcm_dpo/beta": 0.012597526423633099,
      "fcm_dpo/delta": -0.020995954051613808,
      "fcm_dpo/margin": 68.99488830566406,
      "fcm_dpo/q_t": 0.32530438899993896,
      "grad_norm": 14.493057250976562,
      "learning_rate": 1.2168076391719489e-07,
      "logits/chosen": -1.962436318397522,
      "logits/rejected": -1.5211691856384277,
      "logps/chosen": -224.07144165039062,
      "logps/ref_chosen": -91.670166015625,
      "logps/ref_rejected": -98.69490051269531,
      "logps/rejected": -300.091064453125,
      "loss": 0.8948,
      "margin_dpo/margin_mean": 68.99488830566406,
      "margin_dpo/margin_std": 78.63343048095703,
      "step": 481
    },
    {
      "epoch": 0.7077826725403817,
      "fcm_dpo/beta": 0.012524547055363655,
      "fcm_dpo/delta": 0.005113374441862106,
      "fcm_dpo/margin": 67.46731567382812,
      "fcm_dpo/q_t": 0.3271167278289795,
      "grad_norm": 13.560710906982422,
      "learning_rate": 1.2058107576668938e-07,
      "logits/chosen": -1.5037896633148193,
      "logits/rejected": -1.1416594982147217,
      "logps/chosen": -226.85171508789062,
      "logps/ref_chosen": -98.52011108398438,
      "logps/ref_rejected": -94.82945251464844,
      "logps/rejected": -290.62835693359375,
      "loss": 0.8862,
      "margin_dpo/margin_mean": 67.46731567382812,
      "margin_dpo/margin_std": 73.31199645996094,
      "step": 482
    },
    {
      "epoch": 0.7092511013215859,
      "fcm_dpo/beta": 0.01257384568452835,
      "fcm_dpo/delta": -0.05768094211816788,
      "fcm_dpo/margin": 71.71238708496094,
      "fcm_dpo/q_t": 0.315372496843338,
      "grad_norm": 12.017552375793457,
      "learning_rate": 1.194847979251979e-07,
      "logits/chosen": -2.2574076652526855,
      "logits/rejected": -1.6553820371627808,
      "logps/chosen": -239.170654296875,
      "logps/ref_chosen": -107.11860656738281,
      "logps/ref_rejected": -101.11499786376953,
      "logps/rejected": -304.87945556640625,
      "loss": 0.8387,
      "margin_dpo/margin_mean": 71.71238708496094,
      "margin_dpo/margin_std": 70.38067626953125,
      "step": 483
    },
    {
      "epoch": 0.71071953010279,
      "fcm_dpo/beta": 0.012338344939053059,
      "fcm_dpo/delta": -0.10980929434299469,
      "fcm_dpo/margin": 76.90882873535156,
      "fcm_dpo/q_t": 0.31271493434906006,
      "grad_norm": 11.736170768737793,
      "learning_rate": 1.1839195928066101e-07,
      "logits/chosen": -2.2439277172088623,
      "logits/rejected": -1.7512518167495728,
      "logps/chosen": -212.62677001953125,
      "logps/ref_chosen": -86.97991943359375,
      "logps/ref_rejected": -90.72367095947266,
      "logps/rejected": -293.27935791015625,
      "loss": 0.8462,
      "margin_dpo/margin_mean": 76.90882873535156,
      "margin_dpo/margin_std": 79.86015319824219,
      "step": 484
    },
    {
      "epoch": 0.7121879588839941,
      "fcm_dpo/beta": 0.011784037575125694,
      "fcm_dpo/delta": -0.212799072265625,
      "fcm_dpo/margin": 88.29972839355469,
      "fcm_dpo/q_t": 0.28984811902046204,
      "grad_norm": 11.503626823425293,
      "learning_rate": 1.1730258863039347e-07,
      "logits/chosen": -1.5688313245773315,
      "logits/rejected": -1.1244767904281616,
      "logps/chosen": -206.88958740234375,
      "logps/ref_chosen": -94.05874633789062,
      "logps/ref_rejected": -108.56298828125,
      "logps/rejected": -309.69354248046875,
      "loss": 0.7495,
      "margin_dpo/margin_mean": 88.29972839355469,
      "margin_dpo/margin_std": 75.29219818115234,
      "step": 485
    },
    {
      "epoch": 0.7136563876651982,
      "fcm_dpo/beta": 0.011598912999033928,
      "fcm_dpo/delta": -0.06779532879590988,
      "fcm_dpo/margin": 78.58279418945312,
      "fcm_dpo/q_t": 0.31678569316864014,
      "grad_norm": 13.836838722229004,
      "learning_rate": 1.1621671468032493e-07,
      "logits/chosen": -1.9958369731903076,
      "logits/rejected": -1.2726597785949707,
      "logps/chosen": -223.26283264160156,
      "logps/ref_chosen": -93.74588012695312,
      "logps/ref_rejected": -98.07064819335938,
      "logps/rejected": -306.17041015625,
      "loss": 0.8536,
      "margin_dpo/margin_mean": 78.58279418945312,
      "margin_dpo/margin_std": 80.64280700683594,
      "step": 486
    },
    {
      "epoch": 0.7151248164464024,
      "fcm_dpo/beta": 0.011509329080581665,
      "fcm_dpo/delta": 0.025798970833420753,
      "fcm_dpo/margin": 71.78614807128906,
      "fcm_dpo/q_t": 0.3311973214149475,
      "grad_norm": 12.209793090820312,
      "learning_rate": 1.1513436604424378e-07,
      "logits/chosen": -2.163451910018921,
      "logits/rejected": -1.856655240058899,
      "logps/chosen": -215.14894104003906,
      "logps/ref_chosen": -88.0335693359375,
      "logps/ref_rejected": -98.47209930419922,
      "logps/rejected": -297.3736267089844,
      "loss": 0.8644,
      "margin_dpo/margin_mean": 71.78614807128906,
      "margin_dpo/margin_std": 73.53408813476562,
      "step": 487
    },
    {
      "epoch": 0.7165932452276065,
      "fcm_dpo/beta": 0.011546431109309196,
      "fcm_dpo/delta": -0.007139893714338541,
      "fcm_dpo/margin": 74.18093872070312,
      "fcm_dpo/q_t": 0.31756943464279175,
      "grad_norm": 13.333039283752441,
      "learning_rate": 1.1405557124304335e-07,
      "logits/chosen": -1.5974540710449219,
      "logits/rejected": -1.347166657447815,
      "logps/chosen": -205.8610076904297,
      "logps/ref_chosen": -84.78964233398438,
      "logps/ref_rejected": -90.2734603881836,
      "logps/rejected": -285.5257568359375,
      "loss": 0.8218,
      "margin_dpo/margin_mean": 74.18093872070312,
      "margin_dpo/margin_std": 63.89453887939453,
      "step": 488
    },
    {
      "epoch": 0.7180616740088106,
      "fcm_dpo/beta": 0.011623598635196686,
      "fcm_dpo/delta": 0.0482901856303215,
      "fcm_dpo/margin": 69.30853271484375,
      "fcm_dpo/q_t": 0.3355333209037781,
      "grad_norm": 12.906960487365723,
      "learning_rate": 1.1298035870396985e-07,
      "logits/chosen": -1.6089611053466797,
      "logits/rejected": -1.1197456121444702,
      "logps/chosen": -221.5333709716797,
      "logps/ref_chosen": -90.46929931640625,
      "logps/ref_rejected": -86.39761352539062,
      "logps/rejected": -286.77020263671875,
      "loss": 0.8827,
      "margin_dpo/margin_mean": 69.30853271484375,
      "margin_dpo/margin_std": 73.18409729003906,
      "step": 489
    },
    {
      "epoch": 0.7195301027900147,
      "fcm_dpo/beta": 0.012023422867059708,
      "fcm_dpo/delta": 0.13658013939857483,
      "fcm_dpo/margin": 59.857852935791016,
      "fcm_dpo/q_t": 0.3625268340110779,
      "grad_norm": 13.699275016784668,
      "learning_rate": 1.1190875675987355e-07,
      "logits/chosen": -2.1816554069519043,
      "logits/rejected": -2.236203193664551,
      "logps/chosen": -222.6205291748047,
      "logps/ref_chosen": -85.32012939453125,
      "logps/ref_rejected": -115.99385070800781,
      "logps/rejected": -313.152099609375,
      "loss": 1.0544,
      "margin_dpo/margin_mean": 59.857852935791016,
      "margin_dpo/margin_std": 92.6108627319336,
      "step": 490
    },
    {
      "epoch": 0.7209985315712188,
      "fcm_dpo/beta": 0.012138845399022102,
      "fcm_dpo/delta": 0.09064659476280212,
      "fcm_dpo/margin": 63.129661560058594,
      "fcm_dpo/q_t": 0.34301620721817017,
      "grad_norm": 13.435876846313477,
      "learning_rate": 1.1084079364846241e-07,
      "logits/chosen": -2.065945625305176,
      "logits/rejected": -1.585937738418579,
      "logps/chosen": -220.5257568359375,
      "logps/ref_chosen": -86.14352416992188,
      "logps/ref_rejected": -80.67945861816406,
      "logps/rejected": -278.19134521484375,
      "loss": 0.9492,
      "margin_dpo/margin_mean": 63.129661560058594,
      "margin_dpo/margin_std": 78.26299285888672,
      "step": 491
    },
    {
      "epoch": 0.7224669603524229,
      "fcm_dpo/beta": 0.012580521404743195,
      "fcm_dpo/delta": 0.20928707718849182,
      "fcm_dpo/margin": 52.022743225097656,
      "fcm_dpo/q_t": 0.3731645345687866,
      "grad_norm": 17.193553924560547,
      "learning_rate": 1.097764975115576e-07,
      "logits/chosen": -2.168747901916504,
      "logits/rejected": -1.7894206047058105,
      "logps/chosen": -206.310791015625,
      "logps/ref_chosen": -81.10757446289062,
      "logps/ref_rejected": -80.75199890136719,
      "logps/rejected": -257.97796630859375,
      "loss": 1.1032,
      "margin_dpo/margin_mean": 52.02274703979492,
      "margin_dpo/margin_std": 90.33053588867188,
      "step": 492
    },
    {
      "epoch": 0.723935389133627,
      "fcm_dpo/beta": 0.012747792527079582,
      "fcm_dpo/delta": 0.06546922028064728,
      "fcm_dpo/margin": 61.89603805541992,
      "fcm_dpo/q_t": 0.3345375061035156,
      "grad_norm": 15.1096773147583,
      "learning_rate": 1.0871589639435203e-07,
      "logits/chosen": -2.51515793800354,
      "logits/rejected": -1.8059933185577393,
      "logps/chosen": -238.19631958007812,
      "logps/ref_chosen": -112.20733642578125,
      "logps/ref_rejected": -93.60719299316406,
      "logps/rejected": -281.4921875,
      "loss": 0.8945,
      "margin_dpo/margin_mean": 61.89603805541992,
      "margin_dpo/margin_std": 64.05390167236328,
      "step": 493
    },
    {
      "epoch": 0.7254038179148311,
      "fcm_dpo/beta": 0.012348496355116367,
      "fcm_dpo/delta": -0.22160913050174713,
      "fcm_dpo/margin": 84.58407592773438,
      "fcm_dpo/q_t": 0.2897966206073761,
      "grad_norm": 13.276835441589355,
      "learning_rate": 1.0765901824467166e-07,
      "logits/chosen": -1.4075872898101807,
      "logits/rejected": -1.3157413005828857,
      "logps/chosen": -194.07228088378906,
      "logps/ref_chosen": -73.11489868164062,
      "logps/ref_rejected": -92.16300201416016,
      "logps/rejected": -297.7044677734375,
      "loss": 0.7764,
      "margin_dpo/margin_mean": 84.58407592773438,
      "margin_dpo/margin_std": 76.84968566894531,
      "step": 494
    },
    {
      "epoch": 0.7268722466960352,
      "fcm_dpo/beta": 0.01227477751672268,
      "fcm_dpo/delta": -0.024101722985506058,
      "fcm_dpo/margin": 71.03709411621094,
      "fcm_dpo/q_t": 0.3242284059524536,
      "grad_norm": 13.635724067687988,
      "learning_rate": 1.0660589091223854e-07,
      "logits/chosen": -2.0604519844055176,
      "logits/rejected": -1.5982627868652344,
      "logps/chosen": -220.474853515625,
      "logps/ref_chosen": -99.52031707763672,
      "logps/ref_rejected": -97.93089294433594,
      "logps/rejected": -289.92254638671875,
      "loss": 0.8962,
      "margin_dpo/margin_mean": 71.03709411621094,
      "margin_dpo/margin_std": 81.31465911865234,
      "step": 495
    },
    {
      "epoch": 0.7283406754772394,
      "fcm_dpo/beta": 0.012515192851424217,
      "fcm_dpo/delta": 0.1012970507144928,
      "fcm_dpo/margin": 60.364662170410156,
      "fcm_dpo/q_t": 0.3426893353462219,
      "grad_norm": 16.073570251464844,
      "learning_rate": 1.0555654214793722e-07,
      "logits/chosen": -2.1137051582336426,
      "logits/rejected": -1.4734668731689453,
      "logps/chosen": -240.74362182617188,
      "logps/ref_chosen": -107.85675048828125,
      "logps/ref_rejected": -92.77056121826172,
      "logps/rejected": -286.0220947265625,
      "loss": 0.9345,
      "margin_dpo/margin_mean": 60.36465835571289,
      "margin_dpo/margin_std": 70.0220947265625,
      "step": 496
    },
    {
      "epoch": 0.7298091042584435,
      "fcm_dpo/beta": 0.013011728413403034,
      "fcm_dpo/delta": 0.278378963470459,
      "fcm_dpo/margin": 45.2384147644043,
      "fcm_dpo/q_t": 0.38457757234573364,
      "grad_norm": 16.124645233154297,
      "learning_rate": 1.0451099960308374e-07,
      "logits/chosen": -1.7485501766204834,
      "logits/rejected": -1.3026444911956787,
      "logps/chosen": -238.62753295898438,
      "logps/ref_chosen": -92.08322143554688,
      "logps/ref_rejected": -81.79503631591797,
      "logps/rejected": -273.5777587890625,
      "loss": 1.0902,
      "margin_dpo/margin_mean": 45.23841094970703,
      "margin_dpo/margin_std": 75.66600799560547,
      "step": 497
    },
    {
      "epoch": 0.7312775330396476,
      "fcm_dpo/beta": 0.0131129901856184,
      "fcm_dpo/delta": -0.06823568046092987,
      "fcm_dpo/margin": 69.55242156982422,
      "fcm_dpo/q_t": 0.31473374366760254,
      "grad_norm": 14.976054191589355,
      "learning_rate": 1.0346929082869641e-07,
      "logits/chosen": -1.6781526803970337,
      "logits/rejected": -1.1911282539367676,
      "logps/chosen": -218.24331665039062,
      "logps/ref_chosen": -98.19437408447266,
      "logps/ref_rejected": -90.68745422363281,
      "logps/rejected": -280.288818359375,
      "loss": 0.8364,
      "margin_dpo/margin_mean": 69.55242919921875,
      "margin_dpo/margin_std": 69.84315490722656,
      "step": 498
    },
    {
      "epoch": 0.7327459618208517,
      "fcm_dpo/beta": 0.01302518229931593,
      "fcm_dpo/delta": -0.05266323313117027,
      "fcm_dpo/margin": 68.91853332519531,
      "fcm_dpo/q_t": 0.32476356625556946,
      "grad_norm": 18.89053726196289,
      "learning_rate": 1.0243144327477013e-07,
      "logits/chosen": -2.098900318145752,
      "logits/rejected": -2.0415899753570557,
      "logps/chosen": -211.9065704345703,
      "logps/ref_chosen": -81.0399169921875,
      "logps/ref_rejected": -106.92170715332031,
      "logps/rejected": -306.7069091796875,
      "loss": 0.9069,
      "margin_dpo/margin_mean": 68.91853332519531,
      "margin_dpo/margin_std": 82.49652099609375,
      "step": 499
    },
    {
      "epoch": 0.7342143906020558,
      "fcm_dpo/beta": 0.01297258585691452,
      "fcm_dpo/delta": 0.03623470664024353,
      "fcm_dpo/margin": 62.95831298828125,
      "fcm_dpo/q_t": 0.3484196364879608,
      "grad_norm": 16.346345901489258,
      "learning_rate": 1.0139748428955333e-07,
      "logits/chosen": -1.630037546157837,
      "logits/rejected": -1.4560880661010742,
      "logps/chosen": -227.47012329101562,
      "logps/ref_chosen": -89.248046875,
      "logps/ref_rejected": -100.41021728515625,
      "logps/rejected": -301.590576171875,
      "loss": 0.9588,
      "margin_dpo/margin_mean": 62.95831298828125,
      "margin_dpo/margin_std": 83.92835235595703,
      "step": 500
    },
    {
      "epoch": 0.7342143906020558,
      "eval_fcm_dpo/beta": 0.013016212731599808,
      "eval_logits/chosen": -1.8269517421722412,
      "eval_logits/rejected": -1.456815481185913,
      "eval_logps/chosen": -233.72581481933594,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -285.7688903808594,
      "eval_loss": 0.4848414361476898,
      "eval_margin_dpo/margin_mean": 58.46889877319336,
      "eval_margin_dpo/margin_std": 75.4513168334961,
      "eval_runtime": 43.7742,
      "eval_samples_per_second": 53.433,
      "eval_steps_per_second": 1.69,
      "step": 500
    },
    {
      "epoch": 0.73568281938326,
      "fcm_dpo/beta": 0.013117056339979172,
      "fcm_dpo/delta": -0.04707694798707962,
      "fcm_dpo/margin": 67.81106567382812,
      "fcm_dpo/q_t": 0.34009435772895813,
      "grad_norm": 17.60281753540039,
      "learning_rate": 1.0036744111882672e-07,
      "logits/chosen": -1.5807795524597168,
      "logits/rejected": -1.1496039628982544,
      "logps/chosen": -212.7261199951172,
      "logps/ref_chosen": -90.596923828125,
      "logps/ref_rejected": -85.2972640991211,
      "logps/rejected": -275.237548828125,
      "loss": 0.9848,
      "margin_dpo/margin_mean": 67.81106567382812,
      "margin_dpo/margin_std": 93.16748809814453,
      "step": 501
    },
    {
      "epoch": 0.737151248164464,
      "fcm_dpo/beta": 0.013137854635715485,
      "fcm_dpo/delta": 0.034719593822956085,
      "fcm_dpo/margin": 61.971439361572266,
      "fcm_dpo/q_t": 0.34164291620254517,
      "grad_norm": 16.562210083007812,
      "learning_rate": 9.934134090518592e-08,
      "logits/chosen": -1.7543015480041504,
      "logits/rejected": -1.0072808265686035,
      "logps/chosen": -227.01095581054688,
      "logps/ref_chosen": -103.47073364257812,
      "logps/ref_rejected": -89.04415893554688,
      "logps/rejected": -274.55584716796875,
      "loss": 0.9784,
      "margin_dpo/margin_mean": 61.971435546875,
      "margin_dpo/margin_std": 80.59944152832031,
      "step": 502
    },
    {
      "epoch": 0.7386196769456681,
      "fcm_dpo/beta": 0.012945367023348808,
      "fcm_dpo/delta": -0.0793260931968689,
      "fcm_dpo/margin": 71.16193389892578,
      "fcm_dpo/q_t": 0.3206115961074829,
      "grad_norm": 14.09860610961914,
      "learning_rate": 9.831921068732571e-08,
      "logits/chosen": -1.8249884843826294,
      "logits/rejected": -1.3317406177520752,
      "logps/chosen": -212.64935302734375,
      "logps/ref_chosen": -93.72998809814453,
      "logps/ref_rejected": -87.60896301269531,
      "logps/rejected": -277.69024658203125,
      "loss": 0.8949,
      "margin_dpo/margin_mean": 71.16192626953125,
      "margin_dpo/margin_std": 81.064208984375,
      "step": 503
    },
    {
      "epoch": 0.7400881057268722,
      "fcm_dpo/beta": 0.01244557648897171,
      "fcm_dpo/delta": -0.20257821679115295,
      "fcm_dpo/margin": 82.92597961425781,
      "fcm_dpo/q_t": 0.29116249084472656,
      "grad_norm": 16.30722999572754,
      "learning_rate": 9.730107739932805e-08,
      "logits/chosen": -1.9234204292297363,
      "logits/rejected": -1.5795867443084717,
      "logps/chosen": -212.29249572753906,
      "logps/ref_chosen": -92.2918701171875,
      "logps/ref_rejected": -109.03765106201172,
      "logps/rejected": -311.9642333984375,
      "loss": 0.7785,
      "margin_dpo/margin_mean": 82.92598724365234,
      "margin_dpo/margin_std": 76.13577270507812,
      "step": 504
    },
    {
      "epoch": 0.7415565345080763,
      "fcm_dpo/beta": 0.012345531024038792,
      "fcm_dpo/delta": 0.030347377061843872,
      "fcm_dpo/margin": 66.58810424804688,
      "fcm_dpo/q_t": 0.33173486590385437,
      "grad_norm": 13.986702919006348,
      "learning_rate": 9.628696786995188e-08,
      "logits/chosen": -2.3997888565063477,
      "logits/rejected": -1.855478048324585,
      "logps/chosen": -240.9890594482422,
      "logps/ref_chosen": -106.27693176269531,
      "logps/ref_rejected": -94.20611572265625,
      "logps/rejected": -295.50634765625,
      "loss": 0.8808,
      "margin_dpo/margin_mean": 66.58810424804688,
      "margin_dpo/margin_std": 69.91239929199219,
      "step": 505
    },
    {
      "epoch": 0.7430249632892805,
      "fcm_dpo/beta": 0.012313488870859146,
      "fcm_dpo/delta": 0.00921265035867691,
      "fcm_dpo/margin": 68.321533203125,
      "fcm_dpo/q_t": 0.32348087430000305,
      "grad_norm": 13.006158828735352,
      "learning_rate": 9.527690882192635e-08,
      "logits/chosen": -1.9842698574066162,
      "logits/rejected": -1.6244713068008423,
      "logps/chosen": -212.46749877929688,
      "logps/ref_chosen": -81.44086456298828,
      "logps/ref_rejected": -85.66439819335938,
      "logps/rejected": -285.0125427246094,
      "loss": 0.8633,
      "margin_dpo/margin_mean": 68.321533203125,
      "margin_dpo/margin_std": 67.43982696533203,
      "step": 506
    },
    {
      "epoch": 0.7444933920704846,
      "fcm_dpo/beta": 0.01229831762611866,
      "fcm_dpo/delta": -0.052599359303712845,
      "fcm_dpo/margin": 73.00973510742188,
      "fcm_dpo/q_t": 0.3235676884651184,
      "grad_norm": 13.053921699523926,
      "learning_rate": 9.427092687124691e-08,
      "logits/chosen": -2.104893207550049,
      "logits/rejected": -1.6638824939727783,
      "logps/chosen": -234.46234130859375,
      "logps/ref_chosen": -99.42694854736328,
      "logps/ref_rejected": -102.25296020507812,
      "logps/rejected": -310.298095703125,
      "loss": 0.8921,
      "margin_dpo/margin_mean": 73.00973510742188,
      "margin_dpo/margin_std": 84.01727294921875,
      "step": 507
    },
    {
      "epoch": 0.7459618208516887,
      "fcm_dpo/beta": 0.012360332533717155,
      "fcm_dpo/delta": 0.08519750088453293,
      "fcm_dpo/margin": 62.41271209716797,
      "fcm_dpo/q_t": 0.35167741775512695,
      "grad_norm": 14.7086763381958,
      "learning_rate": 9.326904852647344e-08,
      "logits/chosen": -2.081237554550171,
      "logits/rejected": -1.6077560186386108,
      "logps/chosen": -230.1769256591797,
      "logps/ref_chosen": -97.9156494140625,
      "logps/ref_rejected": -103.79782104492188,
      "logps/rejected": -298.4718017578125,
      "loss": 0.9813,
      "margin_dpo/margin_mean": 62.4127197265625,
      "margin_dpo/margin_std": 83.03570556640625,
      "step": 508
    },
    {
      "epoch": 0.7474302496328928,
      "fcm_dpo/beta": 0.012669201008975506,
      "fcm_dpo/delta": 0.0940362885594368,
      "fcm_dpo/margin": 60.193206787109375,
      "fcm_dpo/q_t": 0.3435523509979248,
      "grad_norm": 17.660673141479492,
      "learning_rate": 9.227130018803195e-08,
      "logits/chosen": -1.9336552619934082,
      "logits/rejected": -1.2759418487548828,
      "logps/chosen": -226.89523315429688,
      "logps/ref_chosen": -100.72984313964844,
      "logps/ref_rejected": -86.943359375,
      "logps/rejected": -273.3019714355469,
      "loss": 0.9661,
      "margin_dpo/margin_mean": 60.193206787109375,
      "margin_dpo/margin_std": 76.53768920898438,
      "step": 509
    },
    {
      "epoch": 0.748898678414097,
      "fcm_dpo/beta": 0.012600088492035866,
      "fcm_dpo/delta": -0.0554606169462204,
      "fcm_dpo/margin": 71.46772766113281,
      "fcm_dpo/q_t": 0.3216525912284851,
      "grad_norm": 15.701079368591309,
      "learning_rate": 9.127770814751932e-08,
      "logits/chosen": -1.5974844694137573,
      "logits/rejected": -1.398494005203247,
      "logps/chosen": -234.45346069335938,
      "logps/ref_chosen": -89.16938781738281,
      "logps/ref_rejected": -107.91939544677734,
      "logps/rejected": -324.67120361328125,
      "loss": 0.8857,
      "margin_dpo/margin_mean": 71.46772766113281,
      "margin_dpo/margin_std": 79.08045196533203,
      "step": 510
    },
    {
      "epoch": 0.750367107195301,
      "fcm_dpo/beta": 0.012481224723160267,
      "fcm_dpo/delta": -0.06006241962313652,
      "fcm_dpo/margin": 72.47528076171875,
      "fcm_dpo/q_t": 0.32413989305496216,
      "grad_norm": 13.066450119018555,
      "learning_rate": 9.028829858700973e-08,
      "logits/chosen": -2.066761016845703,
      "logits/rejected": -1.504467248916626,
      "logps/chosen": -223.00457763671875,
      "logps/ref_chosen": -89.74979400634766,
      "logps/ref_rejected": -97.73127746582031,
      "logps/rejected": -303.4613342285156,
      "loss": 0.863,
      "margin_dpo/margin_mean": 72.47528076171875,
      "margin_dpo/margin_std": 80.30681610107422,
      "step": 511
    },
    {
      "epoch": 0.7518355359765051,
      "fcm_dpo/beta": 0.01244446448981762,
      "fcm_dpo/delta": -0.025120899081230164,
      "fcm_dpo/margin": 70.07406616210938,
      "fcm_dpo/q_t": 0.3278118968009949,
      "grad_norm": 15.821978569030762,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": -1.7284451723098755,
      "logits/rejected": -1.2756907939910889,
      "logps/chosen": -213.42648315429688,
      "logps/ref_chosen": -85.73515319824219,
      "logps/ref_rejected": -87.6248779296875,
      "logps/rejected": -285.3902587890625,
      "loss": 0.9145,
      "margin_dpo/margin_mean": 70.07405853271484,
      "margin_dpo/margin_std": 83.03077697753906,
      "step": 512
    },
    {
      "epoch": 0.7533039647577092,
      "fcm_dpo/beta": 0.012370715849101543,
      "fcm_dpo/delta": 0.07577605545520782,
      "fcm_dpo/margin": 63.00016403198242,
      "fcm_dpo/q_t": 0.3346492052078247,
      "grad_norm": 14.578227043151855,
      "learning_rate": 8.832213108254863e-08,
      "logits/chosen": -2.4033966064453125,
      "logits/rejected": -1.7322185039520264,
      "logps/chosen": -229.25015258789062,
      "logps/ref_chosen": -96.19050598144531,
      "logps/ref_rejected": -82.91773986816406,
      "logps/rejected": -278.9775390625,
      "loss": 0.919,
      "margin_dpo/margin_mean": 63.00016403198242,
      "margin_dpo/margin_std": 69.4985580444336,
      "step": 513
    },
    {
      "epoch": 0.7547723935389133,
      "fcm_dpo/beta": 0.012548575177788734,
      "fcm_dpo/delta": 0.03780820220708847,
      "fcm_dpo/margin": 64.95950317382812,
      "fcm_dpo/q_t": 0.3395010828971863,
      "grad_norm": 15.710688591003418,
      "learning_rate": 8.734542494893954e-08,
      "logits/chosen": -1.7342252731323242,
      "logits/rejected": -1.1726717948913574,
      "logps/chosen": -225.5660858154297,
      "logps/ref_chosen": -97.82363891601562,
      "logps/ref_rejected": -85.36982727050781,
      "logps/rejected": -278.07177734375,
      "loss": 0.9158,
      "margin_dpo/margin_mean": 64.95950317382812,
      "margin_dpo/margin_std": 76.12434387207031,
      "step": 514
    },
    {
      "epoch": 0.7562408223201175,
      "fcm_dpo/beta": 0.012832360342144966,
      "fcm_dpo/delta": 0.13085728883743286,
      "fcm_dpo/margin": 56.79905319213867,
      "fcm_dpo/q_t": 0.35357731580734253,
      "grad_norm": 19.61615753173828,
      "learning_rate": 8.637300491465272e-08,
      "logits/chosen": -1.9679467678070068,
      "logits/rejected": -1.7441508769989014,
      "logps/chosen": -210.46856689453125,
      "logps/ref_chosen": -81.93355560302734,
      "logps/ref_rejected": -93.57656860351562,
      "logps/rejected": -278.91064453125,
      "loss": 0.9894,
      "margin_dpo/margin_mean": 56.79905319213867,
      "margin_dpo/margin_std": 76.72676086425781,
      "step": 515
    },
    {
      "epoch": 0.7577092511013216,
      "fcm_dpo/beta": 0.01302811224013567,
      "fcm_dpo/delta": 0.023421771824359894,
      "fcm_dpo/margin": 63.583045959472656,
      "fcm_dpo/q_t": 0.34561917185783386,
      "grad_norm": 15.823124885559082,
      "learning_rate": 8.540489660386064e-08,
      "logits/chosen": -2.2833476066589355,
      "logits/rejected": -2.0779836177825928,
      "logps/chosen": -224.8201141357422,
      "logps/ref_chosen": -97.09166717529297,
      "logps/ref_rejected": -118.55667114257812,
      "logps/rejected": -309.8681640625,
      "loss": 1.0038,
      "margin_dpo/margin_mean": 63.583045959472656,
      "margin_dpo/margin_std": 91.61796569824219,
      "step": 516
    },
    {
      "epoch": 0.7591776798825257,
      "fcm_dpo/beta": 0.012637123465538025,
      "fcm_dpo/delta": -0.1259511113166809,
      "fcm_dpo/margin": 75.99790954589844,
      "fcm_dpo/q_t": 0.31130844354629517,
      "grad_norm": 14.500532150268555,
      "learning_rate": 8.444112552711752e-08,
      "logits/chosen": -1.8676999807357788,
      "logits/rejected": -1.4454941749572754,
      "logps/chosen": -229.08517456054688,
      "logps/ref_chosen": -93.18656921386719,
      "logps/ref_rejected": -95.5927963256836,
      "logps/rejected": -307.48931884765625,
      "loss": 0.8434,
      "margin_dpo/margin_mean": 75.99791717529297,
      "margin_dpo/margin_std": 77.92581176757812,
      "step": 517
    },
    {
      "epoch": 0.7606461086637298,
      "fcm_dpo/beta": 0.012939132750034332,
      "fcm_dpo/delta": 0.1308937817811966,
      "fcm_dpo/margin": 56.31306838989258,
      "fcm_dpo/q_t": 0.35669273138046265,
      "grad_norm": 17.170347213745117,
      "learning_rate": 8.348171708068747e-08,
      "logits/chosen": -1.8192179203033447,
      "logits/rejected": -1.4760829210281372,
      "logps/chosen": -229.3511962890625,
      "logps/ref_chosen": -87.0213623046875,
      "logps/ref_rejected": -91.24456024169922,
      "logps/rejected": -289.887451171875,
      "loss": 0.9965,
      "margin_dpo/margin_mean": 56.313072204589844,
      "margin_dpo/margin_std": 78.62522888183594,
      "step": 518
    },
    {
      "epoch": 0.762114537444934,
      "fcm_dpo/beta": 0.012952839955687523,
      "fcm_dpo/delta": 0.02188480645418167,
      "fcm_dpo/margin": 63.96747970581055,
      "fcm_dpo/q_t": 0.33184218406677246,
      "grad_norm": 13.27155590057373,
      "learning_rate": 8.25266965458755e-08,
      "logits/chosen": -1.8997950553894043,
      "logits/rejected": -1.5174694061279297,
      "logps/chosen": -232.2674560546875,
      "logps/ref_chosen": -100.68755340576172,
      "logps/ref_rejected": -109.03042602539062,
      "logps/rejected": -304.5777893066406,
      "loss": 0.8681,
      "margin_dpo/margin_mean": 63.96747589111328,
      "margin_dpo/margin_std": 64.88044738769531,
      "step": 519
    },
    {
      "epoch": 0.7635829662261381,
      "fcm_dpo/beta": 0.012810420244932175,
      "fcm_dpo/delta": -0.10472933202981949,
      "fcm_dpo/margin": 73.66597747802734,
      "fcm_dpo/q_t": 0.3046000599861145,
      "grad_norm": 14.583555221557617,
      "learning_rate": 8.15760890883607e-08,
      "logits/chosen": -1.6726075410842896,
      "logits/rejected": -1.24037504196167,
      "logps/chosen": -218.2542724609375,
      "logps/ref_chosen": -98.77762603759766,
      "logps/ref_rejected": -106.9457015991211,
      "logps/rejected": -300.08831787109375,
      "loss": 0.7999,
      "margin_dpo/margin_mean": 73.66596984863281,
      "margin_dpo/margin_std": 66.00288391113281,
      "step": 520
    },
    {
      "epoch": 0.7650513950073421,
      "fcm_dpo/beta": 0.012788889929652214,
      "fcm_dpo/delta": -0.056156057864427567,
      "fcm_dpo/margin": 70.42420959472656,
      "fcm_dpo/q_t": 0.3153071999549866,
      "grad_norm": 14.226971626281738,
      "learning_rate": 8.062991975753378e-08,
      "logits/chosen": -2.0882833003997803,
      "logits/rejected": -1.7669947147369385,
      "logps/chosen": -214.2764129638672,
      "logps/ref_chosen": -86.35108947753906,
      "logps/ref_rejected": -90.39317321777344,
      "logps/rejected": -288.7427062988281,
      "loss": 0.8338,
      "margin_dpo/margin_mean": 70.42420959472656,
      "margin_dpo/margin_std": 67.74490356445312,
      "step": 521
    },
    {
      "epoch": 0.7665198237885462,
      "fcm_dpo/beta": 0.012705507688224316,
      "fcm_dpo/delta": -0.01384912058711052,
      "fcm_dpo/margin": 67.86552429199219,
      "fcm_dpo/q_t": 0.3353983163833618,
      "grad_norm": 14.730003356933594,
      "learning_rate": 7.968821348583643e-08,
      "logits/chosen": -1.7902132272720337,
      "logits/rejected": -1.4231998920440674,
      "logps/chosen": -204.54205322265625,
      "logps/ref_chosen": -76.008056640625,
      "logps/ref_rejected": -74.44125366210938,
      "logps/rejected": -270.84075927734375,
      "loss": 0.9099,
      "margin_dpo/margin_mean": 67.86552429199219,
      "margin_dpo/margin_std": 81.35099792480469,
      "step": 522
    },
    {
      "epoch": 0.7679882525697503,
      "fcm_dpo/beta": 0.013018842786550522,
      "fcm_dpo/delta": 0.22799676656723022,
      "fcm_dpo/margin": 48.934547424316406,
      "fcm_dpo/q_t": 0.3743229806423187,
      "grad_norm": 20.326669692993164,
      "learning_rate": 7.875099508810484e-08,
      "logits/chosen": -2.163761615753174,
      "logits/rejected": -1.4758130311965942,
      "logps/chosen": -242.2533721923828,
      "logps/ref_chosen": -96.49627685546875,
      "logps/ref_rejected": -91.0152816772461,
      "logps/rejected": -285.7069396972656,
      "loss": 1.0867,
      "margin_dpo/margin_mean": 48.934547424316406,
      "margin_dpo/margin_std": 81.15193176269531,
      "step": 523
    },
    {
      "epoch": 0.7694566813509545,
      "fcm_dpo/beta": 0.013128972612321377,
      "fcm_dpo/delta": -0.031645577400922775,
      "fcm_dpo/margin": 66.92581176757812,
      "fcm_dpo/q_t": 0.3286508023738861,
      "grad_norm": 15.882726669311523,
      "learning_rate": 7.781828926091535e-08,
      "logits/chosen": -2.137564182281494,
      "logits/rejected": -1.4685100317001343,
      "logps/chosen": -236.1751708984375,
      "logps/ref_chosen": -103.40550994873047,
      "logps/ref_rejected": -87.91015625,
      "logps/rejected": -287.60565185546875,
      "loss": 0.8876,
      "margin_dpo/margin_mean": 66.92581939697266,
      "margin_dpo/margin_std": 76.40330505371094,
      "step": 524
    },
    {
      "epoch": 0.7709251101321586,
      "fcm_dpo/beta": 0.012723056599497795,
      "fcm_dpo/delta": -0.09891921281814575,
      "fcm_dpo/margin": 73.25902557373047,
      "fcm_dpo/q_t": 0.3239433765411377,
      "grad_norm": 13.863753318786621,
      "learning_rate": 7.689012058193384e-08,
      "logits/chosen": -1.6756742000579834,
      "logits/rejected": -1.5884668827056885,
      "logps/chosen": -208.05990600585938,
      "logps/ref_chosen": -84.20648193359375,
      "logps/ref_rejected": -106.26158905029297,
      "logps/rejected": -303.3740234375,
      "loss": 0.8701,
      "margin_dpo/margin_mean": 73.25902557373047,
      "margin_dpo/margin_std": 79.67434692382812,
      "step": 525
    },
    {
      "epoch": 0.7723935389133627,
      "fcm_dpo/beta": 0.012752903625369072,
      "fcm_dpo/delta": -0.08910477161407471,
      "fcm_dpo/margin": 72.99378967285156,
      "fcm_dpo/q_t": 0.3187520205974579,
      "grad_norm": 12.174683570861816,
      "learning_rate": 7.596651350926836e-08,
      "logits/chosen": -1.6931428909301758,
      "logits/rejected": -1.0449802875518799,
      "logps/chosen": -234.44082641601562,
      "logps/ref_chosen": -102.85411071777344,
      "logps/ref_rejected": -93.65145874023438,
      "logps/rejected": -298.23199462890625,
      "loss": 0.8781,
      "margin_dpo/margin_mean": 72.99378967285156,
      "margin_dpo/margin_std": 82.09791564941406,
      "step": 526
    },
    {
      "epoch": 0.7738619676945668,
      "fcm_dpo/beta": 0.012565705925226212,
      "fcm_dpo/delta": -0.046583615243434906,
      "fcm_dpo/margin": 71.01980590820312,
      "fcm_dpo/q_t": 0.317333459854126,
      "grad_norm": 12.275128364562988,
      "learning_rate": 7.504749238082414e-08,
      "logits/chosen": -3.2490463256835938,
      "logits/rejected": -2.4732651710510254,
      "logps/chosen": -226.13568115234375,
      "logps/ref_chosen": -96.76543426513672,
      "logps/ref_rejected": -85.39518737792969,
      "logps/rejected": -285.78521728515625,
      "loss": 0.8292,
      "margin_dpo/margin_mean": 71.01980590820312,
      "margin_dpo/margin_std": 68.41850280761719,
      "step": 527
    },
    {
      "epoch": 0.775330396475771,
      "fcm_dpo/beta": 0.012453284114599228,
      "fcm_dpo/delta": -0.058921098709106445,
      "fcm_dpo/margin": 72.56159973144531,
      "fcm_dpo/q_t": 0.31946584582328796,
      "grad_norm": 13.592493057250977,
      "learning_rate": 7.413308141366254e-08,
      "logits/chosen": -2.2505486011505127,
      "logits/rejected": -1.7673816680908203,
      "logps/chosen": -215.16543579101562,
      "logps/ref_chosen": -94.1649169921875,
      "logps/ref_rejected": -100.372314453125,
      "logps/rejected": -293.9344482421875,
      "loss": 0.8619,
      "margin_dpo/margin_mean": 72.56159973144531,
      "margin_dpo/margin_std": 77.28302764892578,
      "step": 528
    },
    {
      "epoch": 0.7767988252569751,
      "fcm_dpo/beta": 0.012501591816544533,
      "fcm_dpo/delta": 0.05900266766548157,
      "fcm_dpo/margin": 63.64674758911133,
      "fcm_dpo/q_t": 0.34856054186820984,
      "grad_norm": 15.007195472717285,
      "learning_rate": 7.322330470336313e-08,
      "logits/chosen": -1.9466891288757324,
      "logits/rejected": -1.7657006978988647,
      "logps/chosen": -221.71871948242188,
      "logps/ref_chosen": -80.77680206298828,
      "logps/ref_rejected": -97.191162109375,
      "logps/rejected": -301.77984619140625,
      "loss": 0.9653,
      "margin_dpo/margin_mean": 63.64674758911133,
      "margin_dpo/margin_std": 85.80581665039062,
      "step": 529
    },
    {
      "epoch": 0.7782672540381792,
      "fcm_dpo/beta": 0.012481624260544777,
      "fcm_dpo/delta": 0.0072569213807582855,
      "fcm_dpo/margin": 67.52183532714844,
      "fcm_dpo/q_t": 0.3254541754722595,
      "grad_norm": 12.42430305480957,
      "learning_rate": 7.231818622338822e-08,
      "logits/chosen": -1.5578473806381226,
      "logits/rejected": -1.227747917175293,
      "logps/chosen": -209.4708251953125,
      "logps/ref_chosen": -79.7757339477539,
      "logps/ref_rejected": -92.05390167236328,
      "logps/rejected": -289.27081298828125,
      "loss": 0.8365,
      "margin_dpo/margin_mean": 67.5218276977539,
      "margin_dpo/margin_std": 62.72408676147461,
      "step": 530
    },
    {
      "epoch": 0.7797356828193832,
      "fcm_dpo/beta": 0.012329411692917347,
      "fcm_dpo/delta": -0.1382937878370285,
      "fcm_dpo/margin": 79.07818603515625,
      "fcm_dpo/q_t": 0.2973644733428955,
      "grad_norm": 11.442774772644043,
      "learning_rate": 7.141774982445147e-08,
      "logits/chosen": -1.8888896703720093,
      "logits/rejected": -1.2946207523345947,
      "logps/chosen": -206.50308227539062,
      "logps/ref_chosen": -79.31919860839844,
      "logps/ref_rejected": -78.10832214355469,
      "logps/rejected": -284.37042236328125,
      "loss": 0.7856,
      "margin_dpo/margin_mean": 79.07820129394531,
      "margin_dpo/margin_std": 69.38872528076172,
      "step": 531
    },
    {
      "epoch": 0.7812041116005873,
      "fcm_dpo/beta": 0.01183237787336111,
      "fcm_dpo/delta": -0.23072879016399384,
      "fcm_dpo/margin": 89.30570983886719,
      "fcm_dpo/q_t": 0.28500646352767944,
      "grad_norm": 12.133696556091309,
      "learning_rate": 7.052201923388953e-08,
      "logits/chosen": -1.5650770664215088,
      "logits/rejected": -0.9464230537414551,
      "logps/chosen": -235.76779174804688,
      "logps/ref_chosen": -101.63691711425781,
      "logps/ref_rejected": -93.34539031982422,
      "logps/rejected": -316.781982421875,
      "loss": 0.7458,
      "margin_dpo/margin_mean": 89.30570983886719,
      "margin_dpo/margin_std": 74.34683227539062,
      "step": 532
    },
    {
      "epoch": 0.7826725403817915,
      "fcm_dpo/beta": 0.012062860652804375,
      "fcm_dpo/delta": 0.2239835560321808,
      "fcm_dpo/margin": 53.07807159423828,
      "fcm_dpo/q_t": 0.372709721326828,
      "grad_norm": 14.061158180236816,
      "learning_rate": 6.963101805503646e-08,
      "logits/chosen": -2.508441925048828,
      "logits/rejected": -1.9440903663635254,
      "logps/chosen": -220.5234832763672,
      "logps/ref_chosen": -90.74664306640625,
      "logps/ref_rejected": -82.88848876953125,
      "logps/rejected": -265.743408203125,
      "loss": 1.0218,
      "margin_dpo/margin_mean": 53.07807540893555,
      "margin_dpo/margin_std": 76.28727722167969,
      "step": 533
    },
    {
      "epoch": 0.7841409691629956,
      "fcm_dpo/beta": 0.012088856659829617,
      "fcm_dpo/delta": -0.07449840009212494,
      "fcm_dpo/margin": 75.91458129882812,
      "fcm_dpo/q_t": 0.3064717650413513,
      "grad_norm": 11.550370216369629,
      "learning_rate": 6.874476976660184e-08,
      "logits/chosen": -1.9885363578796387,
      "logits/rejected": -1.613828182220459,
      "logps/chosen": -205.0845947265625,
      "logps/ref_chosen": -86.6308364868164,
      "logps/ref_rejected": -84.87196350097656,
      "logps/rejected": -279.24029541015625,
      "loss": 0.7894,
      "margin_dpo/margin_mean": 75.91458129882812,
      "margin_dpo/margin_std": 63.02655029296875,
      "step": 534
    },
    {
      "epoch": 0.7856093979441997,
      "fcm_dpo/beta": 0.011910516768693924,
      "fcm_dpo/delta": -0.1172776147723198,
      "fcm_dpo/margin": 80.23560333251953,
      "fcm_dpo/q_t": 0.3084285855293274,
      "grad_norm": 12.470861434936523,
      "learning_rate": 6.786329772205246e-08,
      "logits/chosen": -1.8749089241027832,
      "logits/rejected": -1.6865028142929077,
      "logps/chosen": -207.06907653808594,
      "logps/ref_chosen": -90.34539031982422,
      "logps/ref_rejected": -100.7779769897461,
      "logps/rejected": -297.7372741699219,
      "loss": 0.826,
      "margin_dpo/margin_mean": 80.23561096191406,
      "margin_dpo/margin_std": 79.02330017089844,
      "step": 535
    },
    {
      "epoch": 0.7870778267254038,
      "fcm_dpo/beta": 0.011703304946422577,
      "fcm_dpo/delta": -0.013192192651331425,
      "fcm_dpo/margin": 73.65609741210938,
      "fcm_dpo/q_t": 0.32144200801849365,
      "grad_norm": 12.106589317321777,
      "learning_rate": 6.698662514899638e-08,
      "logits/chosen": -2.0477590560913086,
      "logits/rejected": -1.803931713104248,
      "logps/chosen": -203.61679077148438,
      "logps/ref_chosen": -78.95956420898438,
      "logps/ref_rejected": -93.08779907226562,
      "logps/rejected": -291.401123046875,
      "loss": 0.8582,
      "margin_dpo/margin_mean": 73.65609741210938,
      "margin_dpo/margin_std": 74.89938354492188,
      "step": 536
    },
    {
      "epoch": 0.788546255506608,
      "fcm_dpo/beta": 0.011510086245834827,
      "fcm_dpo/delta": -0.182314932346344,
      "fcm_dpo/margin": 88.06642150878906,
      "fcm_dpo/q_t": 0.2944716215133667,
      "grad_norm": 9.836989402770996,
      "learning_rate": 6.611477514857114e-08,
      "logits/chosen": -1.6331989765167236,
      "logits/rejected": -0.941495418548584,
      "logps/chosen": -192.248779296875,
      "logps/ref_chosen": -86.70939636230469,
      "logps/ref_rejected": -78.90766906738281,
      "logps/rejected": -272.51348876953125,
      "loss": 0.7689,
      "margin_dpo/margin_mean": 88.06642150878906,
      "margin_dpo/margin_std": 75.45658874511719,
      "step": 537
    },
    {
      "epoch": 0.7900146842878121,
      "fcm_dpo/beta": 0.011464505456387997,
      "fcm_dpo/delta": 0.08413688838481903,
      "fcm_dpo/margin": 67.333984375,
      "fcm_dpo/q_t": 0.33640989661216736,
      "grad_norm": 13.633325576782227,
      "learning_rate": 6.524777069483525e-08,
      "logits/chosen": -1.8363103866577148,
      "logits/rejected": -1.3120193481445312,
      "logps/chosen": -238.7105712890625,
      "logps/ref_chosen": -98.44214630126953,
      "logps/ref_rejected": -91.09465026855469,
      "logps/rejected": -298.69708251953125,
      "loss": 0.874,
      "margin_dpo/margin_mean": 67.333984375,
      "margin_dpo/margin_std": 65.11286926269531,
      "step": 538
    },
    {
      "epoch": 0.7914831130690162,
      "fcm_dpo/beta": 0.01155300997197628,
      "fcm_dpo/delta": 0.028006993234157562,
      "fcm_dpo/margin": 71.33673095703125,
      "fcm_dpo/q_t": 0.33324581384658813,
      "grad_norm": 11.587471961975098,
      "learning_rate": 6.438563463416221e-08,
      "logits/chosen": -2.5877199172973633,
      "logits/rejected": -2.1347360610961914,
      "logps/chosen": -217.88844299316406,
      "logps/ref_chosen": -90.93211364746094,
      "logps/ref_rejected": -97.44546508789062,
      "logps/rejected": -295.738525390625,
      "loss": 0.9109,
      "margin_dpo/margin_mean": 71.33673095703125,
      "margin_dpo/margin_std": 82.09566497802734,
      "step": 539
    },
    {
      "epoch": 0.7929515418502202,
      "fcm_dpo/beta": 0.011464366689324379,
      "fcm_dpo/delta": -0.09363038092851639,
      "fcm_dpo/margin": 81.52059936523438,
      "fcm_dpo/q_t": 0.3140173852443695,
      "grad_norm": 13.714081764221191,
      "learning_rate": 6.352838968463919e-08,
      "logits/chosen": -2.0992918014526367,
      "logits/rejected": -1.8551020622253418,
      "logps/chosen": -236.30484008789062,
      "logps/ref_chosen": -107.86990356445312,
      "logps/ref_rejected": -121.39350128173828,
      "logps/rejected": -331.3490295410156,
      "loss": 0.8469,
      "margin_dpo/margin_mean": 81.52059936523438,
      "margin_dpo/margin_std": 84.92539978027344,
      "step": 540
    },
    {
      "epoch": 0.7944199706314243,
      "fcm_dpo/beta": 0.01155078411102295,
      "fcm_dpo/delta": 0.18257349729537964,
      "fcm_dpo/margin": 58.86783981323242,
      "fcm_dpo/q_t": 0.35946542024612427,
      "grad_norm": 13.322444915771484,
      "learning_rate": 6.267605843546767e-08,
      "logits/chosen": -2.391913414001465,
      "logits/rejected": -1.9251333475112915,
      "logps/chosen": -250.4494171142578,
      "logps/ref_chosen": -106.56748962402344,
      "logps/ref_rejected": -109.06633758544922,
      "logps/rejected": -311.81610107421875,
      "loss": 0.96,
      "margin_dpo/margin_mean": 58.86784362792969,
      "margin_dpo/margin_std": 69.12492370605469,
      "step": 541
    },
    {
      "epoch": 0.7958883994126285,
      "fcm_dpo/beta": 0.011575637385249138,
      "fcm_dpo/delta": -0.11270305514335632,
      "fcm_dpo/margin": 82.24880981445312,
      "fcm_dpo/q_t": 0.30517640709877014,
      "grad_norm": 11.769819259643555,
      "learning_rate": 6.182866334636888e-08,
      "logits/chosen": -2.314541816711426,
      "logits/rejected": -2.1913418769836426,
      "logps/chosen": -210.772705078125,
      "logps/ref_chosen": -87.36929321289062,
      "logps/ref_rejected": -102.82034301757812,
      "logps/rejected": -308.47259521484375,
      "loss": 0.8087,
      "margin_dpo/margin_mean": 82.24880981445312,
      "margin_dpo/margin_std": 76.32795715332031,
      "step": 542
    },
    {
      "epoch": 0.7973568281938326,
      "fcm_dpo/beta": 0.011980516836047173,
      "fcm_dpo/delta": 0.3275827467441559,
      "fcm_dpo/margin": 45.187530517578125,
      "fcm_dpo/q_t": 0.39750808477401733,
      "grad_norm": 16.94487762451172,
      "learning_rate": 6.098622674699147e-08,
      "logits/chosen": -1.8013958930969238,
      "logits/rejected": -1.8136345148086548,
      "logps/chosen": -229.92324829101562,
      "logps/ref_chosen": -87.152587890625,
      "logps/ref_rejected": -111.53172302246094,
      "logps/rejected": -299.48992919921875,
      "loss": 1.15,
      "margin_dpo/margin_mean": 45.187530517578125,
      "margin_dpo/margin_std": 86.8045654296875,
      "step": 543
    },
    {
      "epoch": 0.7988252569750367,
      "fcm_dpo/beta": 0.012570763006806374,
      "fcm_dpo/delta": 0.166754812002182,
      "fcm_dpo/margin": 55.278053283691406,
      "fcm_dpo/q_t": 0.36193686723709106,
      "grad_norm": 16.597082138061523,
      "learning_rate": 6.01487708363232e-08,
      "logits/chosen": -1.80258309841156,
      "logits/rejected": -1.694300651550293,
      "logps/chosen": -230.8009033203125,
      "logps/ref_chosen": -86.14691162109375,
      "logps/ref_rejected": -106.3939208984375,
      "logps/rejected": -306.32598876953125,
      "loss": 0.9627,
      "margin_dpo/margin_mean": 55.278053283691406,
      "margin_dpo/margin_std": 71.11997985839844,
      "step": 544
    },
    {
      "epoch": 0.8002936857562408,
      "fcm_dpo/beta": 0.01272234320640564,
      "fcm_dpo/delta": -0.014371376484632492,
      "fcm_dpo/margin": 67.8095474243164,
      "fcm_dpo/q_t": 0.33010321855545044,
      "grad_norm": 15.109139442443848,
      "learning_rate": 5.9316317682106294e-08,
      "logits/chosen": -1.6739213466644287,
      "logits/rejected": -1.2589714527130127,
      "logps/chosen": -229.88116455078125,
      "logps/ref_chosen": -96.42424011230469,
      "logps/ref_rejected": -100.73405456542969,
      "logps/rejected": -302.0005187988281,
      "loss": 0.9374,
      "margin_dpo/margin_mean": 67.8095474243164,
      "margin_dpo/margin_std": 83.960205078125,
      "step": 545
    },
    {
      "epoch": 0.801762114537445,
      "fcm_dpo/beta": 0.012656424194574356,
      "fcm_dpo/delta": 0.03976099565625191,
      "fcm_dpo/margin": 64.24735260009766,
      "fcm_dpo/q_t": 0.334933340549469,
      "grad_norm": 14.526140213012695,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": -1.5082588195800781,
      "logits/rejected": -1.1441333293914795,
      "logps/chosen": -222.2100830078125,
      "logps/ref_chosen": -86.02112579345703,
      "logps/ref_rejected": -87.37263488769531,
      "logps/rejected": -287.8089599609375,
      "loss": 0.9323,
      "margin_dpo/margin_mean": 64.24735260009766,
      "margin_dpo/margin_std": 77.9768295288086,
      "step": 546
    },
    {
      "epoch": 0.8032305433186491,
      "fcm_dpo/beta": 0.012681668624281883,
      "fcm_dpo/delta": -0.08525756001472473,
      "fcm_dpo/margin": 73.09080505371094,
      "fcm_dpo/q_t": 0.3053755462169647,
      "grad_norm": 13.63310432434082,
      "learning_rate": 5.7666507254280265e-08,
      "logits/chosen": -1.8355967998504639,
      "logits/rejected": -1.389392375946045,
      "logps/chosen": -232.23291015625,
      "logps/ref_chosen": -100.6510009765625,
      "logps/ref_rejected": -97.9117431640625,
      "logps/rejected": -302.58447265625,
      "loss": 0.7958,
      "margin_dpo/margin_mean": 73.09080505371094,
      "margin_dpo/margin_std": 62.29001235961914,
      "step": 547
    },
    {
      "epoch": 0.8046989720998532,
      "fcm_dpo/beta": 0.012857835739850998,
      "fcm_dpo/delta": 0.1577250212430954,
      "fcm_dpo/margin": 54.670692443847656,
      "fcm_dpo/q_t": 0.3624279201030731,
      "grad_norm": 16.492679595947266,
      "learning_rate": 5.684919345471029e-08,
      "logits/chosen": -2.2445602416992188,
      "logits/rejected": -1.89778733253479,
      "logps/chosen": -231.13241577148438,
      "logps/ref_chosen": -96.03010559082031,
      "logps/ref_rejected": -98.41322326660156,
      "logps/rejected": -288.18621826171875,
      "loss": 1.0519,
      "margin_dpo/margin_mean": 54.67068862915039,
      "margin_dpo/margin_std": 85.29281616210938,
      "step": 548
    },
    {
      "epoch": 0.8061674008810573,
      "fcm_dpo/beta": 0.012982659041881561,
      "fcm_dpo/delta": -0.01578594744205475,
      "fcm_dpo/margin": 66.53627014160156,
      "fcm_dpo/q_t": 0.33002233505249023,
      "grad_norm": 15.831024169921875,
      "learning_rate": 5.603696935852426e-08,
      "logits/chosen": -1.904597520828247,
      "logits/rejected": -1.4875590801239014,
      "logps/chosen": -207.3582305908203,
      "logps/ref_chosen": -83.45095825195312,
      "logps/ref_rejected": -81.38023376464844,
      "logps/rejected": -271.8237609863281,
      "loss": 0.9136,
      "margin_dpo/margin_mean": 66.53626251220703,
      "margin_dpo/margin_std": 78.87059020996094,
      "step": 549
    },
    {
      "epoch": 0.8076358296622613,
      "fcm_dpo/beta": 0.012999649159610271,
      "fcm_dpo/delta": 0.010091569274663925,
      "fcm_dpo/margin": 64.60523986816406,
      "fcm_dpo/q_t": 0.32790806889533997,
      "grad_norm": 14.324188232421875,
      "learning_rate": 5.5229856368582376e-08,
      "logits/chosen": -1.564119815826416,
      "logits/rejected": -1.3338861465454102,
      "logps/chosen": -226.260498046875,
      "logps/ref_chosen": -89.59434509277344,
      "logps/ref_rejected": -101.31549835205078,
      "logps/rejected": -302.5868835449219,
      "loss": 0.8718,
      "margin_dpo/margin_mean": 64.60523986816406,
      "margin_dpo/margin_std": 65.74769592285156,
      "step": 550
    },
    {
      "epoch": 0.8091042584434655,
      "fcm_dpo/beta": 0.01258097868412733,
      "fcm_dpo/delta": -0.110738605260849,
      "fcm_dpo/margin": 75.28948974609375,
      "fcm_dpo/q_t": 0.31237906217575073,
      "grad_norm": 16.70378875732422,
      "learning_rate": 5.4427875753062734e-08,
      "logits/chosen": -1.785810947418213,
      "logits/rejected": -1.634450912475586,
      "logps/chosen": -216.82296752929688,
      "logps/ref_chosen": -95.08216094970703,
      "logps/ref_rejected": -117.7192611694336,
      "logps/rejected": -314.74957275390625,
      "loss": 0.8109,
      "margin_dpo/margin_mean": 75.28948974609375,
      "margin_dpo/margin_std": 73.0545883178711,
      "step": 551
    },
    {
      "epoch": 0.8105726872246696,
      "fcm_dpo/beta": 0.011867163702845573,
      "fcm_dpo/delta": -0.32839635014533997,
      "fcm_dpo/margin": 95.468994140625,
      "fcm_dpo/q_t": 0.2767210304737091,
      "grad_norm": 11.077062606811523,
      "learning_rate": 5.363104864490034e-08,
      "logits/chosen": -2.2213876247406006,
      "logits/rejected": -1.8646334409713745,
      "logps/chosen": -221.26583862304688,
      "logps/ref_chosen": -101.40918731689453,
      "logps/ref_rejected": -110.83843994140625,
      "logps/rejected": -326.1640625,
      "loss": 0.7468,
      "margin_dpo/margin_mean": 95.46898651123047,
      "margin_dpo/margin_std": 84.19966125488281,
      "step": 552
    },
    {
      "epoch": 0.8120411160058737,
      "fcm_dpo/beta": 0.012067961506545544,
      "fcm_dpo/delta": 0.11354109644889832,
      "fcm_dpo/margin": 61.68964385986328,
      "fcm_dpo/q_t": 0.3463653326034546,
      "grad_norm": 14.573863983154297,
      "learning_rate": 5.2839396041230415e-08,
      "logits/chosen": -1.5574626922607422,
      "logits/rejected": -1.2425198554992676,
      "logps/chosen": -228.42669677734375,
      "logps/ref_chosen": -97.96388244628906,
      "logps/ref_rejected": -103.57719421386719,
      "logps/rejected": -295.72967529296875,
      "loss": 0.9434,
      "margin_dpo/margin_mean": 61.68965148925781,
      "margin_dpo/margin_std": 73.47972106933594,
      "step": 553
    },
    {
      "epoch": 0.8135095447870778,
      "fcm_dpo/beta": 0.012185090221464634,
      "fcm_dpo/delta": -0.0028081685304641724,
      "fcm_dpo/margin": 69.8938217163086,
      "fcm_dpo/q_t": 0.33668839931488037,
      "grad_norm": 14.112452507019043,
      "learning_rate": 5.205293880283551e-08,
      "logits/chosen": -1.8471527099609375,
      "logits/rejected": -1.0901433229446411,
      "logps/chosen": -237.7957000732422,
      "logps/ref_chosen": -103.68680572509766,
      "logps/ref_rejected": -92.99210357666016,
      "logps/rejected": -296.99481201171875,
      "loss": 0.9633,
      "margin_dpo/margin_mean": 69.8938217163086,
      "margin_dpo/margin_std": 92.53191375732422,
      "step": 554
    },
    {
      "epoch": 0.8149779735682819,
      "fcm_dpo/beta": 0.012208528816699982,
      "fcm_dpo/delta": 0.06300178170204163,
      "fcm_dpo/margin": 64.8729248046875,
      "fcm_dpo/q_t": 0.3472994565963745,
      "grad_norm": 14.670280456542969,
      "learning_rate": 5.127169765359515e-08,
      "logits/chosen": -1.7789452075958252,
      "logits/rejected": -1.679337978363037,
      "logps/chosen": -227.99090576171875,
      "logps/ref_chosen": -86.20362854003906,
      "logps/ref_rejected": -114.54719543457031,
      "logps/rejected": -321.2073974609375,
      "loss": 0.962,
      "margin_dpo/margin_mean": 64.8729248046875,
      "margin_dpo/margin_std": 85.02949523925781,
      "step": 555
    },
    {
      "epoch": 0.8164464023494861,
      "fcm_dpo/beta": 0.012215223163366318,
      "fcm_dpo/delta": -0.005190940573811531,
      "fcm_dpo/margin": 69.96516418457031,
      "fcm_dpo/q_t": 0.326467365026474,
      "grad_norm": 13.305697441101074,
      "learning_rate": 5.049569317994012e-08,
      "logits/chosen": -1.7430949211120605,
      "logits/rejected": -1.4270676374435425,
      "logps/chosen": -215.5836639404297,
      "logps/ref_chosen": -88.40348815917969,
      "logps/ref_rejected": -106.63911437988281,
      "logps/rejected": -303.7844543457031,
      "loss": 0.8898,
      "margin_dpo/margin_mean": 69.96516418457031,
      "margin_dpo/margin_std": 77.99856567382812,
      "step": 556
    },
    {
      "epoch": 0.8179148311306902,
      "fcm_dpo/beta": 0.011995375156402588,
      "fcm_dpo/delta": -0.07914155721664429,
      "fcm_dpo/margin": 76.68682861328125,
      "fcm_dpo/q_t": 0.3210620880126953,
      "grad_norm": 11.786112785339355,
      "learning_rate": 4.9724945830310144e-08,
      "logits/chosen": -2.0711090564727783,
      "logits/rejected": -1.9035547971725464,
      "logps/chosen": -234.16250610351562,
      "logps/ref_chosen": -98.83523559570312,
      "logps/ref_rejected": -115.64888763427734,
      "logps/rejected": -327.6629943847656,
      "loss": 0.8671,
      "margin_dpo/margin_mean": 76.68682861328125,
      "margin_dpo/margin_std": 83.49786376953125,
      "step": 557
    },
    {
      "epoch": 0.8193832599118943,
      "fcm_dpo/beta": 0.011821841821074486,
      "fcm_dpo/delta": -0.09789042174816132,
      "fcm_dpo/margin": 79.33963012695312,
      "fcm_dpo/q_t": 0.3074336647987366,
      "grad_norm": 12.818613052368164,
      "learning_rate": 4.8959475914614554e-08,
      "logits/chosen": -2.476902961730957,
      "logits/rejected": -1.8757622241973877,
      "logps/chosen": -243.42117309570312,
      "logps/ref_chosen": -106.01183319091797,
      "logps/ref_rejected": -108.39131927490234,
      "logps/rejected": -325.1402893066406,
      "loss": 0.8074,
      "margin_dpo/margin_mean": 79.33963012695312,
      "margin_dpo/margin_std": 73.38945770263672,
      "step": 558
    },
    {
      "epoch": 0.8208516886930984,
      "fcm_dpo/beta": 0.01184871792793274,
      "fcm_dpo/delta": 0.024944249540567398,
      "fcm_dpo/margin": 69.80717468261719,
      "fcm_dpo/q_t": 0.32996243238449097,
      "grad_norm": 14.529841423034668,
      "learning_rate": 4.8199303603697614e-08,
      "logits/chosen": -2.562124252319336,
      "logits/rejected": -2.099799871444702,
      "logps/chosen": -242.94049072265625,
      "logps/ref_chosen": -100.42474365234375,
      "logps/ref_rejected": -100.41270446777344,
      "logps/rejected": -312.7356262207031,
      "loss": 0.9057,
      "margin_dpo/margin_mean": 69.80717468261719,
      "margin_dpo/margin_std": 79.37966918945312,
      "step": 559
    },
    {
      "epoch": 0.8223201174743024,
      "fcm_dpo/beta": 0.011990891769528389,
      "fcm_dpo/delta": 0.07405167818069458,
      "fcm_dpo/margin": 65.19851684570312,
      "fcm_dpo/q_t": 0.34072330594062805,
      "grad_norm": 13.63769245147705,
      "learning_rate": 4.7444448928806615e-08,
      "logits/chosen": -1.3843114376068115,
      "logits/rejected": -0.9339506030082703,
      "logps/chosen": -229.32911682128906,
      "logps/ref_chosen": -87.63551330566406,
      "logps/ref_rejected": -87.37985229492188,
      "logps/rejected": -294.27197265625,
      "loss": 0.9346,
      "margin_dpo/margin_mean": 65.1985092163086,
      "margin_dpo/margin_std": 78.38851928710938,
      "step": 560
    },
    {
      "epoch": 0.8237885462555066,
      "fcm_dpo/beta": 0.012294452637434006,
      "fcm_dpo/delta": 0.0976395532488823,
      "fcm_dpo/margin": 61.706260681152344,
      "fcm_dpo/q_t": 0.34566307067871094,
      "grad_norm": 13.019865989685059,
      "learning_rate": 4.669493178106432e-08,
      "logits/chosen": -1.9657562971115112,
      "logits/rejected": -1.9275360107421875,
      "logps/chosen": -221.61062622070312,
      "logps/ref_chosen": -74.88140106201172,
      "logps/ref_rejected": -106.04796600341797,
      "logps/rejected": -314.48345947265625,
      "loss": 0.9111,
      "margin_dpo/margin_mean": 61.706260681152344,
      "margin_dpo/margin_std": 68.51325225830078,
      "step": 561
    },
    {
      "epoch": 0.8252569750367107,
      "fcm_dpo/beta": 0.012495906092226505,
      "fcm_dpo/delta": 0.11548486351966858,
      "fcm_dpo/margin": 59.469627380371094,
      "fcm_dpo/q_t": 0.35613298416137695,
      "grad_norm": 16.456663131713867,
      "learning_rate": 4.5950771910944596e-08,
      "logits/chosen": -2.2097997665405273,
      "logits/rejected": -1.7643963098526,
      "logps/chosen": -233.54342651367188,
      "logps/ref_chosen": -100.66322326660156,
      "logps/ref_rejected": -102.24087524414062,
      "logps/rejected": -294.5906982421875,
      "loss": 1.0184,
      "margin_dpo/margin_mean": 59.469627380371094,
      "margin_dpo/margin_std": 88.21414184570312,
      "step": 562
    },
    {
      "epoch": 0.8267254038179148,
      "fcm_dpo/beta": 0.012950560078024864,
      "fcm_dpo/delta": 0.14667396247386932,
      "fcm_dpo/margin": 54.94389343261719,
      "fcm_dpo/q_t": 0.35650813579559326,
      "grad_norm": 15.449488639831543,
      "learning_rate": 4.521198892775202e-08,
      "logits/chosen": -1.6731395721435547,
      "logits/rejected": -1.3347598314285278,
      "logps/chosen": -239.73776245117188,
      "logps/ref_chosen": -96.25921630859375,
      "logps/ref_rejected": -101.20503997802734,
      "logps/rejected": -299.6274719238281,
      "loss": 0.9925,
      "margin_dpo/margin_mean": 54.94389343261719,
      "margin_dpo/margin_std": 74.90940856933594,
      "step": 563
    },
    {
      "epoch": 0.8281938325991189,
      "fcm_dpo/beta": 0.01319155003875494,
      "fcm_dpo/delta": 0.06683439761400223,
      "fcm_dpo/margin": 59.61730194091797,
      "fcm_dpo/q_t": 0.3362036943435669,
      "grad_norm": 13.23291015625,
      "learning_rate": 4.447860229910544e-08,
      "logits/chosen": -2.3670005798339844,
      "logits/rejected": -1.8159103393554688,
      "logps/chosen": -251.44268798828125,
      "logps/ref_chosen": -112.96040344238281,
      "logps/ref_rejected": -99.52803802490234,
      "logps/rejected": -297.6276550292969,
      "loss": 0.8982,
      "margin_dpo/margin_mean": 59.6172981262207,
      "margin_dpo/margin_std": 62.563438415527344,
      "step": 564
    },
    {
      "epoch": 0.8296622613803231,
      "fcm_dpo/beta": 0.012753048911690712,
      "fcm_dpo/delta": -0.17041520774364471,
      "fcm_dpo/margin": 78.61331176757812,
      "fcm_dpo/q_t": 0.30064839124679565,
      "grad_norm": 12.86750316619873,
      "learning_rate": 4.375063135042445e-08,
      "logits/chosen": -1.7283689975738525,
      "logits/rejected": -1.225877285003662,
      "logps/chosen": -224.79379272460938,
      "logps/ref_chosen": -95.36558532714844,
      "logps/ref_rejected": -92.21737670898438,
      "logps/rejected": -300.2589111328125,
      "loss": 0.803,
      "margin_dpo/margin_mean": 78.61331176757812,
      "margin_dpo/margin_std": 75.22626495361328,
      "step": 565
    },
    {
      "epoch": 0.8311306901615272,
      "fcm_dpo/beta": 0.012551356106996536,
      "fcm_dpo/delta": -0.08185572922229767,
      "fcm_dpo/margin": 73.64617919921875,
      "fcm_dpo/q_t": 0.31478655338287354,
      "grad_norm": 12.691439628601074,
      "learning_rate": 4.3028095264420525e-08,
      "logits/chosen": -1.8287672996520996,
      "logits/rejected": -1.3916640281677246,
      "logps/chosen": -235.4650421142578,
      "logps/ref_chosen": -98.97320556640625,
      "logps/ref_rejected": -110.82090759277344,
      "logps/rejected": -320.95892333984375,
      "loss": 0.8519,
      "margin_dpo/margin_mean": 73.64617156982422,
      "margin_dpo/margin_std": 77.48297119140625,
      "step": 566
    },
    {
      "epoch": 0.8325991189427313,
      "fcm_dpo/beta": 0.012515963986515999,
      "fcm_dpo/delta": 0.07364558428525925,
      "fcm_dpo/margin": 62.43628692626953,
      "fcm_dpo/q_t": 0.3408878445625305,
      "grad_norm": 15.552478790283203,
      "learning_rate": 4.231101308059165e-08,
      "logits/chosen": -2.209388256072998,
      "logits/rejected": -1.5480117797851562,
      "logps/chosen": -234.72930908203125,
      "logps/ref_chosen": -92.7035903930664,
      "logps/ref_rejected": -91.22431945800781,
      "logps/rejected": -295.6863098144531,
      "loss": 0.9282,
      "margin_dpo/margin_mean": 62.43628692626953,
      "margin_dpo/margin_std": 72.3102035522461,
      "step": 567
    },
    {
      "epoch": 0.8340675477239354,
      "fcm_dpo/beta": 0.012427978217601776,
      "fcm_dpo/delta": -0.09094738960266113,
      "fcm_dpo/margin": 74.95256805419922,
      "fcm_dpo/q_t": 0.3132409453392029,
      "grad_norm": 12.722140312194824,
      "learning_rate": 4.1599403694720145e-08,
      "logits/chosen": -1.535170316696167,
      "logits/rejected": -1.3898873329162598,
      "logps/chosen": -206.69818115234375,
      "logps/ref_chosen": -73.13226318359375,
      "logps/ref_rejected": -94.95265197753906,
      "logps/rejected": -303.47113037109375,
      "loss": 0.8189,
      "margin_dpo/margin_mean": 74.95256805419922,
      "margin_dpo/margin_std": 72.48225402832031,
      "step": 568
    },
    {
      "epoch": 0.8355359765051396,
      "fcm_dpo/beta": 0.012630300596356392,
      "fcm_dpo/delta": 0.0689060166478157,
      "fcm_dpo/margin": 62.19904327392578,
      "fcm_dpo/q_t": 0.3414260149002075,
      "grad_norm": 16.49064064025879,
      "learning_rate": 4.089328585837512e-08,
      "logits/chosen": -2.0152597427368164,
      "logits/rejected": -1.65643310546875,
      "logps/chosen": -234.0463409423828,
      "logps/ref_chosen": -89.21708679199219,
      "logps/ref_rejected": -86.41105651855469,
      "logps/rejected": -293.4393615722656,
      "loss": 0.9867,
      "margin_dpo/margin_mean": 62.19904327392578,
      "margin_dpo/margin_std": 83.06907653808594,
      "step": 569
    },
    {
      "epoch": 0.8370044052863436,
      "fcm_dpo/beta": 0.012341935187578201,
      "fcm_dpo/delta": -0.1500765085220337,
      "fcm_dpo/margin": 79.84074401855469,
      "fcm_dpo/q_t": 0.29707303643226624,
      "grad_norm": 12.651300430297852,
      "learning_rate": 4.019267817841834e-08,
      "logits/chosen": -2.3013365268707275,
      "logits/rejected": -1.6241872310638428,
      "logps/chosen": -225.6119384765625,
      "logps/ref_chosen": -94.76277160644531,
      "logps/ref_rejected": -88.846923828125,
      "logps/rejected": -299.5368347167969,
      "loss": 0.7799,
      "margin_dpo/margin_mean": 79.84074401855469,
      "margin_dpo/margin_std": 70.12899780273438,
      "step": 570
    },
    {
      "epoch": 0.8384728340675477,
      "fcm_dpo/beta": 0.012180352583527565,
      "fcm_dpo/delta": -0.05313785746693611,
      "fcm_dpo/margin": 73.75041961669922,
      "fcm_dpo/q_t": 0.32369863986968994,
      "grad_norm": 15.93890380859375,
      "learning_rate": 3.9497599116513705e-08,
      "logits/chosen": -1.891650676727295,
      "logits/rejected": -1.7370983362197876,
      "logps/chosen": -211.00099182128906,
      "logps/ref_chosen": -80.8212890625,
      "logps/ref_rejected": -97.01528930664062,
      "logps/rejected": -300.9454345703125,
      "loss": 0.8998,
      "margin_dpo/margin_mean": 73.75041198730469,
      "margin_dpo/margin_std": 85.5877914428711,
      "step": 571
    },
    {
      "epoch": 0.8399412628487518,
      "fcm_dpo/beta": 0.011995727196335793,
      "fcm_dpo/delta": -0.03884110599756241,
      "fcm_dpo/margin": 73.78263854980469,
      "fcm_dpo/q_t": 0.328910231590271,
      "grad_norm": 12.733126640319824,
      "learning_rate": 3.880806698864086e-08,
      "logits/chosen": -1.7663264274597168,
      "logits/rejected": -1.6016298532485962,
      "logps/chosen": -211.55044555664062,
      "logps/ref_chosen": -77.37992858886719,
      "logps/ref_rejected": -90.56597900390625,
      "logps/rejected": -298.5191650390625,
      "loss": 0.8762,
      "margin_dpo/margin_mean": 73.78263854980469,
      "margin_dpo/margin_std": 83.24708557128906,
      "step": 572
    },
    {
      "epoch": 0.8414096916299559,
      "fcm_dpo/beta": 0.01217666082084179,
      "fcm_dpo/delta": -0.011429935693740845,
      "fcm_dpo/margin": 70.3113021850586,
      "fcm_dpo/q_t": 0.32651984691619873,
      "grad_norm": 13.776387214660645,
      "learning_rate": 3.812409996461275e-08,
      "logits/chosen": -2.3168587684631348,
      "logits/rejected": -2.1058921813964844,
      "logps/chosen": -215.7606201171875,
      "logps/ref_chosen": -83.7060775756836,
      "logps/ref_rejected": -90.69746398925781,
      "logps/rejected": -293.06329345703125,
      "loss": 0.9059,
      "margin_dpo/margin_mean": 70.3113021850586,
      "margin_dpo/margin_std": 77.29948425292969,
      "step": 573
    },
    {
      "epoch": 0.8428781204111601,
      "fcm_dpo/beta": 0.012046435847878456,
      "fcm_dpo/delta": 0.0020178817212581635,
      "fcm_dpo/margin": 70.36782836914062,
      "fcm_dpo/q_t": 0.33117565512657166,
      "grad_norm": 14.58945369720459,
      "learning_rate": 3.74457160675965e-08,
      "logits/chosen": -2.1412198543548584,
      "logits/rejected": -1.8174076080322266,
      "logps/chosen": -223.34002685546875,
      "logps/ref_chosen": -85.12948608398438,
      "logps/ref_rejected": -99.05403137207031,
      "logps/rejected": -307.63238525390625,
      "loss": 0.9014,
      "margin_dpo/margin_mean": 70.36782836914062,
      "margin_dpo/margin_std": 80.83290100097656,
      "step": 574
    },
    {
      "epoch": 0.8443465491923642,
      "fcm_dpo/beta": 0.011899670585989952,
      "fcm_dpo/delta": -0.02533550187945366,
      "fcm_dpo/margin": 73.34465789794922,
      "fcm_dpo/q_t": 0.32771629095077515,
      "grad_norm": 17.919416427612305,
      "learning_rate": 3.677293317363864e-08,
      "logits/chosen": -1.301839828491211,
      "logits/rejected": -0.845508873462677,
      "logps/chosen": -246.7624969482422,
      "logps/ref_chosen": -105.77084350585938,
      "logps/ref_rejected": -100.94944763183594,
      "logps/rejected": -315.2857666015625,
      "loss": 0.9172,
      "margin_dpo/margin_mean": 73.34465789794922,
      "margin_dpo/margin_std": 88.70787048339844,
      "step": 575
    },
    {
      "epoch": 0.8458149779735683,
      "fcm_dpo/beta": 0.012019166722893715,
      "fcm_dpo/delta": 0.01799788698554039,
      "fcm_dpo/margin": 69.30175018310547,
      "fcm_dpo/q_t": 0.33523285388946533,
      "grad_norm": 13.958468437194824,
      "learning_rate": 3.6105769011194224e-08,
      "logits/chosen": -2.088580846786499,
      "logits/rejected": -2.000969886779785,
      "logps/chosen": -223.76405334472656,
      "logps/ref_chosen": -81.20452880859375,
      "logps/ref_rejected": -105.40940856933594,
      "logps/rejected": -317.2706604003906,
      "loss": 0.9374,
      "margin_dpo/margin_mean": 69.30174255371094,
      "margin_dpo/margin_std": 85.55188751220703,
      "step": 576
    },
    {
      "epoch": 0.8472834067547724,
      "fcm_dpo/beta": 0.011663603596389294,
      "fcm_dpo/delta": -0.13514071702957153,
      "fcm_dpo/margin": 83.21434783935547,
      "fcm_dpo/q_t": 0.30806174874305725,
      "grad_norm": 11.793753623962402,
      "learning_rate": 3.5444241160659304e-08,
      "logits/chosen": -2.0075855255126953,
      "logits/rejected": -1.3254092931747437,
      "logps/chosen": -228.4785919189453,
      "logps/ref_chosen": -101.46419525146484,
      "logps/ref_rejected": -89.32637023925781,
      "logps/rejected": -299.55511474609375,
      "loss": 0.8493,
      "margin_dpo/margin_mean": 83.21435546875,
      "margin_dpo/margin_std": 87.91677856445312,
      "step": 577
    },
    {
      "epoch": 0.8487518355359766,
      "fcm_dpo/beta": 0.011880055069923401,
      "fcm_dpo/delta": 0.11570513248443604,
      "fcm_dpo/margin": 62.5034294128418,
      "fcm_dpo/q_t": 0.34822767972946167,
      "grad_norm": 15.396873474121094,
      "learning_rate": 3.478836705390808e-08,
      "logits/chosen": -1.3165785074234009,
      "logits/rejected": -1.0804065465927124,
      "logps/chosen": -223.2236328125,
      "logps/ref_chosen": -77.29241943359375,
      "logps/ref_rejected": -88.24701690673828,
      "logps/rejected": -296.681640625,
      "loss": 0.9618,
      "margin_dpo/margin_mean": 62.50342559814453,
      "margin_dpo/margin_std": 78.88618469238281,
      "step": 578
    },
    {
      "epoch": 0.8502202643171806,
      "fcm_dpo/beta": 0.012005902826786041,
      "fcm_dpo/delta": 0.08513668179512024,
      "fcm_dpo/margin": 64.23765563964844,
      "fcm_dpo/q_t": 0.34554487466812134,
      "grad_norm": 13.368642807006836,
      "learning_rate": 3.41381639738331e-08,
      "logits/chosen": -1.716012954711914,
      "logits/rejected": -1.379181146621704,
      "logps/chosen": -222.61764526367188,
      "logps/ref_chosen": -87.34305572509766,
      "logps/ref_rejected": -102.12100982666016,
      "logps/rejected": -301.6332702636719,
      "loss": 0.936,
      "margin_dpo/margin_mean": 64.23765563964844,
      "margin_dpo/margin_std": 75.44595336914062,
      "step": 579
    },
    {
      "epoch": 0.8516886930983847,
      "fcm_dpo/beta": 0.011975444853305817,
      "fcm_dpo/delta": -0.028606295585632324,
      "fcm_dpo/margin": 73.02272033691406,
      "fcm_dpo/q_t": 0.32629266381263733,
      "grad_norm": 13.48125171661377,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -1.2470667362213135,
      "logits/rejected": -0.930492639541626,
      "logps/chosen": -220.02879333496094,
      "logps/ref_chosen": -88.92879486083984,
      "logps/ref_rejected": -88.69661712646484,
      "logps/rejected": -292.8193359375,
      "loss": 0.8775,
      "margin_dpo/margin_mean": 73.02272033691406,
      "margin_dpo/margin_std": 78.72087860107422,
      "step": 580
    },
    {
      "epoch": 0.8531571218795888,
      "fcm_dpo/beta": 0.012202093377709389,
      "fcm_dpo/delta": 0.066944919526577,
      "fcm_dpo/margin": 64.5920181274414,
      "fcm_dpo/q_t": 0.34423375129699707,
      "grad_norm": 15.27383041381836,
      "learning_rate": 3.285483927764726e-08,
      "logits/chosen": -2.02988338470459,
      "logits/rejected": -1.6924233436584473,
      "logps/chosen": -231.8140869140625,
      "logps/ref_chosen": -92.22323608398438,
      "logps/ref_rejected": -99.38943481445312,
      "logps/rejected": -303.57232666015625,
      "loss": 0.9915,
      "margin_dpo/margin_mean": 64.5920181274414,
      "margin_dpo/margin_std": 88.16921997070312,
      "step": 581
    },
    {
      "epoch": 0.8546255506607929,
      "fcm_dpo/beta": 0.012522203847765923,
      "fcm_dpo/delta": 0.06351852416992188,
      "fcm_dpo/margin": 62.802978515625,
      "fcm_dpo/q_t": 0.3435629606246948,
      "grad_norm": 13.179994583129883,
      "learning_rate": 3.222175147833556e-08,
      "logits/chosen": -2.2159390449523926,
      "logits/rejected": -2.1951913833618164,
      "logps/chosen": -209.8260040283203,
      "logps/ref_chosen": -87.33561706542969,
      "logps/ref_rejected": -115.1544189453125,
      "logps/rejected": -300.4477844238281,
      "loss": 0.9489,
      "margin_dpo/margin_mean": 62.802974700927734,
      "margin_dpo/margin_std": 77.04085540771484,
      "step": 582
    },
    {
      "epoch": 0.856093979441997,
      "fcm_dpo/beta": 0.01241613645106554,
      "fcm_dpo/delta": 0.048651568591594696,
      "fcm_dpo/margin": 64.84408569335938,
      "fcm_dpo/q_t": 0.34346115589141846,
      "grad_norm": 14.555140495300293,
      "learning_rate": 3.159440233840763e-08,
      "logits/chosen": -1.4759223461151123,
      "logits/rejected": -1.245713233947754,
      "logps/chosen": -229.76559448242188,
      "logps/ref_chosen": -83.71707153320312,
      "logps/ref_rejected": -94.58465576171875,
      "logps/rejected": -305.477294921875,
      "loss": 0.9223,
      "margin_dpo/margin_mean": 64.84408569335938,
      "margin_dpo/margin_std": 78.4787368774414,
      "step": 583
    },
    {
      "epoch": 0.8575624082232012,
      "fcm_dpo/beta": 0.012546549551188946,
      "fcm_dpo/delta": -0.057240843772888184,
      "fcm_dpo/margin": 71.72808837890625,
      "fcm_dpo/q_t": 0.3223274052143097,
      "grad_norm": 14.367459297180176,
      "learning_rate": 3.0972808389096635e-08,
      "logits/chosen": -1.9238438606262207,
      "logits/rejected": -1.4491448402404785,
      "logps/chosen": -223.58079528808594,
      "logps/ref_chosen": -94.25955963134766,
      "logps/ref_rejected": -104.48269653320312,
      "logps/rejected": -305.5320129394531,
      "loss": 0.8921,
      "margin_dpo/margin_mean": 71.72808837890625,
      "margin_dpo/margin_std": 80.05842590332031,
      "step": 584
    },
    {
      "epoch": 0.8590308370044053,
      "fcm_dpo/beta": 0.012124446220695972,
      "fcm_dpo/delta": -0.1827441304922104,
      "fcm_dpo/margin": 83.6611557006836,
      "fcm_dpo/q_t": 0.30458420515060425,
      "grad_norm": 12.921794891357422,
      "learning_rate": 3.035698600998121e-08,
      "logits/chosen": -1.9596983194351196,
      "logits/rejected": -1.6787914037704468,
      "logps/chosen": -222.28982543945312,
      "logps/ref_chosen": -85.63676452636719,
      "logps/ref_rejected": -90.28547668457031,
      "logps/rejected": -310.5997314453125,
      "loss": 0.8384,
      "margin_dpo/margin_mean": 83.6611557006836,
      "margin_dpo/margin_std": 88.93515014648438,
      "step": 585
    },
    {
      "epoch": 0.8604992657856094,
      "fcm_dpo/beta": 0.012192027643322945,
      "fcm_dpo/delta": 0.14729395508766174,
      "fcm_dpo/margin": 58.48194885253906,
      "fcm_dpo/q_t": 0.3539287745952606,
      "grad_norm": 13.725818634033203,
      "learning_rate": 2.974695142855388e-08,
      "logits/chosen": -1.8884037733078003,
      "logits/rejected": -1.6240513324737549,
      "logps/chosen": -231.93746948242188,
      "logps/ref_chosen": -83.68114471435547,
      "logps/ref_rejected": -98.35916137695312,
      "logps/rejected": -305.097412109375,
      "loss": 0.9459,
      "margin_dpo/margin_mean": 58.48194885253906,
      "margin_dpo/margin_std": 69.6341552734375,
      "step": 586
    },
    {
      "epoch": 0.8619676945668135,
      "fcm_dpo/beta": 0.012489515356719494,
      "fcm_dpo/delta": 0.07789243757724762,
      "fcm_dpo/margin": 62.21076583862305,
      "fcm_dpo/q_t": 0.3466240167617798,
      "grad_norm": 12.888617515563965,
      "learning_rate": 2.9142720719793122e-08,
      "logits/chosen": -2.3818912506103516,
      "logits/rejected": -2.0904064178466797,
      "logps/chosen": -201.2530975341797,
      "logps/ref_chosen": -77.57750701904297,
      "logps/ref_rejected": -89.32032775878906,
      "logps/rejected": -275.2066650390625,
      "loss": 0.9453,
      "margin_dpo/margin_mean": 62.21076202392578,
      "margin_dpo/margin_std": 77.28038024902344,
      "step": 587
    },
    {
      "epoch": 0.8634361233480177,
      "fcm_dpo/beta": 0.012591280043125153,
      "fcm_dpo/delta": 0.11278443038463593,
      "fcm_dpo/margin": 59.20628356933594,
      "fcm_dpo/q_t": 0.35483455657958984,
      "grad_norm": 14.562524795532227,
      "learning_rate": 2.8544309805740018e-08,
      "logits/chosen": -1.7069602012634277,
      "logits/rejected": -1.6397366523742676,
      "logps/chosen": -215.88272094726562,
      "logps/ref_chosen": -83.80426025390625,
      "logps/ref_rejected": -113.11943054199219,
      "logps/rejected": -304.4042053222656,
      "loss": 1.0127,
      "margin_dpo/margin_mean": 59.20628356933594,
      "margin_dpo/margin_std": 84.50772857666016,
      "step": 588
    },
    {
      "epoch": 0.8649045521292217,
      "fcm_dpo/beta": 0.01244035642594099,
      "fcm_dpo/delta": -0.14263486862182617,
      "fcm_dpo/margin": 78.6182861328125,
      "fcm_dpo/q_t": 0.2986539602279663,
      "grad_norm": 15.344147682189941,
      "learning_rate": 2.7951734455078786e-08,
      "logits/chosen": -1.6551861763000488,
      "logits/rejected": -1.247209072113037,
      "logps/chosen": -235.73348999023438,
      "logps/ref_chosen": -96.46501922607422,
      "logps/ref_rejected": -116.03719329833984,
      "logps/rejected": -333.9239501953125,
      "loss": 0.7867,
      "margin_dpo/margin_mean": 78.6182861328125,
      "margin_dpo/margin_std": 69.69014739990234,
      "step": 589
    },
    {
      "epoch": 0.8663729809104258,
      "fcm_dpo/beta": 0.012330231256783009,
      "fcm_dpo/delta": -0.08336605131626129,
      "fcm_dpo/margin": 75.031005859375,
      "fcm_dpo/q_t": 0.3123021721839905,
      "grad_norm": 12.972137451171875,
      "learning_rate": 2.736501028272095e-08,
      "logits/chosen": -1.9837596416473389,
      "logits/rejected": -1.8040398359298706,
      "logps/chosen": -217.9958038330078,
      "logps/ref_chosen": -88.4278564453125,
      "logps/ref_rejected": -110.19876861572266,
      "logps/rejected": -314.7977294921875,
      "loss": 0.8127,
      "margin_dpo/margin_mean": 75.031005859375,
      "margin_dpo/margin_std": 70.29672241210938,
      "step": 590
    },
    {
      "epoch": 0.8678414096916299,
      "fcm_dpo/beta": 0.011928400956094265,
      "fcm_dpo/delta": -0.12863200902938843,
      "fcm_dpo/margin": 80.97084045410156,
      "fcm_dpo/q_t": 0.30411410331726074,
      "grad_norm": 11.441441535949707,
      "learning_rate": 2.678415274939408e-08,
      "logits/chosen": -2.0621907711029053,
      "logits/rejected": -1.4476604461669922,
      "logps/chosen": -214.27345275878906,
      "logps/ref_chosen": -87.84968566894531,
      "logps/ref_rejected": -89.57919311523438,
      "logps/rejected": -296.97381591796875,
      "loss": 0.7947,
      "margin_dpo/margin_mean": 80.97084045410156,
      "margin_dpo/margin_std": 74.34172058105469,
      "step": 591
    },
    {
      "epoch": 0.869309838472834,
      "fcm_dpo/beta": 0.012076403945684433,
      "fcm_dpo/delta": 0.10909491032361984,
      "fcm_dpo/margin": 62.0214958190918,
      "fcm_dpo/q_t": 0.34997737407684326,
      "grad_norm": 14.482314109802246,
      "learning_rate": 2.6209177161234442e-08,
      "logits/chosen": -1.8271369934082031,
      "logits/rejected": -1.5044286251068115,
      "logps/chosen": -223.06594848632812,
      "logps/ref_chosen": -74.97573852539062,
      "logps/ref_rejected": -81.55262756347656,
      "logps/rejected": -291.66436767578125,
      "loss": 0.9767,
      "margin_dpo/margin_mean": 62.02149963378906,
      "margin_dpo/margin_std": 82.1724853515625,
      "step": 592
    },
    {
      "epoch": 0.8707782672540382,
      "fcm_dpo/beta": 0.012362951412796974,
      "fcm_dpo/delta": 0.17887909710407257,
      "fcm_dpo/margin": 55.27935791015625,
      "fcm_dpo/q_t": 0.36694955825805664,
      "grad_norm": 19.68903160095215,
      "learning_rate": 2.564009866938349e-08,
      "logits/chosen": -0.6591614484786987,
      "logits/rejected": -0.34581655263900757,
      "logps/chosen": -211.53347778320312,
      "logps/ref_chosen": -72.23902893066406,
      "logps/ref_rejected": -70.09652709960938,
      "logps/rejected": -264.67034912109375,
      "loss": 1.0278,
      "margin_dpo/margin_mean": 55.27935791015625,
      "margin_dpo/margin_std": 81.17341613769531,
      "step": 593
    },
    {
      "epoch": 0.8722466960352423,
      "fcm_dpo/beta": 0.0123524721711874,
      "fcm_dpo/delta": -0.1632692813873291,
      "fcm_dpo/margin": 80.72880554199219,
      "fcm_dpo/q_t": 0.2972342073917389,
      "grad_norm": 11.561514854431152,
      "learning_rate": 2.5076932269588708e-08,
      "logits/chosen": -1.9211974143981934,
      "logits/rejected": -1.4067270755767822,
      "logps/chosen": -216.4187774658203,
      "logps/ref_chosen": -90.43771362304688,
      "logps/ref_rejected": -92.267578125,
      "logps/rejected": -298.9774475097656,
      "loss": 0.7671,
      "margin_dpo/margin_mean": 80.72880554199219,
      "margin_dpo/margin_std": 69.18890380859375,
      "step": 594
    },
    {
      "epoch": 0.8737151248164464,
      "fcm_dpo/beta": 0.012263715267181396,
      "fcm_dpo/delta": 0.021671980619430542,
      "fcm_dpo/margin": 67.65665435791016,
      "fcm_dpo/q_t": 0.3382018506526947,
      "grad_norm": 14.33916187286377,
      "learning_rate": 2.451969280180849e-08,
      "logits/chosen": -1.8135511875152588,
      "logits/rejected": -1.3942694664001465,
      "logps/chosen": -212.7932891845703,
      "logps/ref_chosen": -81.51480102539062,
      "logps/ref_rejected": -86.52047729492188,
      "logps/rejected": -285.45562744140625,
      "loss": 0.9179,
      "margin_dpo/margin_mean": 67.65666198730469,
      "margin_dpo/margin_std": 80.72854614257812,
      "step": 595
    },
    {
      "epoch": 0.8751835535976505,
      "fcm_dpo/beta": 0.012409276328980923,
      "fcm_dpo/delta": 0.08794566988945007,
      "fcm_dpo/margin": 61.938507080078125,
      "fcm_dpo/q_t": 0.3473287522792816,
      "grad_norm": 16.3719425201416,
      "learning_rate": 2.396839494982103e-08,
      "logits/chosen": -1.658387541770935,
      "logits/rejected": -1.0812584161758423,
      "logps/chosen": -223.06051635742188,
      "logps/ref_chosen": -88.20805358886719,
      "logps/ref_rejected": -86.09375,
      "logps/rejected": -282.88470458984375,
      "loss": 0.9722,
      "margin_dpo/margin_mean": 61.938507080078125,
      "margin_dpo/margin_std": 80.60945129394531,
      "step": 596
    },
    {
      "epoch": 0.8766519823788547,
      "fcm_dpo/beta": 0.012180918827652931,
      "fcm_dpo/delta": -0.11607787013053894,
      "fcm_dpo/margin": 78.33621215820312,
      "fcm_dpo/q_t": 0.31094270944595337,
      "grad_norm": 11.810127258300781,
      "learning_rate": 2.3423053240837514e-08,
      "logits/chosen": -1.5121232271194458,
      "logits/rejected": -1.3904181718826294,
      "logps/chosen": -216.66140747070312,
      "logps/ref_chosen": -83.58998107910156,
      "logps/ref_rejected": -106.94439697265625,
      "logps/rejected": -318.35205078125,
      "loss": 0.8322,
      "margin_dpo/margin_mean": 78.33621215820312,
      "margin_dpo/margin_std": 80.11341857910156,
      "step": 597
    },
    {
      "epoch": 0.8781204111600588,
      "fcm_dpo/beta": 0.012269245460629463,
      "fcm_dpo/delta": 0.11151318997144699,
      "fcm_dpo/margin": 60.842613220214844,
      "fcm_dpo/q_t": 0.34948280453681946,
      "grad_norm": 13.330480575561523,
      "learning_rate": 2.2883682045119062e-08,
      "logits/chosen": -2.012774705886841,
      "logits/rejected": -1.704032301902771,
      "logps/chosen": -219.08596801757812,
      "logps/ref_chosen": -90.18262481689453,
      "logps/ref_rejected": -94.79856872558594,
      "logps/rejected": -284.5445251464844,
      "loss": 0.9748,
      "margin_dpo/margin_mean": 60.842613220214844,
      "margin_dpo/margin_std": 78.31558227539062,
      "step": 598
    },
    {
      "epoch": 0.8795888399412628,
      "fcm_dpo/beta": 0.01254078559577465,
      "fcm_dpo/delta": 0.1279219090938568,
      "fcm_dpo/margin": 58.25449752807617,
      "fcm_dpo/q_t": 0.35618844628334045,
      "grad_norm": 16.062721252441406,
      "learning_rate": 2.2350295575598367e-08,
      "logits/chosen": -1.863084316253662,
      "logits/rejected": -1.7345049381256104,
      "logps/chosen": -209.37547302246094,
      "logps/ref_chosen": -77.84226989746094,
      "logps/ref_rejected": -89.1976318359375,
      "logps/rejected": -278.9853210449219,
      "loss": 0.9903,
      "margin_dpo/margin_mean": 58.25449752807617,
      "margin_dpo/margin_std": 76.904541015625,
      "step": 599
    },
    {
      "epoch": 0.8810572687224669,
      "fcm_dpo/beta": 0.012659793719649315,
      "fcm_dpo/delta": -0.05752100050449371,
      "fcm_dpo/margin": 71.27790832519531,
      "fcm_dpo/q_t": 0.3214208483695984,
      "grad_norm": 12.589545249938965,
      "learning_rate": 2.1822907887504932e-08,
      "logits/chosen": -2.3141098022460938,
      "logits/rejected": -1.9681317806243896,
      "logps/chosen": -229.8334197998047,
      "logps/ref_chosen": -92.18781280517578,
      "logps/ref_rejected": -92.95429992675781,
      "logps/rejected": -301.8778076171875,
      "loss": 0.8671,
      "margin_dpo/margin_mean": 71.27790832519531,
      "margin_dpo/margin_std": 76.77690887451172,
      "step": 600
    },
    {
      "epoch": 0.8810572687224669,
      "eval_fcm_dpo/beta": 0.012609176337718964,
      "eval_logits/chosen": -1.6990704536437988,
      "eval_logits/rejected": -1.3195072412490845,
      "eval_logps/chosen": -239.63792419433594,
      "eval_logps/ref_chosen": -100.49356842041016,
      "eval_logps/ref_rejected": -94.06775665283203,
      "eval_logps/rejected": -292.5469055175781,
      "eval_loss": 0.48429349064826965,
      "eval_margin_dpo/margin_mean": 59.33479309082031,
      "eval_margin_dpo/margin_std": 75.8063735961914,
      "eval_runtime": 43.7607,
      "eval_samples_per_second": 53.45,
      "eval_steps_per_second": 1.691,
      "step": 600
    },
    {
      "epoch": 0.882525697503671,
      "fcm_dpo/beta": 0.012439209967851639,
      "fcm_dpo/delta": -0.04578261449933052,
      "fcm_dpo/margin": 71.59075927734375,
      "fcm_dpo/q_t": 0.3206157386302948,
      "grad_norm": 13.716604232788086,
      "learning_rate": 2.1301532877994742e-08,
      "logits/chosen": -1.623553991317749,
      "logits/rejected": -1.2954704761505127,
      "logps/chosen": -231.75643920898438,
      "logps/ref_chosen": -89.21614074707031,
      "logps/ref_rejected": -100.17054748535156,
      "logps/rejected": -314.3016052246094,
      "loss": 0.8497,
      "margin_dpo/margin_mean": 71.59075927734375,
      "margin_dpo/margin_std": 73.1295394897461,
      "step": 601
    },
    {
      "epoch": 0.8839941262848752,
      "fcm_dpo/beta": 0.012094026431441307,
      "fcm_dpo/delta": -0.21615482866764069,
      "fcm_dpo/margin": 86.27711486816406,
      "fcm_dpo/q_t": 0.29506054520606995,
      "grad_norm": 12.191177368164062,
      "learning_rate": 2.0786184285784298e-08,
      "logits/chosen": -2.171415090560913,
      "logits/rejected": -1.9284133911132812,
      "logps/chosen": -195.90830993652344,
      "logps/ref_chosen": -80.05760192871094,
      "logps/ref_rejected": -93.197509765625,
      "logps/rejected": -295.3253173828125,
      "loss": 0.793,
      "margin_dpo/margin_mean": 86.27711486816406,
      "margin_dpo/margin_std": 84.91157531738281,
      "step": 602
    },
    {
      "epoch": 0.8854625550660793,
      "fcm_dpo/beta": 0.011891430243849754,
      "fcm_dpo/delta": -0.06066790223121643,
      "fcm_dpo/margin": 76.12080383300781,
      "fcm_dpo/q_t": 0.31821417808532715,
      "grad_norm": 13.448278427124023,
      "learning_rate": 2.0276875690788204e-08,
      "logits/chosen": -1.967059850692749,
      "logits/rejected": -1.3754029273986816,
      "logps/chosen": -224.9002227783203,
      "logps/ref_chosen": -102.30958557128906,
      "logps/ref_rejected": -108.06884765625,
      "logps/rejected": -306.7803039550781,
      "loss": 0.847,
      "margin_dpo/margin_mean": 76.12080383300781,
      "margin_dpo/margin_std": 80.23945617675781,
      "step": 603
    },
    {
      "epoch": 0.8869309838472834,
      "fcm_dpo/beta": 0.011719077825546265,
      "fcm_dpo/delta": -0.028855059295892715,
      "fcm_dpo/margin": 74.73594665527344,
      "fcm_dpo/q_t": 0.31438958644866943,
      "grad_norm": 11.16280460357666,
      "learning_rate": 1.977362051376158e-08,
      "logits/chosen": -1.7443255186080933,
      "logits/rejected": -1.5849244594573975,
      "logps/chosen": -206.21484375,
      "logps/ref_chosen": -78.17408752441406,
      "logps/ref_rejected": -99.49610900878906,
      "logps/rejected": -302.2728271484375,
      "loss": 0.8316,
      "margin_dpo/margin_mean": 74.73595428466797,
      "margin_dpo/margin_std": 68.67886352539062,
      "step": 604
    },
    {
      "epoch": 0.8883994126284875,
      "fcm_dpo/beta": 0.01191855501383543,
      "fcm_dpo/delta": 0.10982121527194977,
      "fcm_dpo/margin": 62.803863525390625,
      "fcm_dpo/q_t": 0.3468341827392578,
      "grad_norm": 12.669288635253906,
      "learning_rate": 1.9276432015946446e-08,
      "logits/chosen": -1.9019720554351807,
      "logits/rejected": -1.4060518741607666,
      "logps/chosen": -232.10272216796875,
      "logps/ref_chosen": -94.77332305908203,
      "logps/ref_rejected": -107.30490112304688,
      "logps/rejected": -307.43817138671875,
      "loss": 0.9308,
      "margin_dpo/margin_mean": 62.80385971069336,
      "margin_dpo/margin_std": 72.63813781738281,
      "step": 605
    },
    {
      "epoch": 0.8898678414096917,
      "fcm_dpo/beta": 0.01211099699139595,
      "fcm_dpo/delta": -0.012885570526123047,
      "fcm_dpo/margin": 71.01239013671875,
      "fcm_dpo/q_t": 0.3213347792625427,
      "grad_norm": 13.087188720703125,
      "learning_rate": 1.8785323298722093e-08,
      "logits/chosen": -2.011061191558838,
      "logits/rejected": -1.5981237888336182,
      "logps/chosen": -220.9056396484375,
      "logps/ref_chosen": -87.7533950805664,
      "logps/ref_rejected": -104.2422103881836,
      "logps/rejected": -308.4068603515625,
      "loss": 0.8646,
      "margin_dpo/margin_mean": 71.01239013671875,
      "margin_dpo/margin_std": 71.47808837890625,
      "step": 606
    },
    {
      "epoch": 0.8913362701908958,
      "fcm_dpo/beta": 0.011857230216264725,
      "fcm_dpo/delta": -0.05570369213819504,
      "fcm_dpo/margin": 75.94203186035156,
      "fcm_dpo/q_t": 0.3244081139564514,
      "grad_norm": 13.238591194152832,
      "learning_rate": 1.8300307303259904e-08,
      "logits/chosen": -1.8522627353668213,
      "logits/rejected": -1.3280799388885498,
      "logps/chosen": -220.33203125,
      "logps/ref_chosen": -88.32904052734375,
      "logps/ref_rejected": -86.76810455322266,
      "logps/rejected": -294.713134765625,
      "loss": 0.9333,
      "margin_dpo/margin_mean": 75.94203186035156,
      "margin_dpo/margin_std": 94.53721618652344,
      "step": 607
    },
    {
      "epoch": 0.8928046989720999,
      "fcm_dpo/beta": 0.011611446738243103,
      "fcm_dpo/delta": -0.07125182449817657,
      "fcm_dpo/margin": 78.56434631347656,
      "fcm_dpo/q_t": 0.3104247748851776,
      "grad_norm": 14.735077857971191,
      "learning_rate": 1.7821396810182437e-08,
      "logits/chosen": -1.979656457901001,
      "logits/rejected": -1.6973553895950317,
      "logps/chosen": -205.81634521484375,
      "logps/ref_chosen": -85.76937866210938,
      "logps/ref_rejected": -100.23281860351562,
      "logps/rejected": -298.8441162109375,
      "loss": 0.8636,
      "margin_dpo/margin_mean": 78.56434631347656,
      "margin_dpo/margin_std": 80.66757202148438,
      "step": 608
    },
    {
      "epoch": 0.8942731277533039,
      "fcm_dpo/beta": 0.011500250548124313,
      "fcm_dpo/delta": -0.07149754464626312,
      "fcm_dpo/margin": 79.48577117919922,
      "fcm_dpo/q_t": 0.3168375492095947,
      "grad_norm": 13.526650428771973,
      "learning_rate": 1.7348604439226617e-08,
      "logits/chosen": -2.310915946960449,
      "logits/rejected": -1.89617919921875,
      "logps/chosen": -219.12498474121094,
      "logps/ref_chosen": -92.9665756225586,
      "logps/ref_rejected": -95.91818237304688,
      "logps/rejected": -301.5623474121094,
      "loss": 0.845,
      "margin_dpo/margin_mean": 79.48577117919922,
      "margin_dpo/margin_std": 80.12174224853516,
      "step": 609
    },
    {
      "epoch": 0.895741556534508,
      "fcm_dpo/beta": 0.011400583200156689,
      "fcm_dpo/delta": -0.05915520340204239,
      "fcm_dpo/margin": 79.26727294921875,
      "fcm_dpo/q_t": 0.3089755177497864,
      "grad_norm": 11.487713813781738,
      "learning_rate": 1.6881942648911074e-08,
      "logits/chosen": -1.8207099437713623,
      "logits/rejected": -1.2872872352600098,
      "logps/chosen": -221.53671264648438,
      "logps/ref_chosen": -94.70028686523438,
      "logps/ref_rejected": -89.68739318847656,
      "logps/rejected": -295.7911071777344,
      "loss": 0.8194,
      "margin_dpo/margin_mean": 79.26727294921875,
      "margin_dpo/margin_std": 72.37725830078125,
      "step": 610
    },
    {
      "epoch": 0.8972099853157122,
      "fcm_dpo/beta": 0.011329730972647667,
      "fcm_dpo/delta": -0.07637893408536911,
      "fcm_dpo/margin": 81.11207580566406,
      "fcm_dpo/q_t": 0.3176420331001282,
      "grad_norm": 12.095680236816406,
      "learning_rate": 1.6421423736208e-08,
      "logits/chosen": -1.9744651317596436,
      "logits/rejected": -1.6172444820404053,
      "logps/chosen": -213.77818298339844,
      "logps/ref_chosen": -86.78334045410156,
      "logps/ref_rejected": -89.84307861328125,
      "logps/rejected": -297.95001220703125,
      "loss": 0.8324,
      "margin_dpo/margin_mean": 81.11207580566406,
      "margin_dpo/margin_std": 81.14633178710938,
      "step": 611
    },
    {
      "epoch": 0.8986784140969163,
      "fcm_dpo/beta": 0.011108027771115303,
      "fcm_dpo/delta": -0.03341337665915489,
      "fcm_dpo/margin": 79.22400665283203,
      "fcm_dpo/q_t": 0.3166874051094055,
      "grad_norm": 12.299027442932129,
      "learning_rate": 1.5967059836219042e-08,
      "logits/chosen": -2.1468470096588135,
      "logits/rejected": -1.3182777166366577,
      "logps/chosen": -237.84152221679688,
      "logps/ref_chosen": -101.02015686035156,
      "logps/ref_rejected": -93.78302764892578,
      "logps/rejected": -309.82843017578125,
      "loss": 0.8232,
      "margin_dpo/margin_mean": 79.22400665283203,
      "margin_dpo/margin_std": 73.36766052246094,
      "step": 612
    },
    {
      "epoch": 0.9001468428781204,
      "fcm_dpo/beta": 0.011010472662746906,
      "fcm_dpo/delta": -0.06952960789203644,
      "fcm_dpo/margin": 82.93061828613281,
      "fcm_dpo/q_t": 0.3106223940849304,
      "grad_norm": 12.303360939025879,
      "learning_rate": 1.551886292185553e-08,
      "logits/chosen": -1.7026329040527344,
      "logits/rejected": -1.4959499835968018,
      "logps/chosen": -206.756591796875,
      "logps/ref_chosen": -88.98861694335938,
      "logps/ref_rejected": -109.99551391601562,
      "logps/rejected": -310.694091796875,
      "loss": 0.7956,
      "margin_dpo/margin_mean": 82.93061828613281,
      "margin_dpo/margin_std": 72.88465881347656,
      "step": 613
    },
    {
      "epoch": 0.9016152716593245,
      "fcm_dpo/beta": 0.011123564094305038,
      "fcm_dpo/delta": 0.09568654745817184,
      "fcm_dpo/margin": 68.47465515136719,
      "fcm_dpo/q_t": 0.3426240086555481,
      "grad_norm": 14.612199783325195,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": -1.474619746208191,
      "logits/rejected": -1.4545549154281616,
      "logps/chosen": -220.43658447265625,
      "logps/ref_chosen": -80.20005798339844,
      "logps/ref_rejected": -109.86239624023438,
      "logps/rejected": -318.57354736328125,
      "loss": 0.9322,
      "margin_dpo/margin_mean": 68.47465515136719,
      "margin_dpo/margin_std": 78.6995849609375,
      "step": 614
    },
    {
      "epoch": 0.9030837004405287,
      "fcm_dpo/beta": 0.01152871549129486,
      "fcm_dpo/delta": 0.1148013100028038,
      "fcm_dpo/margin": 64.1284408569336,
      "fcm_dpo/q_t": 0.3421385884284973,
      "grad_norm": 12.715716361999512,
      "learning_rate": 1.4641017128809801e-08,
      "logits/chosen": -1.756435751914978,
      "logits/rejected": -1.351597785949707,
      "logps/chosen": -231.73660278320312,
      "logps/ref_chosen": -100.43525695800781,
      "logps/ref_rejected": -101.1800537109375,
      "logps/rejected": -296.60986328125,
      "loss": 0.9135,
      "margin_dpo/margin_mean": 64.1284408569336,
      "margin_dpo/margin_std": 64.88854217529297,
      "step": 615
    },
    {
      "epoch": 0.9045521292217328,
      "fcm_dpo/beta": 0.01139888633042574,
      "fcm_dpo/delta": -0.07935352623462677,
      "fcm_dpo/margin": 80.85199737548828,
      "fcm_dpo/q_t": 0.31489297747612,
      "grad_norm": 11.985855102539062,
      "learning_rate": 1.4211391382180637e-08,
      "logits/chosen": -1.806459665298462,
      "logits/rejected": -1.322357177734375,
      "logps/chosen": -227.93759155273438,
      "logps/ref_chosen": -92.49292755126953,
      "logps/ref_rejected": -82.06065368652344,
      "logps/rejected": -298.3572998046875,
      "loss": 0.8462,
      "margin_dpo/margin_mean": 80.85198974609375,
      "margin_dpo/margin_std": 83.83363342285156,
      "step": 616
    },
    {
      "epoch": 0.9060205580029369,
      "fcm_dpo/beta": 0.011469890363514423,
      "fcm_dpo/delta": 0.11235801875591278,
      "fcm_dpo/margin": 65.0345230102539,
      "fcm_dpo/q_t": 0.3446882367134094,
      "grad_norm": 13.72085952758789,
      "learning_rate": 1.378797888467345e-08,
      "logits/chosen": -1.5556066036224365,
      "logits/rejected": -0.915245771408081,
      "logps/chosen": -228.89675903320312,
      "logps/ref_chosen": -91.09699249267578,
      "logps/ref_rejected": -70.41004943847656,
      "logps/rejected": -273.24432373046875,
      "loss": 0.9388,
      "margin_dpo/margin_mean": 65.0345230102539,
      "margin_dpo/margin_std": 76.74852752685547,
      "step": 617
    },
    {
      "epoch": 0.9074889867841409,
      "fcm_dpo/beta": 0.011595992371439934,
      "fcm_dpo/delta": 0.06170836463570595,
      "fcm_dpo/margin": 68.39972686767578,
      "fcm_dpo/q_t": 0.33479219675064087,
      "grad_norm": 15.96437931060791,
      "learning_rate": 1.3370790793601371e-08,
      "logits/chosen": -2.159393310546875,
      "logits/rejected": -1.8255435228347778,
      "logps/chosen": -240.608642578125,
      "logps/ref_chosen": -102.02059936523438,
      "logps/ref_rejected": -99.80119323730469,
      "logps/rejected": -306.7889709472656,
      "loss": 0.931,
      "margin_dpo/margin_mean": 68.39972686767578,
      "margin_dpo/margin_std": 80.0145263671875,
      "step": 618
    },
    {
      "epoch": 0.908957415565345,
      "fcm_dpo/beta": 0.01181875355541706,
      "fcm_dpo/delta": 0.07929763197898865,
      "fcm_dpo/margin": 65.72555541992188,
      "fcm_dpo/q_t": 0.34781360626220703,
      "grad_norm": 12.926453590393066,
      "learning_rate": 1.2959838102258535e-08,
      "logits/chosen": -1.6464307308197021,
      "logits/rejected": -1.2737534046173096,
      "logps/chosen": -222.141845703125,
      "logps/ref_chosen": -89.74136352539062,
      "logps/ref_rejected": -99.90138244628906,
      "logps/rejected": -298.02740478515625,
      "loss": 0.9538,
      "margin_dpo/margin_mean": 65.72555541992188,
      "margin_dpo/margin_std": 84.30938720703125,
      "step": 619
    },
    {
      "epoch": 0.9104258443465492,
      "fcm_dpo/beta": 0.01165106613188982,
      "fcm_dpo/delta": -0.0010028481483459473,
      "fcm_dpo/margin": 72.52044677734375,
      "fcm_dpo/q_t": 0.3334208130836487,
      "grad_norm": 13.063641548156738,
      "learning_rate": 1.2555131639630567e-08,
      "logits/chosen": -2.2908899784088135,
      "logits/rejected": -1.7881639003753662,
      "logps/chosen": -214.04869079589844,
      "logps/ref_chosen": -85.12431335449219,
      "logps/ref_rejected": -85.41253662109375,
      "logps/rejected": -286.85736083984375,
      "loss": 0.9281,
      "margin_dpo/margin_mean": 72.52044677734375,
      "margin_dpo/margin_std": 82.46279907226562,
      "step": 620
    },
    {
      "epoch": 0.9118942731277533,
      "fcm_dpo/beta": 0.012033706530928612,
      "fcm_dpo/delta": 0.00012670457363128662,
      "fcm_dpo/margin": 70.27881622314453,
      "fcm_dpo/q_t": 0.3351132273674011,
      "grad_norm": 13.528855323791504,
      "learning_rate": 1.2156682070109086e-08,
      "logits/chosen": -1.9692211151123047,
      "logits/rejected": -1.7110702991485596,
      "logps/chosen": -219.129638671875,
      "logps/ref_chosen": -89.24842071533203,
      "logps/ref_rejected": -95.46463775634766,
      "logps/rejected": -295.6246643066406,
      "loss": 0.9452,
      "margin_dpo/margin_mean": 70.27881622314453,
      "margin_dpo/margin_std": 86.90354919433594,
      "step": 621
    },
    {
      "epoch": 0.9133627019089574,
      "fcm_dpo/beta": 0.011807737872004509,
      "fcm_dpo/delta": -0.022110551595687866,
      "fcm_dpo/margin": 73.69373321533203,
      "fcm_dpo/q_t": 0.32350027561187744,
      "grad_norm": 15.262984275817871,
      "learning_rate": 1.1764499893210878e-08,
      "logits/chosen": -1.2746450901031494,
      "logits/rejected": -0.5793788433074951,
      "logps/chosen": -229.22116088867188,
      "logps/ref_chosen": -99.79412841796875,
      "logps/ref_rejected": -90.82821655273438,
      "logps/rejected": -293.948974609375,
      "loss": 0.8526,
      "margin_dpo/margin_mean": 73.6937255859375,
      "margin_dpo/margin_std": 74.19442749023438,
      "step": 622
    },
    {
      "epoch": 0.9148311306901615,
      "fcm_dpo/beta": 0.012104466557502747,
      "fcm_dpo/delta": 0.20468077063560486,
      "fcm_dpo/margin": 54.466007232666016,
      "fcm_dpo/q_t": 0.37531763315200806,
      "grad_norm": 16.303054809570312,
      "learning_rate": 1.1378595443300998e-08,
      "logits/chosen": -2.4043192863464355,
      "logits/rejected": -1.9884238243103027,
      "logps/chosen": -225.4462432861328,
      "logps/ref_chosen": -90.45555114746094,
      "logps/ref_rejected": -91.32276153564453,
      "logps/rejected": -280.7794494628906,
      "loss": 1.0759,
      "margin_dpo/margin_mean": 54.466007232666016,
      "margin_dpo/margin_std": 88.74128723144531,
      "step": 623
    },
    {
      "epoch": 0.9162995594713657,
      "fcm_dpo/beta": 0.01205630972981453,
      "fcm_dpo/delta": -0.1360035389661789,
      "fcm_dpo/margin": 80.68819427490234,
      "fcm_dpo/q_t": 0.3046037554740906,
      "grad_norm": 13.448029518127441,
      "learning_rate": 1.0998978889320582e-08,
      "logits/chosen": -2.6992523670196533,
      "logits/rejected": -1.788914442062378,
      "logps/chosen": -244.6612548828125,
      "logps/ref_chosen": -109.87522888183594,
      "logps/ref_rejected": -104.77320861816406,
      "logps/rejected": -320.2474365234375,
      "loss": 0.8112,
      "margin_dpo/margin_mean": 80.68818664550781,
      "margin_dpo/margin_std": 78.43266296386719,
      "step": 624
    },
    {
      "epoch": 0.9177679882525698,
      "fcm_dpo/beta": 0.012182200327515602,
      "fcm_dpo/delta": 0.10253170877695084,
      "fcm_dpo/margin": 61.96144104003906,
      "fcm_dpo/q_t": 0.3474324345588684,
      "grad_norm": 13.418804168701172,
      "learning_rate": 1.0625660234518913e-08,
      "logits/chosen": -1.6345865726470947,
      "logits/rejected": -1.2501837015151978,
      "logps/chosen": -230.34454345703125,
      "logps/ref_chosen": -87.16815948486328,
      "logps/ref_rejected": -91.86148071289062,
      "logps/rejected": -296.99932861328125,
      "loss": 0.9269,
      "margin_dpo/margin_mean": 61.96144104003906,
      "margin_dpo/margin_std": 72.25453186035156,
      "step": 625
    },
    {
      "epoch": 0.9192364170337739,
      "fcm_dpo/beta": 0.012475828640162945,
      "fcm_dpo/delta": 0.10793514549732208,
      "fcm_dpo/margin": 60.04652404785156,
      "fcm_dpo/q_t": 0.3585766553878784,
      "grad_norm": 15.548441886901855,
      "learning_rate": 1.0258649316189721e-08,
      "logits/chosen": -1.4249701499938965,
      "logits/rejected": -0.9961116313934326,
      "logps/chosen": -241.67068481445312,
      "logps/ref_chosen": -104.22421264648438,
      "logps/ref_rejected": -104.1774673461914,
      "logps/rejected": -301.67047119140625,
      "loss": 1.0141,
      "margin_dpo/margin_mean": 60.04652404785156,
      "margin_dpo/margin_std": 88.77542877197266,
      "step": 626
    },
    {
      "epoch": 0.920704845814978,
      "fcm_dpo/beta": 0.012376347556710243,
      "fcm_dpo/delta": -0.04472092539072037,
      "fcm_dpo/margin": 71.95941162109375,
      "fcm_dpo/q_t": 0.32779771089553833,
      "grad_norm": 11.651665687561035,
      "learning_rate": 9.897955805412e-09,
      "logits/chosen": -1.5501372814178467,
      "logits/rejected": -1.6814649105072021,
      "logps/chosen": -200.94879150390625,
      "logps/ref_chosen": -74.93461608886719,
      "logps/ref_rejected": -112.57289123535156,
      "logps/rejected": -310.5464782714844,
      "loss": 0.9017,
      "margin_dpo/margin_mean": 71.95941162109375,
      "margin_dpo/margin_std": 85.76150512695312,
      "step": 627
    },
    {
      "epoch": 0.922173274596182,
      "fcm_dpo/beta": 0.012193206697702408,
      "fcm_dpo/delta": -0.09546506404876709,
      "fcm_dpo/margin": 76.80429077148438,
      "fcm_dpo/q_t": 0.308502733707428,
      "grad_norm": 14.485041618347168,
      "learning_rate": 9.543589206795238e-09,
      "logits/chosen": -2.234900712966919,
      "logits/rejected": -1.9763065576553345,
      "logps/chosen": -223.16265869140625,
      "logps/ref_chosen": -93.69107818603516,
      "logps/ref_rejected": -107.34395599365234,
      "logps/rejected": -313.61981201171875,
      "loss": 0.8052,
      "margin_dpo/margin_mean": 76.80429077148438,
      "margin_dpo/margin_std": 70.0140609741211,
      "step": 628
    },
    {
      "epoch": 0.9236417033773862,
      "fcm_dpo/beta": 0.012154782190918922,
      "fcm_dpo/delta": 0.039435502141714096,
      "fcm_dpo/margin": 66.93384552001953,
      "fcm_dpo/q_t": 0.33409008383750916,
      "grad_norm": 14.45174789428711,
      "learning_rate": 9.19555885822887e-09,
      "logits/chosen": -2.140451431274414,
      "logits/rejected": -1.7289149761199951,
      "logps/chosen": -241.05328369140625,
      "logps/ref_chosen": -103.23037719726562,
      "logps/ref_rejected": -97.16841888427734,
      "logps/rejected": -301.9251708984375,
      "loss": 0.924,
      "margin_dpo/margin_mean": 66.93384552001953,
      "margin_dpo/margin_std": 78.18603515625,
      "step": 629
    },
    {
      "epoch": 0.9251101321585903,
      "fcm_dpo/beta": 0.012714162468910217,
      "fcm_dpo/delta": 0.19320565462112427,
      "fcm_dpo/margin": 52.378536224365234,
      "fcm_dpo/q_t": 0.3655824065208435,
      "grad_norm": 17.370262145996094,
      "learning_rate": 8.85387393063622e-09,
      "logits/chosen": -2.005028247833252,
      "logits/rejected": -1.4612559080123901,
      "logps/chosen": -221.4185333251953,
      "logps/ref_chosen": -93.89755249023438,
      "logps/ref_rejected": -89.3743896484375,
      "logps/rejected": -269.27392578125,
      "loss": 1.0296,
      "margin_dpo/margin_mean": 52.37853240966797,
      "margin_dpo/margin_std": 74.43370819091797,
      "step": 630
    },
    {
      "epoch": 0.9265785609397944,
      "fcm_dpo/beta": 0.012599373236298561,
      "fcm_dpo/delta": -0.037173446267843246,
      "fcm_dpo/margin": 70.12379455566406,
      "fcm_dpo/q_t": 0.3170943558216095,
      "grad_norm": 13.337695121765137,
      "learning_rate": 8.518543427732949e-09,
      "logits/chosen": -2.624936580657959,
      "logits/rejected": -2.2076611518859863,
      "logps/chosen": -211.30435180664062,
      "logps/ref_chosen": -87.77082061767578,
      "logps/ref_rejected": -88.68241882324219,
      "logps/rejected": -282.3397521972656,
      "loss": 0.8397,
      "margin_dpo/margin_mean": 70.12379455566406,
      "margin_dpo/margin_std": 68.069580078125,
      "step": 631
    },
    {
      "epoch": 0.9280469897209985,
      "fcm_dpo/beta": 0.012279413640499115,
      "fcm_dpo/delta": -0.10962569713592529,
      "fcm_dpo/margin": 77.01805114746094,
      "fcm_dpo/q_t": 0.31531599164009094,
      "grad_norm": 13.519450187683105,
      "learning_rate": 8.189576185789637e-09,
      "logits/chosen": -1.8649694919586182,
      "logits/rejected": -1.4337584972381592,
      "logps/chosen": -212.1048583984375,
      "logps/ref_chosen": -88.62652587890625,
      "logps/ref_rejected": -91.45091247558594,
      "logps/rejected": -291.9472961425781,
      "loss": 0.8654,
      "margin_dpo/margin_mean": 77.01805114746094,
      "margin_dpo/margin_std": 84.19517517089844,
      "step": 632
    },
    {
      "epoch": 0.9295154185022027,
      "fcm_dpo/beta": 0.01287509873509407,
      "fcm_dpo/delta": 0.20589163899421692,
      "fcm_dpo/margin": 50.70753479003906,
      "fcm_dpo/q_t": 0.38306355476379395,
      "grad_norm": 18.226625442504883,
      "learning_rate": 7.866980873399015e-09,
      "logits/chosen": -2.24927020072937,
      "logits/rejected": -2.0586564540863037,
      "logps/chosen": -229.56752014160156,
      "logps/ref_chosen": -81.37442016601562,
      "logps/ref_rejected": -98.62571716308594,
      "logps/rejected": -297.5263671875,
      "loss": 1.1293,
      "margin_dpo/margin_mean": 50.70753479003906,
      "margin_dpo/margin_std": 94.33064270019531,
      "step": 633
    },
    {
      "epoch": 0.9309838472834068,
      "fcm_dpo/beta": 0.013133701868355274,
      "fcm_dpo/delta": 0.03609214723110199,
      "fcm_dpo/margin": 61.819000244140625,
      "fcm_dpo/q_t": 0.33319538831710815,
      "grad_norm": 15.686573028564453,
      "learning_rate": 7.550765991247654e-09,
      "logits/chosen": -1.420647382736206,
      "logits/rejected": -1.2333729267120361,
      "logps/chosen": -237.35031127929688,
      "logps/ref_chosen": -96.12284851074219,
      "logps/ref_rejected": -112.84780883789062,
      "logps/rejected": -315.8942565917969,
      "loss": 0.8899,
      "margin_dpo/margin_mean": 61.819000244140625,
      "margin_dpo/margin_std": 63.20722198486328,
      "step": 634
    },
    {
      "epoch": 0.9324522760646109,
      "fcm_dpo/beta": 0.013437749817967415,
      "fcm_dpo/delta": 0.19897405803203583,
      "fcm_dpo/margin": 49.268463134765625,
      "fcm_dpo/q_t": 0.3701525330543518,
      "grad_norm": 18.140663146972656,
      "learning_rate": 7.240939871891699e-09,
      "logits/chosen": -1.8061546087265015,
      "logits/rejected": -1.2960937023162842,
      "logps/chosen": -240.75674438476562,
      "logps/ref_chosen": -98.68411254882812,
      "logps/ref_rejected": -89.89920043945312,
      "logps/rejected": -281.2402648925781,
      "loss": 1.0481,
      "margin_dpo/margin_mean": 49.26845932006836,
      "margin_dpo/margin_std": 77.19035339355469,
      "step": 635
    },
    {
      "epoch": 0.933920704845815,
      "fcm_dpo/beta": 0.013280387967824936,
      "fcm_dpo/delta": -0.12279319763183594,
      "fcm_dpo/margin": 72.37382507324219,
      "fcm_dpo/q_t": 0.3140905499458313,
      "grad_norm": 13.60958194732666,
      "learning_rate": 6.937510679537628e-09,
      "logits/chosen": -1.3869647979736328,
      "logits/rejected": -0.9099713563919067,
      "logps/chosen": -223.68519592285156,
      "logps/ref_chosen": -90.41796112060547,
      "logps/ref_rejected": -87.70687866210938,
      "logps/rejected": -293.34796142578125,
      "loss": 0.8487,
      "margin_dpo/margin_mean": 72.37382507324219,
      "margin_dpo/margin_std": 77.6722640991211,
      "step": 636
    },
    {
      "epoch": 0.9353891336270191,
      "fcm_dpo/beta": 0.012870658189058304,
      "fcm_dpo/delta": -0.16235065460205078,
      "fcm_dpo/margin": 77.41453552246094,
      "fcm_dpo/q_t": 0.30641061067581177,
      "grad_norm": 13.35957145690918,
      "learning_rate": 6.640486409826785e-09,
      "logits/chosen": -2.052138328552246,
      "logits/rejected": -1.9478278160095215,
      "logps/chosen": -216.7566680908203,
      "logps/ref_chosen": -82.44971466064453,
      "logps/ref_rejected": -104.02860260009766,
      "logps/rejected": -315.7500915527344,
      "loss": 0.838,
      "margin_dpo/margin_mean": 77.41453552246094,
      "margin_dpo/margin_std": 81.38263702392578,
      "step": 637
    },
    {
      "epoch": 0.9368575624082232,
      "fcm_dpo/beta": 0.012524936348199844,
      "fcm_dpo/delta": -0.18407437205314636,
      "fcm_dpo/margin": 81.04855346679688,
      "fcm_dpo/q_t": 0.2905474603176117,
      "grad_norm": 11.3641357421875,
      "learning_rate": 6.349874889624962e-09,
      "logits/chosen": -1.6746790409088135,
      "logits/rejected": -1.0038890838623047,
      "logps/chosen": -213.70997619628906,
      "logps/ref_chosen": -91.92498779296875,
      "logps/ref_rejected": -86.28703308105469,
      "logps/rejected": -289.1205749511719,
      "loss": 0.7404,
      "margin_dpo/margin_mean": 81.04855346679688,
      "margin_dpo/margin_std": 61.875587463378906,
      "step": 638
    },
    {
      "epoch": 0.9383259911894273,
      "fcm_dpo/beta": 0.012901335954666138,
      "fcm_dpo/delta": 0.29382050037384033,
      "fcm_dpo/margin": 44.23047637939453,
      "fcm_dpo/q_t": 0.38710546493530273,
      "grad_norm": 17.964210510253906,
      "learning_rate": 6.065683776815933e-09,
      "logits/chosen": -1.2971713542938232,
      "logits/rejected": -0.5256457328796387,
      "logps/chosen": -265.3831481933594,
      "logps/ref_chosen": -104.52755737304688,
      "logps/ref_rejected": -81.4803466796875,
      "logps/rejected": -286.56640625,
      "loss": 1.1298,
      "margin_dpo/margin_mean": 44.23047637939453,
      "margin_dpo/margin_std": 80.78390502929688,
      "step": 639
    },
    {
      "epoch": 0.9397944199706314,
      "fcm_dpo/beta": 0.012978674843907356,
      "fcm_dpo/delta": -0.14304372668266296,
      "fcm_dpo/margin": 75.16079711914062,
      "fcm_dpo/q_t": 0.3036750257015228,
      "grad_norm": 14.67367935180664,
      "learning_rate": 5.7879205600998296e-09,
      "logits/chosen": -1.6546118259429932,
      "logits/rejected": -1.1096612215042114,
      "logps/chosen": -232.30517578125,
      "logps/ref_chosen": -97.88526916503906,
      "logps/ref_rejected": -112.70502471923828,
      "logps/rejected": -322.28570556640625,
      "loss": 0.8363,
      "margin_dpo/margin_mean": 75.16079711914062,
      "margin_dpo/margin_std": 71.88284301757812,
      "step": 640
    },
    {
      "epoch": 0.9412628487518355,
      "fcm_dpo/beta": 0.01246521808207035,
      "fcm_dpo/delta": -0.10708156228065491,
      "fcm_dpo/margin": 75.97200012207031,
      "fcm_dpo/q_t": 0.3164488673210144,
      "grad_norm": 13.984350204467773,
      "learning_rate": 5.516592558795746e-09,
      "logits/chosen": -2.2499518394470215,
      "logits/rejected": -1.7141563892364502,
      "logps/chosen": -221.30508422851562,
      "logps/ref_chosen": -96.4456787109375,
      "logps/ref_rejected": -95.13568878173828,
      "logps/rejected": -295.96710205078125,
      "loss": 0.8656,
      "margin_dpo/margin_mean": 75.97200775146484,
      "margin_dpo/margin_std": 85.18379211425781,
      "step": 641
    },
    {
      "epoch": 0.9427312775330396,
      "fcm_dpo/beta": 0.012517379596829414,
      "fcm_dpo/delta": 0.03650232404470444,
      "fcm_dpo/margin": 65.17138671875,
      "fcm_dpo/q_t": 0.3425823152065277,
      "grad_norm": 14.213191032409668,
      "learning_rate": 5.251706922648868e-09,
      "logits/chosen": -1.5260307788848877,
      "logits/rejected": -1.1929254531860352,
      "logps/chosen": -242.54896545410156,
      "logps/ref_chosen": -100.75984954833984,
      "logps/ref_rejected": -114.7076416015625,
      "logps/rejected": -321.66815185546875,
      "loss": 0.9487,
      "margin_dpo/margin_mean": 65.17138671875,
      "margin_dpo/margin_std": 85.03411865234375,
      "step": 642
    },
    {
      "epoch": 0.9441997063142438,
      "fcm_dpo/beta": 0.012432137504220009,
      "fcm_dpo/delta": 0.05663251131772995,
      "fcm_dpo/margin": 64.00656127929688,
      "fcm_dpo/q_t": 0.33634692430496216,
      "grad_norm": 14.948612213134766,
      "learning_rate": 4.993270631642038e-09,
      "logits/chosen": -2.1365489959716797,
      "logits/rejected": -1.7338988780975342,
      "logps/chosen": -218.6114501953125,
      "logps/ref_chosen": -84.74365997314453,
      "logps/ref_rejected": -94.31842041015625,
      "logps/rejected": -292.1927490234375,
      "loss": 0.9151,
      "margin_dpo/margin_mean": 64.00656127929688,
      "margin_dpo/margin_std": 70.07855224609375,
      "step": 643
    },
    {
      "epoch": 0.9456681350954479,
      "fcm_dpo/beta": 0.012908038683235645,
      "fcm_dpo/delta": 0.14631298184394836,
      "fcm_dpo/margin": 55.31208801269531,
      "fcm_dpo/q_t": 0.35338038206100464,
      "grad_norm": 17.55486488342285,
      "learning_rate": 4.741290495811873e-09,
      "logits/chosen": -1.9027633666992188,
      "logits/rejected": -1.626082181930542,
      "logps/chosen": -219.79922485351562,
      "logps/ref_chosen": -85.32274627685547,
      "logps/ref_rejected": -94.60861206054688,
      "logps/rejected": -284.39715576171875,
      "loss": 0.9612,
      "margin_dpo/margin_mean": 55.31208801269531,
      "margin_dpo/margin_std": 68.78119659423828,
      "step": 644
    },
    {
      "epoch": 0.947136563876652,
      "fcm_dpo/beta": 0.013628844171762466,
      "fcm_dpo/delta": 0.29724252223968506,
      "fcm_dpo/margin": 41.70508575439453,
      "fcm_dpo/q_t": 0.3945990204811096,
      "grad_norm": 17.644214630126953,
      "learning_rate": 4.495773155069299e-09,
      "logits/chosen": -1.4874303340911865,
      "logits/rejected": -1.2977774143218994,
      "logps/chosen": -227.33688354492188,
      "logps/ref_chosen": -82.59024047851562,
      "logps/ref_rejected": -103.00375366210938,
      "logps/rejected": -289.45550537109375,
      "loss": 1.1689,
      "margin_dpo/margin_mean": 41.70508575439453,
      "margin_dpo/margin_std": 83.13195037841797,
      "step": 645
    },
    {
      "epoch": 0.9486049926578561,
      "fcm_dpo/beta": 0.013419127091765404,
      "fcm_dpo/delta": -0.12888050079345703,
      "fcm_dpo/margin": 71.85616302490234,
      "fcm_dpo/q_t": 0.3052224814891815,
      "grad_norm": 14.331748962402344,
      "learning_rate": 4.256725079024553e-09,
      "logits/chosen": -1.8919644355773926,
      "logits/rejected": -1.2960618734359741,
      "logps/chosen": -217.77532958984375,
      "logps/ref_chosen": -91.1920394897461,
      "logps/ref_rejected": -83.77833557128906,
      "logps/rejected": -282.2177734375,
      "loss": 0.8021,
      "margin_dpo/margin_mean": 71.85616302490234,
      "margin_dpo/margin_std": 66.11752319335938,
      "step": 646
    },
    {
      "epoch": 0.9500734214390602,
      "fcm_dpo/beta": 0.01349719800055027,
      "fcm_dpo/delta": 0.049388255923986435,
      "fcm_dpo/margin": 59.609161376953125,
      "fcm_dpo/q_t": 0.3525710701942444,
      "grad_norm": 16.58894920349121,
      "learning_rate": 4.024152566816791e-09,
      "logits/chosen": -1.5568263530731201,
      "logits/rejected": -1.4186091423034668,
      "logps/chosen": -230.43569946289062,
      "logps/ref_chosen": -88.8444595336914,
      "logps/ref_rejected": -99.49833679199219,
      "logps/rejected": -300.6987609863281,
      "loss": 1.0062,
      "margin_dpo/margin_mean": 59.609169006347656,
      "margin_dpo/margin_std": 87.35142517089844,
      "step": 647
    },
    {
      "epoch": 0.9515418502202643,
      "fcm_dpo/beta": 0.01303325779736042,
      "fcm_dpo/delta": -0.30287304520606995,
      "fcm_dpo/margin": 85.91302490234375,
      "fcm_dpo/q_t": 0.283531129360199,
      "grad_norm": 13.930960655212402,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": -2.1560635566711426,
      "logits/rejected": -2.075185537338257,
      "logps/chosen": -216.96524047851562,
      "logps/ref_chosen": -87.84810638427734,
      "logps/ref_rejected": -104.67005920410156,
      "logps/rejected": -319.7001953125,
      "loss": 0.7651,
      "margin_dpo/margin_mean": 85.91301727294922,
      "margin_dpo/margin_std": 82.10009002685547,
      "step": 648
    },
    {
      "epoch": 0.9530102790014684,
      "fcm_dpo/beta": 0.012625191360712051,
      "fcm_dpo/delta": -0.027126122266054153,
      "fcm_dpo/margin": 69.18255615234375,
      "fcm_dpo/q_t": 0.3202654719352722,
      "grad_norm": 13.677901268005371,
      "learning_rate": 3.5784585771215235e-09,
      "logits/chosen": -2.4737744331359863,
      "logits/rejected": -2.0878794193267822,
      "logps/chosen": -210.0411376953125,
      "logps/ref_chosen": -89.6925048828125,
      "logps/ref_rejected": -88.70658111572266,
      "logps/rejected": -278.2377624511719,
      "loss": 0.8399,
      "margin_dpo/margin_mean": 69.18255615234375,
      "margin_dpo/margin_std": 65.37664794921875,
      "step": 649
    },
    {
      "epoch": 0.9544787077826725,
      "fcm_dpo/beta": 0.012697836384177208,
      "fcm_dpo/delta": -0.015169069170951843,
      "fcm_dpo/margin": 68.02926635742188,
      "fcm_dpo/q_t": 0.32649314403533936,
      "grad_norm": 15.16059684753418,
      "learning_rate": 3.3653488440851253e-09,
      "logits/chosen": -1.3418605327606201,
      "logits/rejected": -1.1561648845672607,
      "logps/chosen": -227.9875030517578,
      "logps/ref_chosen": -89.93060302734375,
      "logps/ref_rejected": -102.61282348632812,
      "logps/rejected": -308.698974609375,
      "loss": 0.9067,
      "margin_dpo/margin_mean": 68.02926635742188,
      "margin_dpo/margin_std": 80.34864807128906,
      "step": 650
    },
    {
      "epoch": 0.9559471365638766,
      "fcm_dpo/beta": 0.012280041351914406,
      "fcm_dpo/delta": -0.18812772631645203,
      "fcm_dpo/margin": 82.9175796508789,
      "fcm_dpo/q_t": 0.2923458218574524,
      "grad_norm": 12.51254653930664,
      "learning_rate": 3.158738163478475e-09,
      "logits/chosen": -2.165620803833008,
      "logits/rejected": -2.0002899169921875,
      "logps/chosen": -200.1165771484375,
      "logps/ref_chosen": -79.18731689453125,
      "logps/ref_rejected": -105.93333435058594,
      "logps/rejected": -309.7801513671875,
      "loss": 0.7648,
      "margin_dpo/margin_mean": 82.91757202148438,
      "margin_dpo/margin_std": 71.92076110839844,
      "step": 651
    },
    {
      "epoch": 0.9574155653450808,
      "fcm_dpo/beta": 0.012219004333019257,
      "fcm_dpo/delta": -0.007074171677231789,
      "fcm_dpo/margin": 70.08258819580078,
      "fcm_dpo/q_t": 0.3215482532978058,
      "grad_norm": 12.98484992980957,
      "learning_rate": 2.9586319796851555e-09,
      "logits/chosen": -2.567788600921631,
      "logits/rejected": -2.267869472503662,
      "logps/chosen": -223.45941162109375,
      "logps/ref_chosen": -101.79022216796875,
      "logps/ref_rejected": -116.3245849609375,
      "logps/rejected": -308.07635498046875,
      "loss": 0.8286,
      "margin_dpo/margin_mean": 70.08259582519531,
      "margin_dpo/margin_std": 64.39094543457031,
      "step": 652
    },
    {
      "epoch": 0.9588839941262849,
      "fcm_dpo/beta": 0.012271079234778881,
      "fcm_dpo/delta": 0.017625533044338226,
      "fcm_dpo/margin": 67.90986633300781,
      "fcm_dpo/q_t": 0.3277429938316345,
      "grad_norm": 13.8803071975708,
      "learning_rate": 2.7650355656892166e-09,
      "logits/chosen": -2.142395496368408,
      "logits/rejected": -1.8992211818695068,
      "logps/chosen": -230.5672607421875,
      "logps/ref_chosen": -93.35359191894531,
      "logps/ref_rejected": -109.12324523925781,
      "logps/rejected": -314.2467956542969,
      "loss": 0.8909,
      "margin_dpo/margin_mean": 67.90987396240234,
      "margin_dpo/margin_std": 73.30245208740234,
      "step": 653
    },
    {
      "epoch": 0.960352422907489,
      "fcm_dpo/beta": 0.012467766180634499,
      "fcm_dpo/delta": 0.13704301416873932,
      "fcm_dpo/margin": 57.98382568359375,
      "fcm_dpo/q_t": 0.36294329166412354,
      "grad_norm": 18.976091384887695,
      "learning_rate": 2.577954022936174e-09,
      "logits/chosen": -2.1631104946136475,
      "logits/rejected": -1.970808982849121,
      "logps/chosen": -229.53439331054688,
      "logps/ref_chosen": -89.11553955078125,
      "logps/ref_rejected": -104.91995239257812,
      "logps/rejected": -303.3226318359375,
      "loss": 1.0347,
      "margin_dpo/margin_mean": 57.98382568359375,
      "margin_dpo/margin_std": 89.45317077636719,
      "step": 654
    },
    {
      "epoch": 0.9618208516886931,
      "fcm_dpo/beta": 0.01260659471154213,
      "fcm_dpo/delta": -0.004192095249891281,
      "fcm_dpo/margin": 67.70552062988281,
      "fcm_dpo/q_t": 0.327122300863266,
      "grad_norm": 14.10622787475586,
      "learning_rate": 2.397392281198729e-09,
      "logits/chosen": -2.0303776264190674,
      "logits/rejected": -2.0248003005981445,
      "logps/chosen": -212.3218231201172,
      "logps/ref_chosen": -81.03610229492188,
      "logps/ref_rejected": -102.80233764648438,
      "logps/rejected": -301.7935791015625,
      "loss": 0.9029,
      "margin_dpo/margin_mean": 67.70552825927734,
      "margin_dpo/margin_std": 76.8497314453125,
      "step": 655
    },
    {
      "epoch": 0.9632892804698973,
      "fcm_dpo/beta": 0.012112503871321678,
      "fcm_dpo/delta": -0.2109536975622177,
      "fcm_dpo/margin": 85.67755126953125,
      "fcm_dpo/q_t": 0.2922847270965576,
      "grad_norm": 15.038117408752441,
      "learning_rate": 2.223355098446622e-09,
      "logits/chosen": -1.5206151008605957,
      "logits/rejected": -1.590084195137024,
      "logps/chosen": -213.54559326171875,
      "logps/ref_chosen": -85.32534790039062,
      "logps/ref_rejected": -118.33866119384766,
      "logps/rejected": -332.2364501953125,
      "loss": 0.7521,
      "margin_dpo/margin_mean": 85.67755126953125,
      "margin_dpo/margin_std": 75.42826843261719,
      "step": 656
    },
    {
      "epoch": 0.9647577092511013,
      "fcm_dpo/beta": 0.011937540024518967,
      "fcm_dpo/delta": -0.011723875999450684,
      "fcm_dpo/margin": 72.03126525878906,
      "fcm_dpo/q_t": 0.33278656005859375,
      "grad_norm": 14.324390411376953,
      "learning_rate": 2.055847060721566e-09,
      "logits/chosen": -1.9533765316009521,
      "logits/rejected": -1.7715580463409424,
      "logps/chosen": -213.4752197265625,
      "logps/ref_chosen": -80.19772338867188,
      "logps/ref_rejected": -102.581298828125,
      "logps/rejected": -307.89007568359375,
      "loss": 0.9101,
      "margin_dpo/margin_mean": 72.03126525878906,
      "margin_dpo/margin_std": 85.60971069335938,
      "step": 657
    },
    {
      "epoch": 0.9662261380323054,
      "fcm_dpo/beta": 0.012269849888980389,
      "fcm_dpo/delta": 0.10941661894321442,
      "fcm_dpo/margin": 60.93087387084961,
      "fcm_dpo/q_t": 0.3474779427051544,
      "grad_norm": 14.329385757446289,
      "learning_rate": 1.8948725820160662e-09,
      "logits/chosen": -2.008090019226074,
      "logits/rejected": -1.540211796760559,
      "logps/chosen": -236.1888427734375,
      "logps/ref_chosen": -94.634521484375,
      "logps/ref_rejected": -101.6316146850586,
      "logps/rejected": -304.1168212890625,
      "loss": 0.9428,
      "margin_dpo/margin_mean": 60.930870056152344,
      "margin_dpo/margin_std": 73.01217651367188,
      "step": 658
    },
    {
      "epoch": 0.9676945668135095,
      "fcm_dpo/beta": 0.011998293921351433,
      "fcm_dpo/delta": -0.11588911712169647,
      "fcm_dpo/margin": 79.48670959472656,
      "fcm_dpo/q_t": 0.31771883368492126,
      "grad_norm": 13.859298706054688,
      "learning_rate": 1.7404359041573723e-09,
      "logits/chosen": -2.127384662628174,
      "logits/rejected": -1.1833677291870117,
      "logps/chosen": -234.04229736328125,
      "logps/ref_chosen": -112.55587005615234,
      "logps/ref_rejected": -93.9216079711914,
      "logps/rejected": -294.8947448730469,
      "loss": 0.9104,
      "margin_dpo/margin_mean": 79.48670959472656,
      "margin_dpo/margin_std": 94.99459838867188,
      "step": 659
    },
    {
      "epoch": 0.9691629955947136,
      "fcm_dpo/beta": 0.01178102195262909,
      "fcm_dpo/delta": -0.10657566785812378,
      "fcm_dpo/margin": 80.31880187988281,
      "fcm_dpo/q_t": 0.31062257289886475,
      "grad_norm": 13.350107192993164,
      "learning_rate": 1.592541096695571e-09,
      "logits/chosen": -1.8621575832366943,
      "logits/rejected": -1.2513611316680908,
      "logps/chosen": -220.84124755859375,
      "logps/ref_chosen": -93.37742614746094,
      "logps/ref_rejected": -81.39482116699219,
      "logps/rejected": -289.17742919921875,
      "loss": 0.8279,
      "margin_dpo/margin_mean": 80.31880187988281,
      "margin_dpo/margin_std": 80.75775909423828,
      "step": 660
    },
    {
      "epoch": 0.9706314243759178,
      "fcm_dpo/beta": 0.011570056900382042,
      "fcm_dpo/delta": -0.12267627567052841,
      "fcm_dpo/margin": 83.06248474121094,
      "fcm_dpo/q_t": 0.30892378091812134,
      "grad_norm": 11.488774299621582,
      "learning_rate": 1.4511920567963908e-09,
      "logits/chosen": -1.920933723449707,
      "logits/rejected": -1.4082481861114502,
      "logps/chosen": -223.60641479492188,
      "logps/ref_chosen": -87.85516357421875,
      "logps/ref_rejected": -92.40330505371094,
      "logps/rejected": -311.217041015625,
      "loss": 0.8254,
      "margin_dpo/margin_mean": 83.06248474121094,
      "margin_dpo/margin_std": 83.68098449707031,
      "step": 661
    },
    {
      "epoch": 0.9720998531571219,
      "fcm_dpo/beta": 0.011501193046569824,
      "fcm_dpo/delta": 0.00793197751045227,
      "fcm_dpo/margin": 73.25682067871094,
      "fcm_dpo/q_t": 0.3226693272590637,
      "grad_norm": 11.854416847229004,
      "learning_rate": 1.3163925091384532e-09,
      "logits/chosen": -1.7558529376983643,
      "logits/rejected": -1.2050141096115112,
      "logps/chosen": -232.50390625,
      "logps/ref_chosen": -102.77980041503906,
      "logps/ref_rejected": -95.22531127929688,
      "logps/rejected": -298.20623779296875,
      "loss": 0.8323,
      "margin_dpo/margin_mean": 73.25682067871094,
      "margin_dpo/margin_std": 65.54705810546875,
      "step": 662
    },
    {
      "epoch": 0.973568281938326,
      "fcm_dpo/beta": 0.011573947966098785,
      "fcm_dpo/delta": 0.031047917902469635,
      "fcm_dpo/margin": 70.93704223632812,
      "fcm_dpo/q_t": 0.3348495364189148,
      "grad_norm": 12.32866096496582,
      "learning_rate": 1.1881460058152382e-09,
      "logits/chosen": -2.1356537342071533,
      "logits/rejected": -1.9021480083465576,
      "logps/chosen": -223.847900390625,
      "logps/ref_chosen": -96.34658813476562,
      "logps/ref_rejected": -120.52645874023438,
      "logps/rejected": -318.9648132324219,
      "loss": 0.9124,
      "margin_dpo/margin_mean": 70.93704223632812,
      "margin_dpo/margin_std": 81.27235412597656,
      "step": 663
    },
    {
      "epoch": 0.9750367107195301,
      "fcm_dpo/beta": 0.011720478534698486,
      "fcm_dpo/delta": 0.0888986736536026,
      "fcm_dpo/margin": 65.49781799316406,
      "fcm_dpo/q_t": 0.3386794328689575,
      "grad_norm": 14.023902893066406,
      "learning_rate": 1.066455926241383e-09,
      "logits/chosen": -1.7385942935943604,
      "logits/rejected": -1.5541484355926514,
      "logps/chosen": -230.68548583984375,
      "logps/ref_chosen": -91.84242248535156,
      "logps/ref_rejected": -111.8366928100586,
      "logps/rejected": -316.17755126953125,
      "loss": 0.9057,
      "margin_dpo/margin_mean": 65.49781036376953,
      "margin_dpo/margin_std": 69.79621124267578,
      "step": 664
    },
    {
      "epoch": 0.9765051395007343,
      "fcm_dpo/beta": 0.01180583517998457,
      "fcm_dpo/delta": 0.018876872956752777,
      "fcm_dpo/margin": 70.52334594726562,
      "fcm_dpo/q_t": 0.3239971399307251,
      "grad_norm": 12.435693740844727,
      "learning_rate": 9.513254770636137e-10,
      "logits/chosen": -2.437134265899658,
      "logits/rejected": -2.011138677597046,
      "logps/chosen": -209.5286865234375,
      "logps/ref_chosen": -88.18618774414062,
      "logps/ref_rejected": -91.9120101928711,
      "logps/rejected": -283.77789306640625,
      "loss": 0.853,
      "margin_dpo/margin_mean": 70.52334594726562,
      "margin_dpo/margin_std": 67.84070587158203,
      "step": 665
    },
    {
      "epoch": 0.9779735682819384,
      "fcm_dpo/beta": 0.011835319921374321,
      "fcm_dpo/delta": 0.05988616496324539,
      "fcm_dpo/margin": 67.1385726928711,
      "fcm_dpo/q_t": 0.34297046065330505,
      "grad_norm": 15.745160102844238,
      "learning_rate": 8.427576920763956e-10,
      "logits/chosen": -1.4836091995239258,
      "logits/rejected": -1.1046907901763916,
      "logps/chosen": -237.0187225341797,
      "logps/ref_chosen": -100.97459411621094,
      "logps/ref_rejected": -101.24992370605469,
      "logps/rejected": -304.4326171875,
      "loss": 0.949,
      "margin_dpo/margin_mean": 67.13858032226562,
      "margin_dpo/margin_std": 83.11888122558594,
      "step": 666
    },
    {
      "epoch": 0.9794419970631424,
      "fcm_dpo/beta": 0.011725520715117455,
      "fcm_dpo/delta": -0.0794256329536438,
      "fcm_dpo/margin": 78.543212890625,
      "fcm_dpo/q_t": 0.3089987635612488,
      "grad_norm": 12.336382865905762,
      "learning_rate": 7.407554321417764e-10,
      "logits/chosen": -1.6863865852355957,
      "logits/rejected": -1.050025463104248,
      "logps/chosen": -240.4290771484375,
      "logps/ref_chosen": -97.5711669921875,
      "logps/ref_rejected": -93.58476257324219,
      "logps/rejected": -314.98590087890625,
      "loss": 0.8034,
      "margin_dpo/margin_mean": 78.543212890625,
      "margin_dpo/margin_std": 69.23985290527344,
      "step": 667
    },
    {
      "epoch": 0.9809104258443465,
      "fcm_dpo/beta": 0.011529900133609772,
      "fcm_dpo/delta": -0.15593984723091125,
      "fcm_dpo/margin": 85.928466796875,
      "fcm_dpo/q_t": 0.30568087100982666,
      "grad_norm": 14.262736320495605,
      "learning_rate": 6.453213851142225e-10,
      "logits/chosen": -2.211690902709961,
      "logits/rejected": -1.6778247356414795,
      "logps/chosen": -231.51792907714844,
      "logps/ref_chosen": -102.5750503540039,
      "logps/ref_rejected": -108.81768798828125,
      "logps/rejected": -323.68902587890625,
      "loss": 0.7933,
      "margin_dpo/margin_mean": 85.928466796875,
      "margin_dpo/margin_std": 82.60166931152344,
      "step": 668
    },
    {
      "epoch": 0.9823788546255506,
      "fcm_dpo/beta": 0.011247916147112846,
      "fcm_dpo/delta": -0.08348070085048676,
      "fcm_dpo/margin": 82.30418395996094,
      "fcm_dpo/q_t": 0.3088477849960327,
      "grad_norm": 12.096893310546875,
      "learning_rate": 5.564580657695939e-10,
      "logits/chosen": -1.8613781929016113,
      "logits/rejected": -1.4636726379394531,
      "logps/chosen": -207.69525146484375,
      "logps/ref_chosen": -89.49478149414062,
      "logps/ref_rejected": -82.51950073242188,
      "logps/rejected": -283.024169921875,
      "loss": 0.8056,
      "margin_dpo/margin_mean": 82.30418395996094,
      "margin_dpo/margin_std": 74.32376098632812,
      "step": 669
    },
    {
      "epoch": 0.9838472834067548,
      "fcm_dpo/beta": 0.010946569964289665,
      "fcm_dpo/delta": -0.1527199000120163,
      "fcm_dpo/margin": 90.21934509277344,
      "fcm_dpo/q_t": 0.2935888469219208,
      "grad_norm": 11.206657409667969,
      "learning_rate": 4.741678157389739e-10,
      "logits/chosen": -1.7874915599822998,
      "logits/rejected": -1.2579689025878906,
      "logps/chosen": -214.5462188720703,
      "logps/ref_chosen": -95.45459747314453,
      "logps/ref_rejected": -101.53292846679688,
      "logps/rejected": -310.8439025878906,
      "loss": 0.7782,
      "margin_dpo/margin_mean": 90.21934509277344,
      "margin_dpo/margin_std": 78.12318420410156,
      "step": 670
    },
    {
      "epoch": 0.9853157121879589,
      "fcm_dpo/beta": 0.011242946609854698,
      "fcm_dpo/delta": 0.16490933299064636,
      "fcm_dpo/margin": 61.64239501953125,
      "fcm_dpo/q_t": 0.35765212774276733,
      "grad_norm": 13.497610092163086,
      "learning_rate": 3.9845280344705245e-10,
      "logits/chosen": -1.8117475509643555,
      "logits/rejected": -1.5957112312316895,
      "logps/chosen": -226.856689453125,
      "logps/ref_chosen": -82.12312316894531,
      "logps/ref_rejected": -90.21969604492188,
      "logps/rejected": -296.59564208984375,
      "loss": 0.9702,
      "margin_dpo/margin_mean": 61.64239501953125,
      "margin_dpo/margin_std": 76.5667495727539,
      "step": 671
    },
    {
      "epoch": 0.986784140969163,
      "fcm_dpo/beta": 0.011585546657443047,
      "fcm_dpo/delta": 0.20920199155807495,
      "fcm_dpo/margin": 56.48442459106445,
      "fcm_dpo/q_t": 0.36586183309555054,
      "grad_norm": 13.58948040008545,
      "learning_rate": 3.293150240547549e-10,
      "logits/chosen": -1.972328543663025,
      "logits/rejected": -1.6273032426834106,
      "logps/chosen": -232.64218139648438,
      "logps/ref_chosen": -90.0619125366211,
      "logps/ref_rejected": -100.45323181152344,
      "logps/rejected": -299.5179443359375,
      "loss": 0.9928,
      "margin_dpo/margin_mean": 56.48442840576172,
      "margin_dpo/margin_std": 72.01486206054688,
      "step": 672
    },
    {
      "epoch": 0.9882525697503671,
      "fcm_dpo/beta": 0.011844823136925697,
      "fcm_dpo/delta": 0.054356593638658524,
      "fcm_dpo/margin": 67.50054168701172,
      "fcm_dpo/q_t": 0.33461272716522217,
      "grad_norm": 13.27198314666748,
      "learning_rate": 2.6675629940689504e-10,
      "logits/chosen": -1.9633225202560425,
      "logits/rejected": -1.5942094326019287,
      "logps/chosen": -204.55674743652344,
      "logps/ref_chosen": -79.26315307617188,
      "logps/ref_rejected": -91.34925079345703,
      "logps/rejected": -284.14337158203125,
      "loss": 0.8878,
      "margin_dpo/margin_mean": 67.50054931640625,
      "margin_dpo/margin_std": 71.98117065429688,
      "step": 673
    },
    {
      "epoch": 0.9897209985315712,
      "fcm_dpo/beta": 0.011596374213695526,
      "fcm_dpo/delta": -0.12112964689731598,
      "fcm_dpo/margin": 82.70986938476562,
      "fcm_dpo/q_t": 0.3062339425086975,
      "grad_norm": 13.10261344909668,
      "learning_rate": 2.1077827798404725e-10,
      "logits/chosen": -1.6151472330093384,
      "logits/rejected": -1.2215874195098877,
      "logps/chosen": -197.07003784179688,
      "logps/ref_chosen": -75.45831298828125,
      "logps/ref_rejected": -76.20362854003906,
      "logps/rejected": -280.5252380371094,
      "loss": 0.7916,
      "margin_dpo/margin_mean": 82.70986938476562,
      "margin_dpo/margin_std": 76.59530639648438,
      "step": 674
    },
    {
      "epoch": 0.9911894273127754,
      "fcm_dpo/beta": 0.011463642120361328,
      "fcm_dpo/delta": 0.01827855408191681,
      "fcm_dpo/margin": 72.52214050292969,
      "fcm_dpo/q_t": 0.33195608854293823,
      "grad_norm": 14.91750431060791,
      "learning_rate": 1.6138243485910863e-10,
      "logits/chosen": -1.5982558727264404,
      "logits/rejected": -1.2730377912521362,
      "logps/chosen": -214.935791015625,
      "logps/ref_chosen": -79.90953063964844,
      "logps/ref_rejected": -81.21824645996094,
      "logps/rejected": -288.76666259765625,
      "loss": 0.9378,
      "margin_dpo/margin_mean": 72.52214050292969,
      "margin_dpo/margin_std": 88.32064819335938,
      "step": 675
    },
    {
      "epoch": 0.9926578560939795,
      "fcm_dpo/beta": 0.011480691842734814,
      "fcm_dpo/delta": -0.03261286020278931,
      "fcm_dpo/margin": 76.57514953613281,
      "fcm_dpo/q_t": 0.31988048553466797,
      "grad_norm": 13.69805908203125,
      "learning_rate": 1.1857007165852472e-10,
      "logits/chosen": -2.108896493911743,
      "logits/rejected": -1.548959732055664,
      "logps/chosen": -237.62142944335938,
      "logps/ref_chosen": -98.17111206054688,
      "logps/ref_rejected": -95.024658203125,
      "logps/rejected": -311.05010986328125,
      "loss": 0.8634,
      "margin_dpo/margin_mean": 76.57515716552734,
      "margin_dpo/margin_std": 78.37229919433594,
      "step": 676
    },
    {
      "epoch": 0.9941262848751835,
      "fcm_dpo/beta": 0.011398270726203918,
      "fcm_dpo/delta": -0.053326986730098724,
      "fcm_dpo/margin": 78.80928039550781,
      "fcm_dpo/q_t": 0.311850368976593,
      "grad_norm": 11.52205753326416,
      "learning_rate": 8.23423165278725e-11,
      "logits/chosen": -1.9365110397338867,
      "logits/rejected": -1.2789349555969238,
      "logps/chosen": -218.03697204589844,
      "logps/ref_chosen": -91.37928771972656,
      "logps/ref_rejected": -82.87776947021484,
      "logps/rejected": -288.3447265625,
      "loss": 0.802,
      "margin_dpo/margin_mean": 78.80928039550781,
      "margin_dpo/margin_std": 67.46434020996094,
      "step": 677
    },
    {
      "epoch": 0.9955947136563876,
      "fcm_dpo/beta": 0.011227824725210667,
      "fcm_dpo/delta": -0.1735784113407135,
      "fcm_dpo/margin": 89.56806945800781,
      "fcm_dpo/q_t": 0.2927679419517517,
      "grad_norm": 11.666086196899414,
      "learning_rate": 5.270012410216185e-11,
      "logits/chosen": -1.7108724117279053,
      "logits/rejected": -1.3849983215332031,
      "logps/chosen": -200.70018005371094,
      "logps/ref_chosen": -75.6458740234375,
      "logps/ref_rejected": -86.96611022949219,
      "logps/rejected": -301.5885009765625,
      "loss": 0.7627,
      "margin_dpo/margin_mean": 89.56806945800781,
      "margin_dpo/margin_std": 73.57870483398438,
      "step": 678
    },
    {
      "epoch": 0.9970631424375918,
      "fcm_dpo/beta": 0.011059444397687912,
      "fcm_dpo/delta": 0.03496631607413292,
      "fcm_dpo/margin": 73.9551773071289,
      "fcm_dpo/q_t": 0.3249191641807556,
      "grad_norm": 11.829185485839844,
      "learning_rate": 2.9644275480772416e-11,
      "logits/chosen": -2.3330090045928955,
      "logits/rejected": -1.82859206199646,
      "logps/chosen": -217.5744171142578,
      "logps/ref_chosen": -80.77344512939453,
      "logps/ref_rejected": -82.87850189208984,
      "logps/rejected": -293.6346435546875,
      "loss": 0.8263,
      "margin_dpo/margin_mean": 73.95518493652344,
      "margin_dpo/margin_std": 62.57131576538086,
      "step": 679
    },
    {
      "epoch": 0.9985315712187959,
      "fcm_dpo/beta": 0.011266154237091541,
      "fcm_dpo/delta": 0.01519148051738739,
      "fcm_dpo/margin": 73.86129760742188,
      "fcm_dpo/q_t": 0.3306913375854492,
      "grad_norm": 11.578691482543945,
      "learning_rate": 1.31753782067201e-11,
      "logits/chosen": -1.9968955516815186,
      "logits/rejected": -1.688669204711914,
      "logps/chosen": -249.28564453125,
      "logps/ref_chosen": -107.68292999267578,
      "logps/ref_rejected": -116.0948715209961,
      "logps/rejected": -331.5588684082031,
      "loss": 0.889,
      "margin_dpo/margin_mean": 73.86129760742188,
      "margin_dpo/margin_std": 76.965576171875,
      "step": 680
    },
    {
      "epoch": 1.0,
      "fcm_dpo/beta": 0.011270027607679367,
      "fcm_dpo/delta": 0.07696523517370224,
      "fcm_dpo/margin": 69.10145568847656,
      "fcm_dpo/q_t": 0.3385009765625,
      "grad_norm": 12.040020942687988,
      "learning_rate": 3.2938662507808745e-12,
      "logits/chosen": -2.3690905570983887,
      "logits/rejected": -2.0767529010772705,
      "logps/chosen": -219.39236450195312,
      "logps/ref_chosen": -93.01106262207031,
      "logps/ref_rejected": -94.8221664428711,
      "logps/rejected": -290.304931640625,
      "loss": 0.9025,
      "margin_dpo/margin_mean": 69.10145568847656,
      "margin_dpo/margin_std": 75.53838348388672,
      "step": 681
    },
    {
      "epoch": 1.0,
      "step": 681,
      "total_flos": 0.0,
      "train_loss": 0.9292974616462438,
      "train_runtime": 2239.8819,
      "train_samples_per_second": 19.464,
      "train_steps_per_second": 0.304
    }
  ],
  "logging_steps": 1,
  "max_steps": 681,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}