{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 200,
  "global_step": 681,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0014684287812041115,
      "fcm_dpo/beta": 0.10308968275785446,
      "fcm_dpo/delta": 0.29979780316352844,
      "fcm_dpo/margin": -0.02287006378173828,
      "fcm_dpo/q_t": 0.5005706548690796,
      "grad_norm": 86.183349609375,
      "learning_rate": 0.0,
      "logits/chosen": -0.4974287748336792,
      "logits/rejected": -0.43299180269241333,
      "logps/chosen": -50.1435661315918,
      "logps/ref_chosen": -50.14883804321289,
      "logps/ref_rejected": -74.1280517578125,
      "logps/rejected": -74.09991455078125,
      "loss": 1.389,
      "margin_dpo/margin_mean": -0.02287048101425171,
      "margin_dpo/margin_std": 0.41920793056488037,
      "step": 1
    },
    {
      "epoch": 0.002936857562408223,
      "fcm_dpo/beta": 0.1061793640255928,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.06572261452674866,
      "fcm_dpo/q_t": 0.5017440319061279,
      "grad_norm": 76.63970184326172,
      "learning_rate": 7.246376811594203e-09,
      "logits/chosen": -0.49536412954330444,
      "logits/rejected": -0.4594460427761078,
      "logps/chosen": -52.65568923950195,
      "logps/ref_chosen": -52.620704650878906,
      "logps/ref_rejected": -75.30413818359375,
      "logps/rejected": -75.27340698242188,
      "loss": 1.3936,
      "margin_dpo/margin_mean": -0.06572240591049194,
      "margin_dpo/margin_std": 0.35048407316207886,
      "step": 2
    },
    {
      "epoch": 0.004405286343612335,
      "fcm_dpo/beta": 0.10943304747343063,
      "fcm_dpo/delta": 0.29740938544273376,
      "fcm_dpo/margin": 0.009920269250869751,
      "fcm_dpo/q_t": 0.4997372329235077,
      "grad_norm": 77.7017822265625,
      "learning_rate": 1.4492753623188406e-08,
      "logits/chosen": -0.4817405045032501,
      "logits/rejected": -0.4422529935836792,
      "logps/chosen": -60.98210144042969,
      "logps/ref_chosen": -60.981597900390625,
      "logps/ref_rejected": -68.67259216308594,
      "logps/rejected": -68.68302154541016,
      "loss": 1.3855,
      "margin_dpo/margin_mean": 0.009920388460159302,
      "margin_dpo/margin_std": 0.3543340563774109,
      "step": 3
    },
    {
      "epoch": 0.005873715124816446,
      "fcm_dpo/beta": 0.1161462590098381,
      "fcm_dpo/delta": 0.29794785380363464,
      "fcm_dpo/margin": -0.01907077431678772,
      "fcm_dpo/q_t": 0.5005367994308472,
      "grad_norm": 83.44183349609375,
      "learning_rate": 2.1739130434782606e-08,
      "logits/chosen": -0.46823394298553467,
      "logits/rejected": -0.44061601161956787,
      "logps/chosen": -56.77912139892578,
      "logps/ref_chosen": -56.7677116394043,
      "logps/ref_rejected": -86.64710998535156,
      "logps/rejected": -86.63945007324219,
      "loss": 1.3887,
      "margin_dpo/margin_mean": -0.01907128095626831,
      "margin_dpo/margin_std": 0.34393125772476196,
      "step": 4
    },
    {
      "epoch": 0.007342143906020558,
      "fcm_dpo/beta": 0.13080811500549316,
      "fcm_dpo/delta": 0.5949947834014893,
      "fcm_dpo/margin": 0.04144471883773804,
      "fcm_dpo/q_t": 0.4987483024597168,
      "grad_norm": 116.84333038330078,
      "learning_rate": 2.898550724637681e-08,
      "logits/chosen": -0.5144734978675842,
      "logits/rejected": -0.4707837700843811,
      "logps/chosen": -53.85641860961914,
      "logps/ref_chosen": -53.859375,
      "logps/ref_rejected": -84.14918518066406,
      "logps/rejected": -84.18767547607422,
      "loss": 1.3815,
      "margin_dpo/margin_mean": 0.041444987058639526,
      "margin_dpo/margin_std": 0.32837140560150146,
      "step": 5
    },
    {
      "epoch": 0.00881057268722467,
      "fcm_dpo/beta": 0.13887986540794373,
      "fcm_dpo/delta": 0.2996029257774353,
      "fcm_dpo/margin": -0.049517929553985596,
      "fcm_dpo/q_t": 0.5016666650772095,
      "grad_norm": 127.5752182006836,
      "learning_rate": 3.6231884057971014e-08,
      "logits/chosen": -0.5032555460929871,
      "logits/rejected": -0.4607737958431244,
      "logps/chosen": -63.0675048828125,
      "logps/ref_chosen": -63.007484436035156,
      "logps/ref_rejected": -92.64534759521484,
      "logps/rejected": -92.65585327148438,
      "loss": 1.3938,
      "margin_dpo/margin_mean": -0.04951813817024231,
      "margin_dpo/margin_std": 0.41555964946746826,
      "step": 6
    },
    {
      "epoch": 0.010279001468428781,
      "fcm_dpo/beta": 0.15183104574680328,
      "fcm_dpo/delta": 0.29823803901672363,
      "fcm_dpo/margin": -0.0031449496746063232,
      "fcm_dpo/q_t": 0.5001453161239624,
      "grad_norm": 124.75188446044922,
      "learning_rate": 4.347826086956521e-08,
      "logits/chosen": -0.4999721646308899,
      "logits/rejected": -0.466085284948349,
      "logps/chosen": -57.7862548828125,
      "logps/ref_chosen": -57.774818420410156,
      "logps/ref_rejected": -103.92059326171875,
      "logps/rejected": -103.92887878417969,
      "loss": 1.3877,
      "margin_dpo/margin_mean": -0.0031441152095794678,
      "margin_dpo/margin_std": 0.39584606885910034,
      "step": 7
    },
    {
      "epoch": 0.011747430249632892,
      "fcm_dpo/beta": 0.1610293984413147,
      "fcm_dpo/delta": 0.2940930426120758,
      "fcm_dpo/margin": 0.014331787824630737,
      "fcm_dpo/q_t": 0.4995083510875702,
      "grad_norm": 126.80061340332031,
      "learning_rate": 5.0724637681159424e-08,
      "logits/chosen": -0.5133894085884094,
      "logits/rejected": -0.4880775511264801,
      "logps/chosen": -58.66931915283203,
      "logps/ref_chosen": -58.716033935546875,
      "logps/ref_rejected": -79.3114242553711,
      "logps/rejected": -79.27903747558594,
      "loss": 1.385,
      "margin_dpo/margin_mean": 0.014332354068756104,
      "margin_dpo/margin_std": 0.381694495677948,
      "step": 8
    },
    {
      "epoch": 0.013215859030837005,
      "fcm_dpo/beta": 0.17621421813964844,
      "fcm_dpo/delta": 0.5977947115898132,
      "fcm_dpo/margin": 0.013306349515914917,
      "fcm_dpo/q_t": 0.4994489848613739,
      "grad_norm": 150.93214416503906,
      "learning_rate": 5.797101449275362e-08,
      "logits/chosen": -0.4874652028083801,
      "logits/rejected": -0.44039708375930786,
      "logps/chosen": -69.85830688476562,
      "logps/ref_chosen": -69.8668441772461,
      "logps/ref_rejected": -99.6026611328125,
      "logps/rejected": -99.60743713378906,
      "loss": 1.3852,
      "margin_dpo/margin_mean": 0.013305962085723877,
      "margin_dpo/margin_std": 0.3926519751548767,
      "step": 9
    },
    {
      "epoch": 0.014684287812041116,
      "fcm_dpo/beta": 0.18701426684856415,
      "fcm_dpo/delta": 0.2960095703601837,
      "fcm_dpo/margin": 0.019543275237083435,
      "fcm_dpo/q_t": 0.4991165101528168,
      "grad_norm": 132.65455627441406,
      "learning_rate": 6.521739130434782e-08,
      "logits/chosen": -0.4857853651046753,
      "logits/rejected": -0.4421781301498413,
      "logps/chosen": -48.35417938232422,
      "logps/ref_chosen": -48.35768508911133,
      "logps/ref_rejected": -80.37206268310547,
      "logps/rejected": -80.38810729980469,
      "loss": 1.3838,
      "margin_dpo/margin_mean": 0.01954329013824463,
      "margin_dpo/margin_std": 0.37247973680496216,
      "step": 10
    },
    {
      "epoch": 0.016152716593245228,
      "fcm_dpo/beta": 0.20438191294670105,
      "fcm_dpo/delta": 0.29821285605430603,
      "fcm_dpo/margin": -0.0019693076610565186,
      "fcm_dpo/q_t": 0.5001306533813477,
      "grad_norm": 140.15432739257812,
      "learning_rate": 7.246376811594203e-08,
      "logits/chosen": -0.4591175615787506,
      "logits/rejected": -0.43393969535827637,
      "logps/chosen": -53.05950927734375,
      "logps/ref_chosen": -53.01685333251953,
      "logps/ref_rejected": -87.78038024902344,
      "logps/rejected": -87.82106018066406,
      "loss": 1.388,
      "margin_dpo/margin_mean": -0.001968860626220703,
      "margin_dpo/margin_std": 0.34557127952575684,
      "step": 11
    },
    {
      "epoch": 0.01762114537444934,
      "fcm_dpo/beta": 0.2106446623802185,
      "fcm_dpo/delta": 0.29740071296691895,
      "fcm_dpo/margin": 0.00841417908668518,
      "fcm_dpo/q_t": 0.4995623826980591,
      "grad_norm": 188.97665405273438,
      "learning_rate": 7.971014492753623e-08,
      "logits/chosen": -0.5176132321357727,
      "logits/rejected": -0.4807056188583374,
      "logps/chosen": -61.83100128173828,
      "logps/ref_chosen": -61.80543518066406,
      "logps/ref_rejected": -104.8582763671875,
      "logps/rejected": -104.89225769042969,
      "loss": 1.3861,
      "margin_dpo/margin_mean": 0.008413076400756836,
      "margin_dpo/margin_std": 0.3932174742221832,
      "step": 12
    },
    {
      "epoch": 0.01908957415565345,
      "fcm_dpo/beta": 0.2233951985836029,
      "fcm_dpo/delta": 0.2904997766017914,
      "fcm_dpo/margin": 0.003662973642349243,
      "fcm_dpo/q_t": 0.4998163878917694,
      "grad_norm": 176.16429138183594,
      "learning_rate": 8.695652173913042e-08,
      "logits/chosen": -0.4700925946235657,
      "logits/rejected": -0.4411465525627136,
      "logps/chosen": -64.25465393066406,
      "logps/ref_chosen": -64.2603530883789,
      "logps/ref_rejected": -87.20307922363281,
      "logps/rejected": -87.20104217529297,
      "loss": 1.387,
      "margin_dpo/margin_mean": 0.0036628246307373047,
      "margin_dpo/margin_std": 0.39085596799850464,
      "step": 13
    },
    {
      "epoch": 0.020558002936857563,
      "fcm_dpo/beta": 0.24400313198566437,
      "fcm_dpo/delta": 0.29805275797843933,
      "fcm_dpo/margin": 0.005518615245819092,
      "fcm_dpo/q_t": 0.499683141708374,
      "grad_norm": 209.18350219726562,
      "learning_rate": 9.420289855072464e-08,
      "logits/chosen": -0.47941046953201294,
      "logits/rejected": -0.4401160478591919,
      "logps/chosen": -58.11064910888672,
      "logps/ref_chosen": -58.11021041870117,
      "logps/ref_rejected": -104.04708099365234,
      "logps/rejected": -104.05303192138672,
      "loss": 1.3883,
      "margin_dpo/margin_mean": 0.005518555641174316,
      "margin_dpo/margin_std": 0.47164684534072876,
      "step": 14
    },
    {
      "epoch": 0.022026431718061675,
      "fcm_dpo/beta": 0.25842905044555664,
      "fcm_dpo/delta": 0.2872004508972168,
      "fcm_dpo/margin": 0.03296121954917908,
      "fcm_dpo/q_t": 0.49806299805641174,
      "grad_norm": 164.04367065429688,
      "learning_rate": 1.0144927536231885e-07,
      "logits/chosen": -0.5029046535491943,
      "logits/rejected": -0.4847126603126526,
      "logps/chosen": -56.98225402832031,
      "logps/ref_chosen": -56.96691131591797,
      "logps/ref_rejected": -80.80863952636719,
      "logps/rejected": -80.85694885253906,
      "loss": 1.3795,
      "margin_dpo/margin_mean": 0.03296104073524475,
      "margin_dpo/margin_std": 0.3130000829696655,
      "step": 15
    },
    {
      "epoch": 0.023494860499265784,
      "fcm_dpo/beta": 0.2661767303943634,
      "fcm_dpo/delta": 0.2911548614501953,
      "fcm_dpo/margin": 0.006061911582946777,
      "fcm_dpo/q_t": 0.49960726499557495,
      "grad_norm": 225.11155700683594,
      "learning_rate": 1.0869565217391303e-07,
      "logits/chosen": -0.5254543423652649,
      "logits/rejected": -0.48440250754356384,
      "logps/chosen": -61.74738693237305,
      "logps/ref_chosen": -61.739891052246094,
      "logps/ref_rejected": -84.36947631835938,
      "logps/rejected": -84.38302612304688,
      "loss": 1.3865,
      "margin_dpo/margin_mean": 0.0060617029666900635,
      "margin_dpo/margin_std": 0.35263463854789734,
      "step": 16
    },
    {
      "epoch": 0.024963289280469897,
      "fcm_dpo/beta": 0.29960495233535767,
      "fcm_dpo/delta": 0.5921929478645325,
      "fcm_dpo/margin": 0.02691623568534851,
      "fcm_dpo/q_t": 0.49804800748825073,
      "grad_norm": 234.62332153320312,
      "learning_rate": 1.1594202898550725e-07,
      "logits/chosen": -0.4995841979980469,
      "logits/rejected": -0.4622904658317566,
      "logps/chosen": -67.72857666015625,
      "logps/ref_chosen": -67.71033477783203,
      "logps/ref_rejected": -85.37865447998047,
      "logps/rejected": -85.42382049560547,
      "loss": 1.3808,
      "margin_dpo/margin_mean": 0.02691650390625,
      "margin_dpo/margin_std": 0.35206741094589233,
      "step": 17
    },
    {
      "epoch": 0.02643171806167401,
      "fcm_dpo/beta": 0.3366771340370178,
      "fcm_dpo/delta": 0.5766884088516235,
      "fcm_dpo/margin": 0.07164651155471802,
      "fcm_dpo/q_t": 0.4941858947277069,
      "grad_norm": 273.0143127441406,
      "learning_rate": 1.2318840579710146e-07,
      "logits/chosen": -0.5020445585250854,
      "logits/rejected": -0.44765499234199524,
      "logps/chosen": -47.74727249145508,
      "logps/ref_chosen": -47.7394905090332,
      "logps/ref_rejected": -75.4722900390625,
      "logps/rejected": -75.55171966552734,
      "loss": 1.3652,
      "margin_dpo/margin_mean": 0.07164624333381653,
      "margin_dpo/margin_std": 0.3414478302001953,
      "step": 18
    },
    {
      "epoch": 0.027900146842878122,
      "fcm_dpo/beta": 0.37680280208587646,
      "fcm_dpo/delta": 0.5566200017929077,
      "fcm_dpo/margin": 0.11977455019950867,
      "fcm_dpo/q_t": 0.4892091155052185,
      "grad_norm": 273.0553894042969,
      "learning_rate": 1.3043478260869563e-07,
      "logits/chosen": -0.4712415337562561,
      "logits/rejected": -0.41980695724487305,
      "logps/chosen": -70.16747283935547,
      "logps/ref_chosen": -70.20536041259766,
      "logps/ref_rejected": -89.7575912475586,
      "logps/rejected": -89.83948516845703,
      "loss": 1.3457,
      "margin_dpo/margin_mean": 0.11977550387382507,
      "margin_dpo/margin_std": 0.3553928732872009,
      "step": 19
    },
    {
      "epoch": 0.02936857562408223,
      "fcm_dpo/beta": 0.39811867475509644,
      "fcm_dpo/delta": 0.2839474380016327,
      "fcm_dpo/margin": 0.03722500801086426,
      "fcm_dpo/q_t": 0.49641382694244385,
      "grad_norm": 294.3759460449219,
      "learning_rate": 1.3768115942028986e-07,
      "logits/chosen": -0.5450824499130249,
      "logits/rejected": -0.48803430795669556,
      "logps/chosen": -50.81207275390625,
      "logps/ref_chosen": -50.80324172973633,
      "logps/ref_rejected": -78.82334899902344,
      "logps/rejected": -78.86940002441406,
      "loss": 1.3748,
      "margin_dpo/margin_mean": 0.037224918603897095,
      "margin_dpo/margin_std": 0.3072945773601532,
      "step": 20
    },
    {
      "epoch": 0.030837004405286344,
      "fcm_dpo/beta": 0.443250834941864,
      "fcm_dpo/delta": 0.5273383259773254,
      "fcm_dpo/margin": 0.1729957014322281,
      "fcm_dpo/q_t": 0.48192811012268066,
      "grad_norm": 327.8818054199219,
      "learning_rate": 1.4492753623188405e-07,
      "logits/chosen": -0.517810046672821,
      "logits/rejected": -0.49564769864082336,
      "logps/chosen": -50.032989501953125,
      "logps/ref_chosen": -50.063018798828125,
      "logps/ref_rejected": -77.86878967285156,
      "logps/rejected": -78.01175689697266,
      "loss": 1.3156,
      "margin_dpo/margin_mean": 0.1729956418275833,
      "margin_dpo/margin_std": 0.3008769452571869,
      "step": 21
    },
    {
      "epoch": 0.032305433186490456,
      "fcm_dpo/beta": 0.49217575788497925,
      "fcm_dpo/delta": 0.4990707039833069,
      "fcm_dpo/margin": 0.21139326691627502,
      "fcm_dpo/q_t": 0.4752381443977356,
      "grad_norm": 389.9123229980469,
      "learning_rate": 1.5217391304347825e-07,
      "logits/chosen": -0.48431238532066345,
      "logits/rejected": -0.4406638741493225,
      "logps/chosen": -58.97994613647461,
      "logps/ref_chosen": -59.05763626098633,
      "logps/ref_rejected": -97.50466918945312,
      "logps/rejected": -97.63837432861328,
      "loss": 1.2977,
      "margin_dpo/margin_mean": 0.21139344573020935,
      "margin_dpo/margin_std": 0.46386557817459106,
      "step": 22
    },
    {
      "epoch": 0.033773861967694566,
      "fcm_dpo/beta": 0.5401284694671631,
      "fcm_dpo/delta": 0.4757743179798126,
      "fcm_dpo/margin": 0.24120861291885376,
      "fcm_dpo/q_t": 0.469452828168869,
      "grad_norm": 389.5323791503906,
      "learning_rate": 1.5942028985507245e-07,
      "logits/chosen": -0.4874040186405182,
      "logits/rejected": -0.4647292494773865,
      "logps/chosen": -60.02528381347656,
      "logps/ref_chosen": -60.07769775390625,
      "logps/ref_rejected": -81.13955688476562,
      "logps/rejected": -81.3283462524414,
      "loss": 1.2741,
      "margin_dpo/margin_mean": 0.24120870232582092,
      "margin_dpo/margin_std": 0.418229341506958,
      "step": 23
    },
    {
      "epoch": 0.03524229074889868,
      "fcm_dpo/beta": 0.5957721471786499,
      "fcm_dpo/delta": 0.4795808792114258,
      "fcm_dpo/margin": 0.20983095467090607,
      "fcm_dpo/q_t": 0.47032594680786133,
      "grad_norm": 471.896728515625,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -0.5221496820449829,
      "logits/rejected": -0.5054002404212952,
      "logps/chosen": -44.25273895263672,
      "logps/ref_chosen": -44.29103469848633,
      "logps/ref_rejected": -99.12521362304688,
      "logps/rejected": -99.2967529296875,
      "loss": 1.2789,
      "margin_dpo/margin_mean": 0.20983177423477173,
      "margin_dpo/margin_std": 0.3987899422645569,
      "step": 24
    },
    {
      "epoch": 0.03671071953010279,
      "fcm_dpo/beta": 0.6493321657180786,
      "fcm_dpo/delta": 0.4309549033641815,
      "fcm_dpo/margin": 0.27194589376449585,
      "fcm_dpo/q_t": 0.4587538540363312,
      "grad_norm": 428.5508117675781,
      "learning_rate": 1.7391304347826085e-07,
      "logits/chosen": -0.4805753827095032,
      "logits/rejected": -0.4500219225883484,
      "logps/chosen": -52.49953842163086,
      "logps/ref_chosen": -52.537052154541016,
      "logps/ref_rejected": -89.34219360351562,
      "logps/rejected": -89.57662200927734,
      "loss": 1.2366,
      "margin_dpo/margin_mean": 0.27194616198539734,
      "margin_dpo/margin_std": 0.4171033203601837,
      "step": 25
    },
    {
      "epoch": 0.0381791483113069,
      "fcm_dpo/beta": 0.6972817182540894,
      "fcm_dpo/delta": 0.3234151601791382,
      "fcm_dpo/margin": 0.4091455340385437,
      "fcm_dpo/q_t": 0.4343709349632263,
      "grad_norm": 487.4859619140625,
      "learning_rate": 1.8115942028985507e-07,
      "logits/chosen": -0.5440469980239868,
      "logits/rejected": -0.5126461386680603,
      "logps/chosen": -53.82080078125,
      "logps/ref_chosen": -53.92280578613281,
      "logps/ref_rejected": -103.35971069335938,
      "logps/rejected": -103.66685485839844,
      "loss": 1.1593,
      "margin_dpo/margin_mean": 0.4091459810733795,
      "margin_dpo/margin_std": 0.5674811601638794,
      "step": 26
    },
    {
      "epoch": 0.039647577092511016,
      "fcm_dpo/beta": 0.7315391302108765,
      "fcm_dpo/delta": 0.22802981734275818,
      "fcm_dpo/margin": 0.5205895304679871,
      "fcm_dpo/q_t": 0.41203832626342773,
      "grad_norm": 530.424560546875,
      "learning_rate": 1.8840579710144927e-07,
      "logits/chosen": -0.5132520198822021,
      "logits/rejected": -0.47679802775382996,
      "logps/chosen": -42.7576904296875,
      "logps/ref_chosen": -42.898529052734375,
      "logps/ref_rejected": -98.72419738769531,
      "logps/rejected": -99.10395812988281,
      "loss": 1.0773,
      "margin_dpo/margin_mean": 0.5205891132354736,
      "margin_dpo/margin_std": 0.5357187986373901,
      "step": 27
    },
    {
      "epoch": 0.041116005873715125,
      "fcm_dpo/beta": 0.7871224284172058,
      "fcm_dpo/delta": 0.3837660551071167,
      "fcm_dpo/margin": 0.28373512625694275,
      "fcm_dpo/q_t": 0.44818979501724243,
      "grad_norm": 503.4446105957031,
      "learning_rate": 1.9565217391304347e-07,
      "logits/chosen": -0.5058661699295044,
      "logits/rejected": -0.4506160616874695,
      "logps/chosen": -60.58196258544922,
      "logps/ref_chosen": -60.55650329589844,
      "logps/ref_rejected": -91.40111541748047,
      "logps/rejected": -91.71031188964844,
      "loss": 1.2172,
      "margin_dpo/margin_mean": 0.2837350368499756,
      "margin_dpo/margin_std": 0.5103665590286255,
      "step": 28
    },
    {
      "epoch": 0.042584434654919234,
      "fcm_dpo/beta": 0.8140842914581299,
      "fcm_dpo/delta": 0.12859481573104858,
      "fcm_dpo/margin": 0.5867732763290405,
      "fcm_dpo/q_t": 0.39105674624443054,
      "grad_norm": 521.9832763671875,
      "learning_rate": 2.028985507246377e-07,
      "logits/chosen": -0.5453035831451416,
      "logits/rejected": -0.498327374458313,
      "logps/chosen": -57.687744140625,
      "logps/ref_chosen": -57.80778503417969,
      "logps/ref_rejected": -97.39434814453125,
      "logps/rejected": -97.86107635498047,
      "loss": 1.0177,
      "margin_dpo/margin_mean": 0.5867743492126465,
      "margin_dpo/margin_std": 0.5909589529037476,
      "step": 29
    },
    {
      "epoch": 0.04405286343612335,
      "fcm_dpo/beta": 0.8216707706451416,
      "fcm_dpo/delta": 0.0340820774435997,
      "fcm_dpo/margin": 0.690904438495636,
      "fcm_dpo/q_t": 0.3710797429084778,
      "grad_norm": 468.0931396484375,
      "learning_rate": 2.1014492753623187e-07,
      "logits/chosen": -0.4391070306301117,
      "logits/rejected": -0.40617626905441284,
      "logps/chosen": -52.45497131347656,
      "logps/ref_chosen": -52.577369689941406,
      "logps/ref_rejected": -98.48920440673828,
      "logps/rejected": -99.05770874023438,
      "loss": 0.9603,
      "margin_dpo/margin_mean": 0.6909043788909912,
      "margin_dpo/margin_std": 0.6301661729812622,
      "step": 30
    },
    {
      "epoch": 0.04552129221732746,
      "fcm_dpo/beta": 0.8536430597305298,
      "fcm_dpo/delta": 0.21446718275547028,
      "fcm_dpo/margin": 0.46136218309402466,
      "fcm_dpo/q_t": 0.4132009744644165,
      "grad_norm": 441.92120361328125,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": -0.5114755034446716,
      "logits/rejected": -0.46733975410461426,
      "logps/chosen": -63.724403381347656,
      "logps/ref_chosen": -63.806922912597656,
      "logps/ref_rejected": -72.89400482177734,
      "logps/rejected": -73.2728500366211,
      "loss": 1.0973,
      "margin_dpo/margin_mean": 0.46136197447776794,
      "margin_dpo/margin_std": 0.6410406827926636,
      "step": 31
    },
    {
      "epoch": 0.04698972099853157,
      "fcm_dpo/beta": 0.8558568954467773,
      "fcm_dpo/delta": -0.04837347939610481,
      "fcm_dpo/margin": 0.7538948655128479,
      "fcm_dpo/q_t": 0.363816499710083,
      "grad_norm": 431.97833251953125,
      "learning_rate": 2.2463768115942027e-07,
      "logits/chosen": -0.5344721078872681,
      "logits/rejected": -0.4945007562637329,
      "logps/chosen": -62.547889709472656,
      "logps/ref_chosen": -62.739524841308594,
      "logps/ref_rejected": -89.3175048828125,
      "logps/rejected": -89.8797607421875,
      "loss": 0.95,
      "margin_dpo/margin_mean": 0.7538947463035583,
      "margin_dpo/margin_std": 0.8735177516937256,
      "step": 32
    },
    {
      "epoch": 0.048458149779735685,
      "fcm_dpo/beta": 0.8686736822128296,
      "fcm_dpo/delta": 0.07413993775844574,
      "fcm_dpo/margin": 0.608627438545227,
      "fcm_dpo/q_t": 0.3796751797199249,
      "grad_norm": 434.5819091796875,
      "learning_rate": 2.318840579710145e-07,
      "logits/chosen": -0.48133665323257446,
      "logits/rejected": -0.4549505412578583,
      "logps/chosen": -53.15459442138672,
      "logps/ref_chosen": -53.26097106933594,
      "logps/ref_rejected": -87.8851318359375,
      "logps/rejected": -88.38739013671875,
      "loss": 0.9889,
      "margin_dpo/margin_mean": 0.6086278557777405,
      "margin_dpo/margin_std": 0.5917040109634399,
      "step": 33
    },
    {
      "epoch": 0.049926578560939794,
      "fcm_dpo/beta": 0.8607486486434937,
      "fcm_dpo/delta": -0.029908865690231323,
      "fcm_dpo/margin": 0.7293862104415894,
      "fcm_dpo/q_t": 0.3666630983352661,
      "grad_norm": 413.73388671875,
      "learning_rate": 2.391304347826087e-07,
      "logits/chosen": -0.5056343078613281,
      "logits/rejected": -0.4880971908569336,
      "logps/chosen": -50.74631118774414,
      "logps/ref_chosen": -50.81732940673828,
      "logps/ref_rejected": -101.92184448242188,
      "logps/rejected": -102.58021545410156,
      "loss": 0.9715,
      "margin_dpo/margin_mean": 0.7293854355812073,
      "margin_dpo/margin_std": 0.8646777272224426,
      "step": 34
    },
    {
      "epoch": 0.0513950073421439,
      "fcm_dpo/beta": 0.8129470348358154,
      "fcm_dpo/delta": -0.3546205759048462,
      "fcm_dpo/margin": 1.1309483051300049,
      "fcm_dpo/q_t": 0.3066979646682739,
      "grad_norm": 335.1808166503906,
      "learning_rate": 2.463768115942029e-07,
      "logits/chosen": -0.5237250328063965,
      "logits/rejected": -0.48680445551872253,
      "logps/chosen": -50.88426208496094,
      "logps/ref_chosen": -51.02449035644531,
      "logps/ref_rejected": -106.82443237304688,
      "logps/rejected": -107.81513977050781,
      "loss": 0.7836,
      "margin_dpo/margin_mean": 1.1309483051300049,
      "margin_dpo/margin_std": 1.0144567489624023,
      "step": 35
    },
    {
      "epoch": 0.05286343612334802,
      "fcm_dpo/beta": 0.7814772129058838,
      "fcm_dpo/delta": -0.24869973957538605,
      "fcm_dpo/margin": 1.0601816177368164,
      "fcm_dpo/q_t": 0.3314136862754822,
      "grad_norm": 286.6738586425781,
      "learning_rate": 2.536231884057971e-07,
      "logits/chosen": -0.5335814952850342,
      "logits/rejected": -0.4959707260131836,
      "logps/chosen": -51.95500946044922,
      "logps/ref_chosen": -51.991493225097656,
      "logps/ref_rejected": -86.0406265258789,
      "logps/rejected": -87.06432342529297,
      "loss": 0.8938,
      "margin_dpo/margin_mean": 1.0601813793182373,
      "margin_dpo/margin_std": 1.1594077348709106,
      "step": 36
    },
    {
      "epoch": 0.05433186490455213,
      "fcm_dpo/beta": 0.7387670278549194,
      "fcm_dpo/delta": -0.1298440545797348,
      "fcm_dpo/margin": 0.9678917527198792,
      "fcm_dpo/q_t": 0.35818910598754883,
      "grad_norm": 304.4512939453125,
      "learning_rate": 2.6086956521739126e-07,
      "logits/chosen": -0.5167920589447021,
      "logits/rejected": -0.4735545217990875,
      "logps/chosen": -62.80792999267578,
      "logps/ref_chosen": -62.807106018066406,
      "logps/ref_rejected": -77.89507293701172,
      "logps/rejected": -78.86378479003906,
      "loss": 0.982,
      "margin_dpo/margin_mean": 0.9678912162780762,
      "margin_dpo/margin_std": 1.311614751815796,
      "step": 37
    },
    {
      "epoch": 0.055800293685756244,
      "fcm_dpo/beta": 0.7068374156951904,
      "fcm_dpo/delta": -0.37072476744651794,
      "fcm_dpo/margin": 1.3253750801086426,
      "fcm_dpo/q_t": 0.32258689403533936,
      "grad_norm": 279.52642822265625,
      "learning_rate": 2.681159420289855e-07,
      "logits/chosen": -0.5299612879753113,
      "logits/rejected": -0.49737080931663513,
      "logps/chosen": -48.24340057373047,
      "logps/ref_chosen": -48.39051818847656,
      "logps/ref_rejected": -97.91244506835938,
      "logps/rejected": -99.0906982421875,
      "loss": 0.8506,
      "margin_dpo/margin_mean": 1.3253748416900635,
      "margin_dpo/margin_std": 1.5660966634750366,
      "step": 38
    },
    {
      "epoch": 0.05726872246696035,
      "fcm_dpo/beta": 0.6403067111968994,
      "fcm_dpo/delta": -0.4625703692436218,
      "fcm_dpo/margin": 1.5791468620300293,
      "fcm_dpo/q_t": 0.2878713607788086,
      "grad_norm": 240.9384765625,
      "learning_rate": 2.753623188405797e-07,
      "logits/chosen": -0.5889618396759033,
      "logits/rejected": -0.5511986613273621,
      "logps/chosen": -50.64854431152344,
      "logps/ref_chosen": -50.75047302246094,
      "logps/ref_rejected": -78.56951141357422,
      "logps/rejected": -80.0467300415039,
      "loss": 0.7516,
      "margin_dpo/margin_mean": 1.5791466236114502,
      "margin_dpo/margin_std": 1.3109757900238037,
      "step": 39
    },
    {
      "epoch": 0.05873715124816446,
      "fcm_dpo/beta": 0.6059188842773438,
      "fcm_dpo/delta": -0.3013249933719635,
      "fcm_dpo/margin": 1.4455113410949707,
      "fcm_dpo/q_t": 0.3245118260383606,
      "grad_norm": 211.55166625976562,
      "learning_rate": 2.8260869565217386e-07,
      "logits/chosen": -0.5337985754013062,
      "logits/rejected": -0.5050651431083679,
      "logps/chosen": -57.77853012084961,
      "logps/ref_chosen": -57.985069274902344,
      "logps/ref_rejected": -74.3000717163086,
      "logps/rejected": -75.53904724121094,
      "loss": 0.8502,
      "margin_dpo/margin_mean": 1.445512294769287,
      "margin_dpo/margin_std": 1.521177053451538,
      "step": 40
    },
    {
      "epoch": 0.06020558002936858,
      "fcm_dpo/beta": 0.5612444877624512,
      "fcm_dpo/delta": -0.3904670178890228,
      "fcm_dpo/margin": 1.6995972394943237,
      "fcm_dpo/q_t": 0.31129536032676697,
      "grad_norm": 220.2853546142578,
      "learning_rate": 2.898550724637681e-07,
      "logits/chosen": -0.5450239777565002,
      "logits/rejected": -0.5087831020355225,
      "logps/chosen": -62.68452835083008,
      "logps/ref_chosen": -62.69581604003906,
      "logps/ref_rejected": -97.02352905273438,
      "logps/rejected": -98.71183776855469,
      "loss": 0.8303,
      "margin_dpo/margin_mean": 1.6995975971221924,
      "margin_dpo/margin_std": 1.7764086723327637,
      "step": 41
    },
    {
      "epoch": 0.06167400881057269,
      "fcm_dpo/beta": 0.4963580369949341,
      "fcm_dpo/delta": -0.6783915758132935,
      "fcm_dpo/margin": 2.4108240604400635,
      "fcm_dpo/q_t": 0.2684577703475952,
      "grad_norm": 179.24830627441406,
      "learning_rate": 2.971014492753623e-07,
      "logits/chosen": -0.5304057598114014,
      "logits/rejected": -0.48365843296051025,
      "logps/chosen": -58.75749588012695,
      "logps/ref_chosen": -58.966426849365234,
      "logps/ref_rejected": -109.90837097167969,
      "logps/rejected": -112.11026000976562,
      "loss": 0.6984,
      "margin_dpo/margin_mean": 2.4108238220214844,
      "margin_dpo/margin_std": 2.168299436569214,
      "step": 42
    },
    {
      "epoch": 0.0631424375917768,
      "fcm_dpo/beta": 0.44308847188949585,
      "fcm_dpo/delta": -0.48201635479927063,
      "fcm_dpo/margin": 2.3211944103240967,
      "fcm_dpo/q_t": 0.2817467451095581,
      "grad_norm": 169.86190795898438,
      "learning_rate": 3.043478260869565e-07,
      "logits/chosen": -0.5538966059684753,
      "logits/rejected": -0.5297778844833374,
      "logps/chosen": -53.645652770996094,
      "logps/ref_chosen": -54.15599822998047,
      "logps/ref_rejected": -96.48019409179688,
      "logps/rejected": -98.29104614257812,
      "loss": 0.7188,
      "margin_dpo/margin_mean": 2.3211941719055176,
      "margin_dpo/margin_std": 1.7973530292510986,
      "step": 43
    },
    {
      "epoch": 0.06461086637298091,
      "fcm_dpo/beta": 0.4017961323261261,
      "fcm_dpo/delta": -0.49928557872772217,
      "fcm_dpo/margin": 2.597970485687256,
      "fcm_dpo/q_t": 0.2795543372631073,
      "grad_norm": 164.8052215576172,
      "learning_rate": 3.115942028985507e-07,
      "logits/chosen": -0.4567152261734009,
      "logits/rejected": -0.43692851066589355,
      "logps/chosen": -49.864402770996094,
      "logps/ref_chosen": -50.07849884033203,
      "logps/ref_rejected": -108.78376007080078,
      "logps/rejected": -111.16763305664062,
      "loss": 0.713,
      "margin_dpo/margin_mean": 2.597970962524414,
      "margin_dpo/margin_std": 1.9955940246582031,
      "step": 44
    },
    {
      "epoch": 0.06607929515418502,
      "fcm_dpo/beta": 0.3842885196208954,
      "fcm_dpo/delta": -0.18539269268512726,
      "fcm_dpo/margin": 2.0074782371520996,
      "fcm_dpo/q_t": 0.3449398875236511,
      "grad_norm": 142.70497131347656,
      "learning_rate": 3.188405797101449e-07,
      "logits/chosen": -0.4787759780883789,
      "logits/rejected": -0.46609732508659363,
      "logps/chosen": -48.27836227416992,
      "logps/ref_chosen": -48.4149284362793,
      "logps/ref_rejected": -77.93643188476562,
      "logps/rejected": -79.80734252929688,
      "loss": 0.9085,
      "margin_dpo/margin_mean": 2.0074782371520996,
      "margin_dpo/margin_std": 2.3263068199157715,
      "step": 45
    },
    {
      "epoch": 0.06754772393538913,
      "fcm_dpo/beta": 0.3586476445198059,
      "fcm_dpo/delta": -0.3937702775001526,
      "fcm_dpo/margin": 2.6675519943237305,
      "fcm_dpo/q_t": 0.31235700845718384,
      "grad_norm": 143.95310974121094,
      "learning_rate": 3.260869565217391e-07,
      "logits/chosen": -0.5191431045532227,
      "logits/rejected": -0.46773868799209595,
      "logps/chosen": -55.786354064941406,
      "logps/ref_chosen": -55.999427795410156,
      "logps/ref_rejected": -95.652587890625,
      "logps/rejected": -98.10707092285156,
      "loss": 0.8258,
      "margin_dpo/margin_mean": 2.6675515174865723,
      "margin_dpo/margin_std": 2.8206050395965576,
      "step": 46
    },
    {
      "epoch": 0.06901615271659324,
      "fcm_dpo/beta": 0.33721715211868286,
      "fcm_dpo/delta": -0.31360626220703125,
      "fcm_dpo/margin": 2.6299524307250977,
      "fcm_dpo/q_t": 0.31478771567344666,
      "grad_norm": 143.6584930419922,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -0.5552619695663452,
      "logits/rejected": -0.5014165639877319,
      "logps/chosen": -57.52300262451172,
      "logps/ref_chosen": -57.92607879638672,
      "logps/ref_rejected": -94.67920684814453,
      "logps/rejected": -96.90608215332031,
      "loss": 0.8165,
      "margin_dpo/margin_mean": 2.6299524307250977,
      "margin_dpo/margin_std": 2.3889541625976562,
      "step": 47
    },
    {
      "epoch": 0.07048458149779736,
      "fcm_dpo/beta": 0.31545713543891907,
      "fcm_dpo/delta": -0.23579037189483643,
      "fcm_dpo/margin": 2.579418659210205,
      "fcm_dpo/q_t": 0.3256019353866577,
      "grad_norm": 148.15615844726562,
      "learning_rate": 3.4057971014492755e-07,
      "logits/chosen": -0.584140956401825,
      "logits/rejected": -0.5260648131370544,
      "logps/chosen": -57.080162048339844,
      "logps/ref_chosen": -57.188072204589844,
      "logps/ref_rejected": -88.0166015625,
      "logps/rejected": -90.48811340332031,
      "loss": 0.8611,
      "margin_dpo/margin_mean": 2.5794191360473633,
      "margin_dpo/margin_std": 2.3878588676452637,
      "step": 48
    },
    {
      "epoch": 0.07195301027900147,
      "fcm_dpo/beta": 0.30197674036026,
      "fcm_dpo/delta": -0.30794399976730347,
      "fcm_dpo/margin": 2.9200515747070312,
      "fcm_dpo/q_t": 0.31857985258102417,
      "grad_norm": 391.1684265136719,
      "learning_rate": 3.478260869565217e-07,
      "logits/chosen": -0.5332294702529907,
      "logits/rejected": -0.47411999106407166,
      "logps/chosen": -61.26190948486328,
      "logps/ref_chosen": -61.685272216796875,
      "logps/ref_rejected": -83.76747131347656,
      "logps/rejected": -86.26417541503906,
      "loss": 0.8336,
      "margin_dpo/margin_mean": 2.9200520515441895,
      "margin_dpo/margin_std": 2.897745132446289,
      "step": 49
    },
    {
      "epoch": 0.07342143906020558,
      "fcm_dpo/beta": 0.2793174982070923,
      "fcm_dpo/delta": -0.3477798402309418,
      "fcm_dpo/margin": 3.2729971408843994,
      "fcm_dpo/q_t": 0.3099723756313324,
      "grad_norm": 119.61611938476562,
      "learning_rate": 3.5507246376811595e-07,
      "logits/chosen": -0.5257272124290466,
      "logits/rejected": -0.4897651672363281,
      "logps/chosen": -58.62849426269531,
      "logps/ref_chosen": -58.72413635253906,
      "logps/ref_rejected": -96.35814666748047,
      "logps/rejected": -99.5354995727539,
      "loss": 0.8143,
      "margin_dpo/margin_mean": 3.2729973793029785,
      "margin_dpo/margin_std": 3.099050521850586,
      "step": 50
    },
    {
      "epoch": 0.07488986784140969,
      "fcm_dpo/beta": 0.2618306875228882,
      "fcm_dpo/delta": -0.2724575996398926,
      "fcm_dpo/margin": 3.226135730743408,
      "fcm_dpo/q_t": 0.337465763092041,
      "grad_norm": 97.82896423339844,
      "learning_rate": 3.6231884057971015e-07,
      "logits/chosen": -0.5134993195533752,
      "logits/rejected": -0.48083585500717163,
      "logps/chosen": -61.43326950073242,
      "logps/ref_chosen": -61.3736686706543,
      "logps/ref_rejected": -76.00199890136719,
      "logps/rejected": -79.28773498535156,
      "loss": 0.9065,
      "margin_dpo/margin_mean": 3.2261362075805664,
      "margin_dpo/margin_std": 3.9684386253356934,
      "step": 51
    },
    {
      "epoch": 0.0763582966226138,
      "fcm_dpo/beta": 0.2358231097459793,
      "fcm_dpo/delta": -0.6680800914764404,
      "fcm_dpo/margin": 5.0320281982421875,
      "fcm_dpo/q_t": 0.2603490352630615,
      "grad_norm": 94.4676513671875,
      "learning_rate": 3.695652173913043e-07,
      "logits/chosen": -0.557870090007782,
      "logits/rejected": -0.5032240748405457,
      "logps/chosen": -51.786834716796875,
      "logps/ref_chosen": -52.33735656738281,
      "logps/ref_rejected": -79.97391510009766,
      "logps/rejected": -84.45541381835938,
      "loss": 0.6746,
      "margin_dpo/margin_mean": 5.0320281982421875,
      "margin_dpo/margin_std": 4.007593154907227,
      "step": 52
    },
    {
      "epoch": 0.07782672540381791,
      "fcm_dpo/beta": 0.21381568908691406,
      "fcm_dpo/delta": -0.5255783796310425,
      "fcm_dpo/margin": 5.014308452606201,
      "fcm_dpo/q_t": 0.298156201839447,
      "grad_norm": 91.9140853881836,
      "learning_rate": 3.7681159420289855e-07,
      "logits/chosen": -0.6126323342323303,
      "logits/rejected": -0.5912652015686035,
      "logps/chosen": -53.282958984375,
      "logps/ref_chosen": -53.31465148925781,
      "logps/ref_rejected": -91.78359985351562,
      "logps/rejected": -96.7662124633789,
      "loss": 0.7792,
      "margin_dpo/margin_mean": 5.014307975769043,
      "margin_dpo/margin_std": 4.957146644592285,
      "step": 53
    },
    {
      "epoch": 0.07929515418502203,
      "fcm_dpo/beta": 0.1964169442653656,
      "fcm_dpo/delta": -0.3215777575969696,
      "fcm_dpo/margin": 4.548583984375,
      "fcm_dpo/q_t": 0.3135607838630676,
      "grad_norm": 79.06949615478516,
      "learning_rate": 3.8405797101449274e-07,
      "logits/chosen": -0.5691754817962646,
      "logits/rejected": -0.5145428776741028,
      "logps/chosen": -50.683597564697266,
      "logps/ref_chosen": -50.68865966796875,
      "logps/ref_rejected": -91.71539306640625,
      "logps/rejected": -96.2589111328125,
      "loss": 0.8046,
      "margin_dpo/margin_mean": 4.548584461212158,
      "margin_dpo/margin_std": 4.153512954711914,
      "step": 54
    },
    {
      "epoch": 0.08076358296622614,
      "fcm_dpo/beta": 0.1820051074028015,
      "fcm_dpo/delta": -0.4129885137081146,
      "fcm_dpo/margin": 5.350862979888916,
      "fcm_dpo/q_t": 0.31068581342697144,
      "grad_norm": 79.86705780029297,
      "learning_rate": 3.9130434782608694e-07,
      "logits/chosen": -0.6421248912811279,
      "logits/rejected": -0.5801297426223755,
      "logps/chosen": -63.01799774169922,
      "logps/ref_chosen": -62.615234375,
      "logps/ref_rejected": -88.99349975585938,
      "logps/rejected": -94.74713134765625,
      "loss": 0.8358,
      "margin_dpo/margin_mean": 5.350862503051758,
      "margin_dpo/margin_std": 5.820072650909424,
      "step": 55
    },
    {
      "epoch": 0.08223201174743025,
      "fcm_dpo/beta": 0.17025861144065857,
      "fcm_dpo/delta": -0.2861342430114746,
      "fcm_dpo/margin": 5.0614013671875,
      "fcm_dpo/q_t": 0.32851046323776245,
      "grad_norm": 65.21794891357422,
      "learning_rate": 3.9855072463768114e-07,
      "logits/chosen": -0.6073439717292786,
      "logits/rejected": -0.5643370151519775,
      "logps/chosen": -58.113746643066406,
      "logps/ref_chosen": -57.9327278137207,
      "logps/ref_rejected": -94.1744384765625,
      "logps/rejected": -99.41685485839844,
      "loss": 0.8886,
      "margin_dpo/margin_mean": 5.061400890350342,
      "margin_dpo/margin_std": 5.794804573059082,
      "step": 56
    },
    {
      "epoch": 0.08370044052863436,
      "fcm_dpo/beta": 0.16040679812431335,
      "fcm_dpo/delta": -0.334885835647583,
      "fcm_dpo/margin": 5.6461501121521,
      "fcm_dpo/q_t": 0.3076401352882385,
      "grad_norm": 72.25611877441406,
      "learning_rate": 4.057971014492754e-07,
      "logits/chosen": -0.5476347208023071,
      "logits/rejected": -0.5185646414756775,
      "logps/chosen": -70.79481506347656,
      "logps/ref_chosen": -70.49528503417969,
      "logps/ref_rejected": -95.56546020507812,
      "logps/rejected": -101.51113891601562,
      "loss": 0.806,
      "margin_dpo/margin_mean": 5.646149635314941,
      "margin_dpo/margin_std": 4.98412561416626,
      "step": 57
    },
    {
      "epoch": 0.08516886930983847,
      "fcm_dpo/beta": 0.14835116267204285,
      "fcm_dpo/delta": -0.41062241792678833,
      "fcm_dpo/margin": 6.553308963775635,
      "fcm_dpo/q_t": 0.30739498138427734,
      "grad_norm": 73.45458984375,
      "learning_rate": 4.1304347826086954e-07,
      "logits/chosen": -0.5891647338867188,
      "logits/rejected": -0.513205885887146,
      "logps/chosen": -62.415225982666016,
      "logps/ref_chosen": -62.13294219970703,
      "logps/ref_rejected": -84.61729431152344,
      "logps/rejected": -91.45289611816406,
      "loss": 0.8144,
      "margin_dpo/margin_mean": 6.553309440612793,
      "margin_dpo/margin_std": 6.610756874084473,
      "step": 58
    },
    {
      "epoch": 0.08663729809104258,
      "fcm_dpo/beta": 0.13525643944740295,
      "fcm_dpo/delta": -0.35778504610061646,
      "fcm_dpo/margin": 6.788931846618652,
      "fcm_dpo/q_t": 0.3129827380180359,
      "grad_norm": 67.0660171508789,
      "learning_rate": 4.2028985507246374e-07,
      "logits/chosen": -0.6016139984130859,
      "logits/rejected": -0.558463454246521,
      "logps/chosen": -52.71605682373047,
      "logps/ref_chosen": -51.932525634765625,
      "logps/ref_rejected": -88.88520050048828,
      "logps/rejected": -96.45765686035156,
      "loss": 0.8265,
      "margin_dpo/margin_mean": 6.7889323234558105,
      "margin_dpo/margin_std": 6.713343620300293,
      "step": 59
    },
    {
      "epoch": 0.0881057268722467,
      "fcm_dpo/beta": 0.1309395432472229,
      "fcm_dpo/delta": -0.13870461285114288,
      "fcm_dpo/margin": 5.555472373962402,
      "fcm_dpo/q_t": 0.3427530527114868,
      "grad_norm": 79.94298553466797,
      "learning_rate": 4.2753623188405794e-07,
      "logits/chosen": -0.5748249888420105,
      "logits/rejected": -0.512750506401062,
      "logps/chosen": -62.536293029785156,
      "logps/ref_chosen": -60.94218826293945,
      "logps/ref_rejected": -85.39340209960938,
      "logps/rejected": -92.54298400878906,
      "loss": 0.9199,
      "margin_dpo/margin_mean": 5.555472373962402,
      "margin_dpo/margin_std": 6.041206359863281,
      "step": 60
    },
    {
      "epoch": 0.08957415565345081,
      "fcm_dpo/beta": 0.12486925721168518,
      "fcm_dpo/delta": -0.2642156481742859,
      "fcm_dpo/margin": 6.724519729614258,
      "fcm_dpo/q_t": 0.3386513590812683,
      "grad_norm": 61.240787506103516,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": -0.5803092122077942,
      "logits/rejected": -0.5446271896362305,
      "logps/chosen": -61.23955535888672,
      "logps/ref_chosen": -60.633522033691406,
      "logps/ref_rejected": -89.85249328613281,
      "logps/rejected": -97.18305206298828,
      "loss": 0.9087,
      "margin_dpo/margin_mean": 6.724519729614258,
      "margin_dpo/margin_std": 8.698123931884766,
      "step": 61
    },
    {
      "epoch": 0.09104258443465492,
      "fcm_dpo/beta": 0.12395919859409332,
      "fcm_dpo/delta": -0.035701360553503036,
      "fcm_dpo/margin": 5.107054233551025,
      "fcm_dpo/q_t": 0.3664395213127136,
      "grad_norm": 62.39984893798828,
      "learning_rate": 4.420289855072464e-07,
      "logits/chosen": -0.5827194452285767,
      "logits/rejected": -0.547878086566925,
      "logps/chosen": -57.02729034423828,
      "logps/ref_chosen": -56.15077209472656,
      "logps/ref_rejected": -75.56619262695312,
      "logps/rejected": -81.54976654052734,
      "loss": 0.9761,
      "margin_dpo/margin_mean": 5.107054710388184,
      "margin_dpo/margin_std": 6.175351619720459,
      "step": 62
    },
    {
      "epoch": 0.09251101321585903,
      "fcm_dpo/beta": 0.11800666153430939,
      "fcm_dpo/delta": -0.23345595598220825,
      "fcm_dpo/margin": 6.8794989585876465,
      "fcm_dpo/q_t": 0.3267975449562073,
      "grad_norm": 60.61161422729492,
      "learning_rate": 4.4927536231884053e-07,
      "logits/chosen": -0.5791250467300415,
      "logits/rejected": -0.5316081643104553,
      "logps/chosen": -74.61734008789062,
      "logps/ref_chosen": -73.14739227294922,
      "logps/ref_rejected": -97.61006164550781,
      "logps/rejected": -105.95950317382812,
      "loss": 0.8674,
      "margin_dpo/margin_mean": 6.879498481750488,
      "margin_dpo/margin_std": 6.8499274253845215,
      "step": 63
    },
    {
      "epoch": 0.09397944199706314,
      "fcm_dpo/beta": 0.11056903749704361,
      "fcm_dpo/delta": -0.3205702304840088,
      "fcm_dpo/margin": 8.005148887634277,
      "fcm_dpo/q_t": 0.32072484493255615,
      "grad_norm": 56.07626724243164,
      "learning_rate": 4.5652173913043473e-07,
      "logits/chosen": -0.5584267377853394,
      "logits/rejected": -0.5256708264350891,
      "logps/chosen": -54.30480194091797,
      "logps/ref_chosen": -53.998600006103516,
      "logps/ref_rejected": -93.53019714355469,
      "logps/rejected": -101.841552734375,
      "loss": 0.8584,
      "margin_dpo/margin_mean": 8.005149841308594,
      "margin_dpo/margin_std": 8.640867233276367,
      "step": 64
    },
    {
      "epoch": 0.09544787077826726,
      "fcm_dpo/beta": 0.10528378188610077,
      "fcm_dpo/delta": -0.26119303703308105,
      "fcm_dpo/margin": 7.948797225952148,
      "fcm_dpo/q_t": 0.32479870319366455,
      "grad_norm": 56.537654876708984,
      "learning_rate": 4.63768115942029e-07,
      "logits/chosen": -0.6330504417419434,
      "logits/rejected": -0.6177892684936523,
      "logps/chosen": -66.75464630126953,
      "logps/ref_chosen": -64.83599853515625,
      "logps/ref_rejected": -109.94645690917969,
      "logps/rejected": -119.81390380859375,
      "loss": 0.8793,
      "margin_dpo/margin_mean": 7.948797225952148,
      "margin_dpo/margin_std": 8.451780319213867,
      "step": 65
    },
    {
      "epoch": 0.09691629955947137,
      "fcm_dpo/beta": 0.10108963400125504,
      "fcm_dpo/delta": -0.16579298675060272,
      "fcm_dpo/margin": 7.422573089599609,
      "fcm_dpo/q_t": 0.34628963470458984,
      "grad_norm": 50.0311164855957,
      "learning_rate": 4.7101449275362313e-07,
      "logits/chosen": -0.6193344593048096,
      "logits/rejected": -0.5846483707427979,
      "logps/chosen": -53.2528076171875,
      "logps/ref_chosen": -51.44352722167969,
      "logps/ref_rejected": -75.63629913330078,
      "logps/rejected": -84.86815643310547,
      "loss": 0.9152,
      "margin_dpo/margin_mean": 7.422572135925293,
      "margin_dpo/margin_std": 8.596309661865234,
      "step": 66
    },
    {
      "epoch": 0.09838472834067548,
      "fcm_dpo/beta": 0.09882417321205139,
      "fcm_dpo/delta": -0.17645135521888733,
      "fcm_dpo/margin": 7.723736763000488,
      "fcm_dpo/q_t": 0.3419727683067322,
      "grad_norm": 51.04230499267578,
      "learning_rate": 4.782608695652174e-07,
      "logits/chosen": -0.5773788690567017,
      "logits/rejected": -0.5347466468811035,
      "logps/chosen": -60.845420837402344,
      "logps/ref_chosen": -59.34080505371094,
      "logps/ref_rejected": -72.78728485107422,
      "logps/rejected": -82.01564025878906,
      "loss": 0.9005,
      "margin_dpo/margin_mean": 7.72373628616333,
      "margin_dpo/margin_std": 8.540781021118164,
      "step": 67
    },
    {
      "epoch": 0.09985315712187959,
      "fcm_dpo/beta": 0.09611781686544418,
      "fcm_dpo/delta": -0.1308629810810089,
      "fcm_dpo/margin": 7.5075178146362305,
      "fcm_dpo/q_t": 0.34419840574264526,
      "grad_norm": 48.844181060791016,
      "learning_rate": 4.855072463768116e-07,
      "logits/chosen": -0.6071466207504272,
      "logits/rejected": -0.5482945442199707,
      "logps/chosen": -66.77816772460938,
      "logps/ref_chosen": -65.2058334350586,
      "logps/ref_rejected": -77.20724487304688,
      "logps/rejected": -86.28710174560547,
      "loss": 0.8922,
      "margin_dpo/margin_mean": 7.507518291473389,
      "margin_dpo/margin_std": 7.301938056945801,
      "step": 68
    },
    {
      "epoch": 0.1013215859030837,
      "fcm_dpo/beta": 0.09169034659862518,
      "fcm_dpo/delta": -0.20954345166683197,
      "fcm_dpo/margin": 8.629631042480469,
      "fcm_dpo/q_t": 0.3316349387168884,
      "grad_norm": 50.881256103515625,
      "learning_rate": 4.927536231884058e-07,
      "logits/chosen": -0.5818198323249817,
      "logits/rejected": -0.557062029838562,
      "logps/chosen": -61.803890228271484,
      "logps/ref_chosen": -59.81924057006836,
      "logps/ref_rejected": -103.38886260986328,
      "logps/rejected": -114.00314331054688,
      "loss": 0.8551,
      "margin_dpo/margin_mean": 8.629631042480469,
      "margin_dpo/margin_std": 8.243066787719727,
      "step": 69
    },
    {
      "epoch": 0.1027900146842878,
      "fcm_dpo/beta": 0.08815528452396393,
      "fcm_dpo/delta": -0.25355854630470276,
      "fcm_dpo/margin": 9.44918155670166,
      "fcm_dpo/q_t": 0.3277716040611267,
      "grad_norm": 51.42112350463867,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6246175765991211,
      "logits/rejected": -0.5891268253326416,
      "logps/chosen": -64.80339050292969,
      "logps/ref_chosen": -61.930641174316406,
      "logps/ref_rejected": -91.06078338623047,
      "logps/rejected": -103.38272094726562,
      "loss": 0.8569,
      "margin_dpo/margin_mean": 9.449182510375977,
      "margin_dpo/margin_std": 9.51607894897461,
      "step": 70
    },
    {
      "epoch": 0.10425844346549193,
      "fcm_dpo/beta": 0.08285696059465408,
      "fcm_dpo/delta": -0.30268311500549316,
      "fcm_dpo/margin": 10.571916580200195,
      "fcm_dpo/q_t": 0.3174167275428772,
      "grad_norm": 45.99234390258789,
      "learning_rate": 4.999967061337492e-07,
      "logits/chosen": -0.6398484110832214,
      "logits/rejected": -0.5968210101127625,
      "logps/chosen": -64.16336822509766,
      "logps/ref_chosen": -61.750335693359375,
      "logps/ref_rejected": -97.33662414550781,
      "logps/rejected": -110.32157135009766,
      "loss": 0.8213,
      "margin_dpo/margin_mean": 10.571917533874512,
      "margin_dpo/margin_std": 9.830376625061035,
      "step": 71
    },
    {
      "epoch": 0.10572687224669604,
      "fcm_dpo/beta": 0.07784741371870041,
      "fcm_dpo/delta": -0.2931217849254608,
      "fcm_dpo/margin": 11.129253387451172,
      "fcm_dpo/q_t": 0.31700342893600464,
      "grad_norm": 48.5871696472168,
      "learning_rate": 4.999868246217933e-07,
      "logits/chosen": -0.6470623016357422,
      "logits/rejected": -0.6127582788467407,
      "logps/chosen": -68.55953216552734,
      "logps/ref_chosen": -66.05341339111328,
      "logps/ref_rejected": -95.2869873046875,
      "logps/rejected": -108.92236328125,
      "loss": 0.8263,
      "margin_dpo/margin_mean": 11.129253387451172,
      "margin_dpo/margin_std": 10.417036056518555,
      "step": 72
    },
    {
      "epoch": 0.10719530102790015,
      "fcm_dpo/beta": 0.07408735156059265,
      "fcm_dpo/delta": -0.2234322428703308,
      "fcm_dpo/margin": 10.851455688476562,
      "fcm_dpo/q_t": 0.3458544611930847,
      "grad_norm": 45.48418045043945,
      "learning_rate": 4.999703557245192e-07,
      "logits/chosen": -0.6490052938461304,
      "logits/rejected": -0.6046302914619446,
      "logps/chosen": -70.05899047851562,
      "logps/ref_chosen": -66.25627136230469,
      "logps/ref_rejected": -90.45613098144531,
      "logps/rejected": -105.11030578613281,
      "loss": 0.953,
      "margin_dpo/margin_mean": 10.851455688476562,
      "margin_dpo/margin_std": 14.541223526000977,
      "step": 73
    },
    {
      "epoch": 0.10866372980910426,
      "fcm_dpo/beta": 0.07084085047245026,
      "fcm_dpo/delta": -0.22987580299377441,
      "fcm_dpo/margin": 11.434507369995117,
      "fcm_dpo/q_t": 0.3413064479827881,
      "grad_norm": 47.909542083740234,
      "learning_rate": 4.999472998758977e-07,
      "logits/chosen": -0.6333368420600891,
      "logits/rejected": -0.6184351444244385,
      "logps/chosen": -57.71470260620117,
      "logps/ref_chosen": -53.42488098144531,
      "logps/ref_rejected": -95.94693756103516,
      "logps/rejected": -111.6712646484375,
      "loss": 0.9336,
      "margin_dpo/margin_mean": 11.4345064163208,
      "margin_dpo/margin_std": 15.739431381225586,
      "step": 74
    },
    {
      "epoch": 0.11013215859030837,
      "fcm_dpo/beta": 0.06567974388599396,
      "fcm_dpo/delta": -0.5119467377662659,
      "fcm_dpo/margin": 16.138355255126953,
      "fcm_dpo/q_t": 0.291500985622406,
      "grad_norm": 41.545692443847656,
      "learning_rate": 4.999176576834721e-07,
      "logits/chosen": -0.6396368741989136,
      "logits/rejected": -0.6279274225234985,
      "logps/chosen": -55.962501525878906,
      "logps/ref_chosen": -51.861663818359375,
      "logps/ref_rejected": -111.25398254394531,
      "logps/rejected": -131.4931640625,
      "loss": 0.7651,
      "margin_dpo/margin_mean": 16.138355255126953,
      "margin_dpo/margin_std": 15.203086853027344,
      "step": 75
    },
    {
      "epoch": 0.11160058737151249,
      "fcm_dpo/beta": 0.062492579221725464,
      "fcm_dpo/delta": -0.09926581382751465,
      "fcm_dpo/margin": 11.07880973815918,
      "fcm_dpo/q_t": 0.34875011444091797,
      "grad_norm": 41.94511795043945,
      "learning_rate": 4.998814299283415e-07,
      "logits/chosen": -0.7100133895874023,
      "logits/rejected": -0.6680034399032593,
      "logps/chosen": -58.421531677246094,
      "logps/ref_chosen": -53.26603698730469,
      "logps/ref_rejected": -78.21662902832031,
      "logps/rejected": -94.45092010498047,
      "loss": 0.9298,
      "margin_dpo/margin_mean": 11.078810691833496,
      "margin_dpo/margin_std": 12.21889877319336,
      "step": 76
    },
    {
      "epoch": 0.1130690161527166,
      "fcm_dpo/beta": 0.05767790973186493,
      "fcm_dpo/delta": -0.45353737473487854,
      "fcm_dpo/margin": 17.38974380493164,
      "fcm_dpo/q_t": 0.2953811585903168,
      "grad_norm": 43.50868606567383,
      "learning_rate": 4.998386175651409e-07,
      "logits/chosen": -0.6637353897094727,
      "logits/rejected": -0.6201448440551758,
      "logps/chosen": -62.73433303833008,
      "logps/ref_chosen": -58.0966796875,
      "logps/ref_rejected": -93.77361297607422,
      "logps/rejected": -115.80101013183594,
      "loss": 0.7985,
      "margin_dpo/margin_mean": 17.38974380493164,
      "margin_dpo/margin_std": 16.823715209960938,
      "step": 77
    },
    {
      "epoch": 0.1145374449339207,
      "fcm_dpo/beta": 0.0555085651576519,
      "fcm_dpo/delta": -0.18481989204883575,
      "fcm_dpo/margin": 13.888815879821777,
      "fcm_dpo/q_t": 0.3378889262676239,
      "grad_norm": 39.630531311035156,
      "learning_rate": 4.997892217220159e-07,
      "logits/chosen": -0.6263148784637451,
      "logits/rejected": -0.5966655015945435,
      "logps/chosen": -60.46125030517578,
      "logps/ref_chosen": -55.61378479003906,
      "logps/ref_rejected": -84.93436431884766,
      "logps/rejected": -103.67064666748047,
      "loss": 0.8902,
      "margin_dpo/margin_mean": 13.888814926147461,
      "margin_dpo/margin_std": 14.465457916259766,
      "step": 78
    },
    {
      "epoch": 0.11600587371512482,
      "fcm_dpo/beta": 0.05275849997997284,
      "fcm_dpo/delta": -0.24378372728824615,
      "fcm_dpo/margin": 15.608173370361328,
      "fcm_dpo/q_t": 0.3392306864261627,
      "grad_norm": 33.420074462890625,
      "learning_rate": 4.997332437005931e-07,
      "logits/chosen": -0.6634904146194458,
      "logits/rejected": -0.6324459314346313,
      "logps/chosen": -60.526432037353516,
      "logps/ref_chosen": -55.45048522949219,
      "logps/ref_rejected": -87.64756774902344,
      "logps/rejected": -108.33168029785156,
      "loss": 0.9079,
      "margin_dpo/margin_mean": 15.608173370361328,
      "margin_dpo/margin_std": 18.54351043701172,
      "step": 79
    },
    {
      "epoch": 0.11747430249632893,
      "fcm_dpo/beta": 0.05068628117442131,
      "fcm_dpo/delta": -0.12069609016180038,
      "fcm_dpo/margin": 13.985533714294434,
      "fcm_dpo/q_t": 0.3596472144126892,
      "grad_norm": 37.298439025878906,
      "learning_rate": 4.996706849759452e-07,
      "logits/chosen": -0.7097140550613403,
      "logits/rejected": -0.6619396805763245,
      "logps/chosen": -66.05055236816406,
      "logps/ref_chosen": -58.519290924072266,
      "logps/ref_rejected": -87.54750061035156,
      "logps/rejected": -109.06429290771484,
      "loss": 0.9679,
      "margin_dpo/margin_mean": 13.98553466796875,
      "margin_dpo/margin_std": 17.724082946777344,
      "step": 80
    },
    {
      "epoch": 0.11894273127753303,
      "fcm_dpo/beta": 0.047711554914712906,
      "fcm_dpo/delta": -0.3803252577781677,
      "fcm_dpo/margin": 19.70598602294922,
      "fcm_dpo/q_t": 0.31643375754356384,
      "grad_norm": 38.47947692871094,
      "learning_rate": 4.996015471965529e-07,
      "logits/chosen": -0.6981919407844543,
      "logits/rejected": -0.6639083623886108,
      "logps/chosen": -73.06269836425781,
      "logps/ref_chosen": -66.44886779785156,
      "logps/ref_rejected": -129.66270446777344,
      "logps/rejected": -155.98251342773438,
      "loss": 0.8483,
      "margin_dpo/margin_mean": 19.70598602294922,
      "margin_dpo/margin_std": 22.23188018798828,
      "step": 81
    },
    {
      "epoch": 0.12041116005873716,
      "fcm_dpo/beta": 0.04673580825328827,
      "fcm_dpo/delta": -0.15118417143821716,
      "fcm_dpo/margin": 15.803937911987305,
      "fcm_dpo/q_t": 0.3553317189216614,
      "grad_norm": 42.93745422363281,
      "learning_rate": 4.995258321842611e-07,
      "logits/chosen": -0.6455098390579224,
      "logits/rejected": -0.6292978525161743,
      "logps/chosen": -61.03137969970703,
      "logps/ref_chosen": -52.232383728027344,
      "logps/ref_rejected": -90.74325561523438,
      "logps/rejected": -115.34617614746094,
      "loss": 1.018,
      "margin_dpo/margin_mean": 15.803937911987305,
      "margin_dpo/margin_std": 23.58461570739746,
      "step": 82
    },
    {
      "epoch": 0.12187958883994127,
      "fcm_dpo/beta": 0.04404474422335625,
      "fcm_dpo/delta": -0.23587247729301453,
      "fcm_dpo/margin": 18.518720626831055,
      "fcm_dpo/q_t": 0.33574339747428894,
      "grad_norm": 40.53355407714844,
      "learning_rate": 4.994435419342304e-07,
      "logits/chosen": -0.6932875514030457,
      "logits/rejected": -0.6512936353683472,
      "logps/chosen": -64.86161804199219,
      "logps/ref_chosen": -55.82738494873047,
      "logps/ref_rejected": -103.71589660644531,
      "logps/rejected": -131.26885986328125,
      "loss": 0.9031,
      "margin_dpo/margin_mean": 18.518720626831055,
      "margin_dpo/margin_std": 21.3782901763916,
      "step": 83
    },
    {
      "epoch": 0.12334801762114538,
      "fcm_dpo/beta": 0.042480941861867905,
      "fcm_dpo/delta": -0.12278434634208679,
      "fcm_dpo/margin": 16.74671173095703,
      "fcm_dpo/q_t": 0.34350255131721497,
      "grad_norm": 34.17844772338867,
      "learning_rate": 4.993546786148857e-07,
      "logits/chosen": -0.6624174118041992,
      "logits/rejected": -0.624229907989502,
      "logps/chosen": -75.19337463378906,
      "logps/ref_chosen": -67.1761703491211,
      "logps/ref_rejected": -87.29859924316406,
      "logps/rejected": -112.06253051757812,
      "loss": 0.9041,
      "margin_dpo/margin_mean": 16.74671173095703,
      "margin_dpo/margin_std": 16.270580291748047,
      "step": 84
    },
    {
      "epoch": 0.12481644640234948,
      "fcm_dpo/beta": 0.042263105511665344,
      "fcm_dpo/delta": -0.1225767433643341,
      "fcm_dpo/margin": 16.87063217163086,
      "fcm_dpo/q_t": 0.3529721796512604,
      "grad_norm": 36.07400131225586,
      "learning_rate": 4.992592445678582e-07,
      "logits/chosen": -0.645004391670227,
      "logits/rejected": -0.6133354902267456,
      "logps/chosen": -67.26490783691406,
      "logps/ref_chosen": -58.4066162109375,
      "logps/ref_rejected": -78.63880157470703,
      "logps/rejected": -104.36772155761719,
      "loss": 0.9467,
      "margin_dpo/margin_mean": 16.87063217163086,
      "margin_dpo/margin_std": 19.575843811035156,
      "step": 85
    },
    {
      "epoch": 0.1262848751835536,
      "fcm_dpo/beta": 0.04085496813058853,
      "fcm_dpo/delta": -0.15022027492523193,
      "fcm_dpo/margin": 18.096515655517578,
      "fcm_dpo/q_t": 0.3664831817150116,
      "grad_norm": 40.77984619140625,
      "learning_rate": 4.991572423079235e-07,
      "logits/chosen": -0.6773054599761963,
      "logits/rejected": -0.6616036891937256,
      "logps/chosen": -67.63337707519531,
      "logps/ref_chosen": -56.13746643066406,
      "logps/ref_rejected": -88.12165069580078,
      "logps/rejected": -117.71407318115234,
      "loss": 1.0372,
      "margin_dpo/margin_mean": 18.096515655517578,
      "margin_dpo/margin_std": 29.443721771240234,
      "step": 86
    },
    {
      "epoch": 0.1277533039647577,
      "fcm_dpo/beta": 0.03880458325147629,
      "fcm_dpo/delta": -0.21359695494174957,
      "fcm_dpo/margin": 20.468475341796875,
      "fcm_dpo/q_t": 0.3394929766654968,
      "grad_norm": 33.87302780151367,
      "learning_rate": 4.990486745229364e-07,
      "logits/chosen": -0.7122533917427063,
      "logits/rejected": -0.6807093620300293,
      "logps/chosen": -67.1455078125,
      "logps/ref_chosen": -55.63609313964844,
      "logps/ref_rejected": -95.46757507324219,
      "logps/rejected": -127.44546508789062,
      "loss": 0.943,
      "margin_dpo/margin_mean": 20.468475341796875,
      "margin_dpo/margin_std": 25.73678207397461,
      "step": 87
    },
    {
      "epoch": 0.12922173274596183,
      "fcm_dpo/beta": 0.038395434617996216,
      "fcm_dpo/delta": 0.0005285218358039856,
      "fcm_dpo/margin": 15.591540336608887,
      "fcm_dpo/q_t": 0.3785722851753235,
      "grad_norm": 36.46826171875,
      "learning_rate": 4.989335440737586e-07,
      "logits/chosen": -0.6723873615264893,
      "logits/rejected": -0.6606534719467163,
      "logps/chosen": -88.06733703613281,
      "logps/ref_chosen": -73.67115020751953,
      "logps/ref_rejected": -106.70849609375,
      "logps/rejected": -136.69622802734375,
      "loss": 1.0542,
      "margin_dpo/margin_mean": 15.59153938293457,
      "margin_dpo/margin_std": 22.799100875854492,
      "step": 88
    },
    {
      "epoch": 0.13069016152716592,
      "fcm_dpo/beta": 0.03851180896162987,
      "fcm_dpo/delta": -0.06854115426540375,
      "fcm_dpo/margin": 17.231027603149414,
      "fcm_dpo/q_t": 0.36006081104278564,
      "grad_norm": 31.71763038635254,
      "learning_rate": 4.988118539941847e-07,
      "logits/chosen": -0.7132691144943237,
      "logits/rejected": -0.6763544082641602,
      "logps/chosen": -69.2666015625,
      "logps/ref_chosen": -60.624916076660156,
      "logps/ref_rejected": -82.08354949951172,
      "logps/rejected": -107.95626831054688,
      "loss": 0.9577,
      "margin_dpo/margin_mean": 17.231027603149414,
      "margin_dpo/margin_std": 20.84075927734375,
      "step": 89
    },
    {
      "epoch": 0.13215859030837004,
      "fcm_dpo/beta": 0.037114374339580536,
      "fcm_dpo/delta": -0.26081135869026184,
      "fcm_dpo/margin": 22.61049461364746,
      "fcm_dpo/q_t": 0.34644296765327454,
      "grad_norm": 35.28056716918945,
      "learning_rate": 4.986836074908615e-07,
      "logits/chosen": -0.6307741403579712,
      "logits/rejected": -0.6357797384262085,
      "logps/chosen": -64.71424102783203,
      "logps/ref_chosen": -53.285308837890625,
      "logps/ref_rejected": -111.54470825195312,
      "logps/rejected": -145.58413696289062,
      "loss": 0.9619,
      "margin_dpo/margin_mean": 22.610496520996094,
      "margin_dpo/margin_std": 31.246440887451172,
      "step": 90
    },
    {
      "epoch": 0.13362701908957417,
      "fcm_dpo/beta": 0.035702183842659,
      "fcm_dpo/delta": -0.12644991278648376,
      "fcm_dpo/margin": 20.09602928161621,
      "fcm_dpo/q_t": 0.3567592203617096,
      "grad_norm": 33.433372497558594,
      "learning_rate": 4.985488079432037e-07,
      "logits/chosen": -0.6847516894340515,
      "logits/rejected": -0.6470792889595032,
      "logps/chosen": -73.15221405029297,
      "logps/ref_chosen": -61.802955627441406,
      "logps/ref_rejected": -87.87395477294922,
      "logps/rejected": -119.31924438476562,
      "loss": 0.9724,
      "margin_dpo/margin_mean": 20.096027374267578,
      "margin_dpo/margin_std": 26.328693389892578,
      "step": 91
    },
    {
      "epoch": 0.13509544787077826,
      "fcm_dpo/beta": 0.03499949723482132,
      "fcm_dpo/delta": -0.10029930621385574,
      "fcm_dpo/margin": 19.803434371948242,
      "fcm_dpo/q_t": 0.36030054092407227,
      "grad_norm": 30.455114364624023,
      "learning_rate": 4.984074589033043e-07,
      "logits/chosen": -0.7354650497436523,
      "logits/rejected": -0.7112290859222412,
      "logps/chosen": -62.292938232421875,
      "logps/ref_chosen": -51.640769958496094,
      "logps/ref_rejected": -77.88117980957031,
      "logps/rejected": -108.3367919921875,
      "loss": 0.9797,
      "margin_dpo/margin_mean": 19.803434371948242,
      "margin_dpo/margin_std": 25.99898910522461,
      "step": 92
    },
    {
      "epoch": 0.13656387665198239,
      "fcm_dpo/beta": 0.03412316367030144,
      "fcm_dpo/delta": -0.08976341784000397,
      "fcm_dpo/margin": 20.03380584716797,
      "fcm_dpo/q_t": 0.3562747538089752,
      "grad_norm": 30.054187774658203,
      "learning_rate": 4.982595640958425e-07,
      "logits/chosen": -0.7533690333366394,
      "logits/rejected": -0.6998313069343567,
      "logps/chosen": -64.41513061523438,
      "logps/ref_chosen": -52.529239654541016,
      "logps/ref_rejected": -77.16075134277344,
      "logps/rejected": -109.0804443359375,
      "loss": 0.9377,
      "margin_dpo/margin_mean": 20.03380584716797,
      "margin_dpo/margin_std": 23.324481964111328,
      "step": 93
    },
    {
      "epoch": 0.13803230543318648,
      "fcm_dpo/beta": 0.0329783670604229,
      "fcm_dpo/delta": -0.1528889387845993,
      "fcm_dpo/margin": 22.42888641357422,
      "fcm_dpo/q_t": 0.3440149128437042,
      "grad_norm": 30.21152114868164,
      "learning_rate": 4.98105127417984e-07,
      "logits/chosen": -0.6753390431404114,
      "logits/rejected": -0.6542376279830933,
      "logps/chosen": -74.20423889160156,
      "logps/ref_chosen": -61.22261047363281,
      "logps/ref_rejected": -99.59902954101562,
      "logps/rejected": -135.00955200195312,
      "loss": 0.9066,
      "margin_dpo/margin_mean": 22.42888641357422,
      "margin_dpo/margin_std": 23.697879791259766,
      "step": 94
    },
    {
      "epoch": 0.1395007342143906,
      "fcm_dpo/beta": 0.03280120715498924,
      "fcm_dpo/delta": -0.006751883774995804,
      "fcm_dpo/margin": 18.477895736694336,
      "fcm_dpo/q_t": 0.36756014823913574,
      "grad_norm": 28.341552734375,
      "learning_rate": 4.979441529392784e-07,
      "logits/chosen": -0.732563853263855,
      "logits/rejected": -0.701493501663208,
      "logps/chosen": -64.28013610839844,
      "logps/ref_chosen": -52.523643493652344,
      "logps/ref_rejected": -75.8803482055664,
      "logps/rejected": -106.11474609375,
      "loss": 0.981,
      "margin_dpo/margin_mean": 18.477895736694336,
      "margin_dpo/margin_std": 21.757883071899414,
      "step": 95
    },
    {
      "epoch": 0.14096916299559473,
      "fcm_dpo/beta": 0.03166916221380234,
      "fcm_dpo/delta": -0.20488545298576355,
      "fcm_dpo/margin": 24.822622299194336,
      "fcm_dpo/q_t": 0.3381555676460266,
      "grad_norm": 29.113483428955078,
      "learning_rate": 4.977766449015534e-07,
      "logits/chosen": -0.7252618670463562,
      "logits/rejected": -0.6913504600524902,
      "logps/chosen": -73.15214538574219,
      "logps/ref_chosen": -62.15697479248047,
      "logps/ref_rejected": -96.59601593017578,
      "logps/rejected": -132.413818359375,
      "loss": 0.888,
      "margin_dpo/margin_mean": 24.82262420654297,
      "margin_dpo/margin_std": 27.286436080932617,
      "step": 96
    },
    {
      "epoch": 0.14243759177679882,
      "fcm_dpo/beta": 0.03180006146430969,
      "fcm_dpo/delta": -0.026856690645217896,
      "fcm_dpo/margin": 19.596725463867188,
      "fcm_dpo/q_t": 0.36258357763290405,
      "grad_norm": 30.518535614013672,
      "learning_rate": 4.976026077188012e-07,
      "logits/chosen": -0.6421018838882446,
      "logits/rejected": -0.591498851776123,
      "logps/chosen": -66.89000701904297,
      "logps/ref_chosen": -54.646366119384766,
      "logps/ref_rejected": -76.96475219726562,
      "logps/rejected": -108.80511474609375,
      "loss": 0.9602,
      "margin_dpo/margin_mean": 19.596725463867188,
      "margin_dpo/margin_std": 20.48422622680664,
      "step": 97
    },
    {
      "epoch": 0.14390602055800295,
      "fcm_dpo/beta": 0.03093864396214485,
      "fcm_dpo/delta": -0.09095153957605362,
      "fcm_dpo/margin": 22.12537384033203,
      "fcm_dpo/q_t": 0.35298076272010803,
      "grad_norm": 31.037092208862305,
      "learning_rate": 4.974220459770639e-07,
      "logits/chosen": -0.7037702798843384,
      "logits/rejected": -0.684203028678894,
      "logps/chosen": -80.09527587890625,
      "logps/ref_chosen": -65.25862884521484,
      "logps/ref_rejected": -96.5274887084961,
      "logps/rejected": -133.48951721191406,
      "loss": 0.9689,
      "margin_dpo/margin_mean": 22.12537384033203,
      "margin_dpo/margin_std": 27.224925994873047,
      "step": 98
    },
    {
      "epoch": 0.14537444933920704,
      "fcm_dpo/beta": 0.029884759336709976,
      "fcm_dpo/delta": -0.19739775359630585,
      "fcm_dpo/margin": 26.147565841674805,
      "fcm_dpo/q_t": 0.3427448868751526,
      "grad_norm": 27.304397583007812,
      "learning_rate": 4.972349644343108e-07,
      "logits/chosen": -0.694936990737915,
      "logits/rejected": -0.6911288499832153,
      "logps/chosen": -57.74766159057617,
      "logps/ref_chosen": -45.638484954833984,
      "logps/ref_rejected": -86.43793487548828,
      "logps/rejected": -124.69467163085938,
      "loss": 0.9019,
      "margin_dpo/margin_mean": 26.147567749023438,
      "margin_dpo/margin_std": 30.577857971191406,
      "step": 99
    },
    {
      "epoch": 0.14684287812041116,
      "fcm_dpo/beta": 0.0302484929561615,
      "fcm_dpo/delta": 0.1044018343091011,
      "fcm_dpo/margin": 16.52963638305664,
      "fcm_dpo/q_t": 0.3935774266719818,
      "grad_norm": 31.379667282104492,
      "learning_rate": 4.970413680203148e-07,
      "logits/chosen": -0.6751364469528198,
      "logits/rejected": -0.6300145983695984,
      "logps/chosen": -70.98727416992188,
      "logps/ref_chosen": -57.59397888183594,
      "logps/ref_rejected": -74.06021118164062,
      "logps/rejected": -103.98313903808594,
      "loss": 1.0889,
      "margin_dpo/margin_mean": 16.529634475708008,
      "margin_dpo/margin_std": 25.98699188232422,
      "step": 100
    },
    {
      "epoch": 0.14831130690161526,
      "fcm_dpo/beta": 0.030119918286800385,
      "fcm_dpo/delta": 0.012738943099975586,
      "fcm_dpo/margin": 19.492538452148438,
      "fcm_dpo/q_t": 0.382551908493042,
      "grad_norm": 31.488115310668945,
      "learning_rate": 4.968412618365215e-07,
      "logits/chosen": -0.7016730308532715,
      "logits/rejected": -0.6665774583816528,
      "logps/chosen": -77.9628677368164,
      "logps/ref_chosen": -61.64885330200195,
      "logps/ref_rejected": -83.18968200683594,
      "logps/rejected": -118.99624633789062,
      "loss": 1.0464,
      "margin_dpo/margin_mean": 19.492538452148438,
      "margin_dpo/margin_std": 29.35474395751953,
      "step": 101
    },
    {
      "epoch": 0.14977973568281938,
      "fcm_dpo/beta": 0.03130955249071121,
      "fcm_dpo/delta": 0.14849652349948883,
      "fcm_dpo/margin": 14.546876907348633,
      "fcm_dpo/q_t": 0.40587127208709717,
      "grad_norm": 37.16018295288086,
      "learning_rate": 4.966346511559149e-07,
      "logits/chosen": -0.6912135481834412,
      "logits/rejected": -0.6412276029586792,
      "logps/chosen": -82.47193145751953,
      "logps/ref_chosen": -64.0788803100586,
      "logps/ref_rejected": -68.18707275390625,
      "logps/rejected": -101.12701416015625,
      "loss": 1.142,
      "margin_dpo/margin_mean": 14.546875,
      "margin_dpo/margin_std": 26.82202911376953,
      "step": 102
    },
    {
      "epoch": 0.1512481644640235,
      "fcm_dpo/beta": 0.030011659488081932,
      "fcm_dpo/delta": -0.2606973648071289,
      "fcm_dpo/margin": 27.926231384277344,
      "fcm_dpo/q_t": 0.3282929062843323,
      "grad_norm": 28.890342712402344,
      "learning_rate": 4.964215414228785e-07,
      "logits/chosen": -0.6866222620010376,
      "logits/rejected": -0.6497406959533691,
      "logps/chosen": -74.54656982421875,
      "logps/ref_chosen": -61.299278259277344,
      "logps/ref_rejected": -93.57270812988281,
      "logps/rejected": -134.74624633789062,
      "loss": 0.8619,
      "margin_dpo/margin_mean": 27.926231384277344,
      "margin_dpo/margin_std": 29.437318801879883,
      "step": 103
    },
    {
      "epoch": 0.1527165932452276,
      "fcm_dpo/beta": 0.028996918350458145,
      "fcm_dpo/delta": -0.1786722093820572,
      "fcm_dpo/margin": 26.393293380737305,
      "fcm_dpo/q_t": 0.3505978584289551,
      "grad_norm": 29.279157638549805,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": -0.7208267450332642,
      "logits/rejected": -0.683314859867096,
      "logps/chosen": -69.56233215332031,
      "logps/ref_chosen": -54.372772216796875,
      "logps/ref_rejected": -89.5647201538086,
      "logps/rejected": -131.14756774902344,
      "loss": 0.9468,
      "margin_dpo/margin_mean": 26.393293380737305,
      "margin_dpo/margin_std": 33.748008728027344,
      "step": 104
    },
    {
      "epoch": 0.15418502202643172,
      "fcm_dpo/beta": 0.026863520964980125,
      "fcm_dpo/delta": -0.43884339928627014,
      "fcm_dpo/margin": 37.05156707763672,
      "fcm_dpo/q_t": 0.28996556997299194,
      "grad_norm": 27.160659790039062,
      "learning_rate": 4.959758474331832e-07,
      "logits/chosen": -0.7008599638938904,
      "logits/rejected": -0.6714988946914673,
      "logps/chosen": -68.3555679321289,
      "logps/ref_chosen": -54.638946533203125,
      "logps/ref_rejected": -97.97351837158203,
      "logps/rejected": -148.74169921875,
      "loss": 0.7501,
      "margin_dpo/margin_mean": 37.05156707763672,
      "margin_dpo/margin_std": 30.501293182373047,
      "step": 105
    },
    {
      "epoch": 0.15565345080763582,
      "fcm_dpo/beta": 0.026156390085816383,
      "fcm_dpo/delta": -0.01053478755056858,
      "fcm_dpo/margin": 23.31535530090332,
      "fcm_dpo/q_t": 0.3665970265865326,
      "grad_norm": 27.37348747253418,
      "learning_rate": 4.957432749209755e-07,
      "logits/chosen": -0.650317370891571,
      "logits/rejected": -0.6077761650085449,
      "logps/chosen": -70.25245666503906,
      "logps/ref_chosen": -54.83289337158203,
      "logps/ref_rejected": -85.22461700439453,
      "logps/rejected": -123.95954895019531,
      "loss": 0.9688,
      "margin_dpo/margin_mean": 23.315357208251953,
      "margin_dpo/margin_std": 26.133378982543945,
      "step": 106
    },
    {
      "epoch": 0.15712187958883994,
      "fcm_dpo/beta": 0.025723904371261597,
      "fcm_dpo/delta": -0.0867917537689209,
      "fcm_dpo/margin": 26.438936233520508,
      "fcm_dpo/q_t": 0.3539083003997803,
      "grad_norm": 26.59319496154785,
      "learning_rate": 4.955042268449307e-07,
      "logits/chosen": -0.7173153162002563,
      "logits/rejected": -0.6675245761871338,
      "logps/chosen": -88.4300537109375,
      "logps/ref_chosen": -69.70780944824219,
      "logps/ref_rejected": -94.73950958251953,
      "logps/rejected": -139.90069580078125,
      "loss": 0.9479,
      "margin_dpo/margin_mean": 26.438936233520508,
      "margin_dpo/margin_std": 29.977489471435547,
      "step": 107
    },
    {
      "epoch": 0.15859030837004406,
      "fcm_dpo/beta": 0.025118088349699974,
      "fcm_dpo/delta": -0.1593261957168579,
      "fcm_dpo/margin": 29.753965377807617,
      "fcm_dpo/q_t": 0.3548673987388611,
      "grad_norm": 27.372953414916992,
      "learning_rate": 4.952587095041881e-07,
      "logits/chosen": -0.6825627088546753,
      "logits/rejected": -0.6495173573493958,
      "logps/chosen": -73.43077087402344,
      "logps/ref_chosen": -56.0098876953125,
      "logps/ref_rejected": -95.79601287841797,
      "logps/rejected": -142.97085571289062,
      "loss": 0.9717,
      "margin_dpo/margin_mean": 29.75396728515625,
      "margin_dpo/margin_std": 39.8942756652832,
      "step": 108
    },
    {
      "epoch": 0.16005873715124816,
      "fcm_dpo/beta": 0.02407240867614746,
      "fcm_dpo/delta": -0.20850417017936707,
      "fcm_dpo/margin": 32.8599853515625,
      "fcm_dpo/q_t": 0.33347180485725403,
      "grad_norm": 26.96596908569336,
      "learning_rate": 4.95006729368358e-07,
      "logits/chosen": -0.6184978485107422,
      "logits/rejected": -0.5932357311248779,
      "logps/chosen": -78.72866821289062,
      "logps/ref_chosen": -62.88549041748047,
      "logps/ref_rejected": -98.68573760986328,
      "logps/rejected": -147.38890075683594,
      "loss": 0.8832,
      "margin_dpo/margin_mean": 32.8599853515625,
      "margin_dpo/margin_std": 33.96752166748047,
      "step": 109
    },
    {
      "epoch": 0.16152716593245228,
      "fcm_dpo/beta": 0.023202266544103622,
      "fcm_dpo/delta": -0.10694625228643417,
      "fcm_dpo/margin": 29.92901611328125,
      "fcm_dpo/q_t": 0.3550639748573303,
      "grad_norm": 24.584754943847656,
      "learning_rate": 4.947482930773511e-07,
      "logits/chosen": -0.6574271321296692,
      "logits/rejected": -0.6152883768081665,
      "logps/chosen": -74.33702087402344,
      "logps/ref_chosen": -58.753684997558594,
      "logps/ref_rejected": -79.75001525878906,
      "logps/rejected": -125.26237487792969,
      "loss": 0.9612,
      "margin_dpo/margin_mean": 29.92901611328125,
      "margin_dpo/margin_std": 35.73463439941406,
      "step": 110
    },
    {
      "epoch": 0.16299559471365638,
      "fcm_dpo/beta": 0.022714221850037575,
      "fcm_dpo/delta": -0.15530934929847717,
      "fcm_dpo/margin": 32.667232513427734,
      "fcm_dpo/q_t": 0.347843736410141,
      "grad_norm": 27.264070510864258,
      "learning_rate": 4.944834074412042e-07,
      "logits/chosen": -0.6808880567550659,
      "logits/rejected": -0.6555056571960449,
      "logps/chosen": -87.53121948242188,
      "logps/ref_chosen": -68.62410736083984,
      "logps/ref_rejected": -98.42886352539062,
      "logps/rejected": -150.00320434570312,
      "loss": 0.9492,
      "margin_dpo/margin_mean": 32.667232513427734,
      "margin_dpo/margin_std": 39.969459533691406,
      "step": 111
    },
    {
      "epoch": 0.1644640234948605,
      "fcm_dpo/beta": 0.022934526205062866,
      "fcm_dpo/delta": 0.11087077856063843,
      "fcm_dpo/margin": 21.566303253173828,
      "fcm_dpo/q_t": 0.39232131838798523,
      "grad_norm": 25.337993621826172,
      "learning_rate": 4.942120794399002e-07,
      "logits/chosen": -0.6598977446556091,
      "logits/rejected": -0.613295316696167,
      "logps/chosen": -67.96663665771484,
      "logps/ref_chosen": -50.24964141845703,
      "logps/ref_rejected": -64.77442932128906,
      "logps/rejected": -104.05772399902344,
      "loss": 1.0543,
      "margin_dpo/margin_mean": 21.566301345825195,
      "margin_dpo/margin_std": 29.049222946166992,
      "step": 112
    },
    {
      "epoch": 0.16593245227606462,
      "fcm_dpo/beta": 0.023325610905885696,
      "fcm_dpo/delta": 0.040511354804039,
      "fcm_dpo/margin": 24.083799362182617,
      "fcm_dpo/q_t": 0.37566256523132324,
      "grad_norm": 25.780298233032227,
      "learning_rate": 4.939343162231841e-07,
      "logits/chosen": -0.647671639919281,
      "logits/rejected": -0.5998808741569519,
      "logps/chosen": -87.71575927734375,
      "logps/ref_chosen": -66.71295166015625,
      "logps/ref_rejected": -77.96870422363281,
      "logps/rejected": -123.0553207397461,
      "loss": 0.9905,
      "margin_dpo/margin_mean": 24.083797454833984,
      "margin_dpo/margin_std": 27.016586303710938,
      "step": 113
    },
    {
      "epoch": 0.16740088105726872,
      "fcm_dpo/beta": 0.02235432155430317,
      "fcm_dpo/delta": -0.24365481734275818,
      "fcm_dpo/margin": 36.693424224853516,
      "fcm_dpo/q_t": 0.33775386214256287,
      "grad_norm": 25.40207862854004,
      "learning_rate": 4.936501251103751e-07,
      "logits/chosen": -0.6867313385009766,
      "logits/rejected": -0.6511885523796082,
      "logps/chosen": -77.00772094726562,
      "logps/ref_chosen": -57.78507995605469,
      "logps/ref_rejected": -87.10966491699219,
      "logps/rejected": -143.02572631835938,
      "loss": 0.8963,
      "margin_dpo/margin_mean": 36.693424224853516,
      "margin_dpo/margin_std": 44.51530456542969,
      "step": 114
    },
    {
      "epoch": 0.16886930983847284,
      "fcm_dpo/beta": 0.022399384528398514,
      "fcm_dpo/delta": 0.014765216037631035,
      "fcm_dpo/margin": 26.151029586791992,
      "fcm_dpo/q_t": 0.38476622104644775,
      "grad_norm": 38.612613677978516,
      "learning_rate": 4.933595135901732e-07,
      "logits/chosen": -0.6795483827590942,
      "logits/rejected": -0.6489361524581909,
      "logps/chosen": -91.45716094970703,
      "logps/ref_chosen": -65.5826416015625,
      "logps/ref_rejected": -98.56552124023438,
      "logps/rejected": -150.591064453125,
      "loss": 1.0837,
      "margin_dpo/margin_mean": 26.15102767944336,
      "margin_dpo/margin_std": 44.87915802001953,
      "step": 115
    },
    {
      "epoch": 0.17033773861967694,
      "fcm_dpo/beta": 0.02236928418278694,
      "fcm_dpo/delta": -0.02009018138051033,
      "fcm_dpo/margin": 27.650157928466797,
      "fcm_dpo/q_t": 0.3669181764125824,
      "grad_norm": 28.57147979736328,
      "learning_rate": 4.930624893204624e-07,
      "logits/chosen": -0.6698657274246216,
      "logits/rejected": -0.6501831412315369,
      "logps/chosen": -70.91404724121094,
      "logps/ref_chosen": -51.40031433105469,
      "logps/ref_rejected": -80.5218505859375,
      "logps/rejected": -127.68574523925781,
      "loss": 0.9627,
      "margin_dpo/margin_mean": 27.650157928466797,
      "margin_dpo/margin_std": 31.316654205322266,
      "step": 116
    },
    {
      "epoch": 0.17180616740088106,
      "fcm_dpo/beta": 0.022425107657909393,
      "fcm_dpo/delta": 0.05522657185792923,
      "fcm_dpo/margin": 24.42792510986328,
      "fcm_dpo/q_t": 0.3833027184009552,
      "grad_norm": 33.80198287963867,
      "learning_rate": 4.927590601281083e-07,
      "logits/chosen": -0.6356416940689087,
      "logits/rejected": -0.594796895980835,
      "logps/chosen": -93.27058410644531,
      "logps/ref_chosen": -69.29840850830078,
      "logps/ref_rejected": -66.583984375,
      "logps/rejected": -114.98409271240234,
      "loss": 1.0417,
      "margin_dpo/margin_mean": 24.427927017211914,
      "margin_dpo/margin_std": 34.87114715576172,
      "step": 117
    },
    {
      "epoch": 0.17327459618208516,
      "fcm_dpo/beta": 0.022451553493738174,
      "fcm_dpo/delta": -0.012901581823825836,
      "fcm_dpo/margin": 27.263755798339844,
      "fcm_dpo/q_t": 0.36932849884033203,
      "grad_norm": 27.19830894470215,
      "learning_rate": 4.924492340087524e-07,
      "logits/chosen": -0.6901913285255432,
      "logits/rejected": -0.6656088829040527,
      "logps/chosen": -76.18692779541016,
      "logps/ref_chosen": -55.6409797668457,
      "logps/ref_rejected": -75.66905975341797,
      "logps/rejected": -123.478759765625,
      "loss": 0.9741,
      "margin_dpo/margin_mean": 27.263755798339844,
      "margin_dpo/margin_std": 32.05895233154297,
      "step": 118
    },
    {
      "epoch": 0.17474302496328928,
      "fcm_dpo/beta": 0.02224084362387657,
      "fcm_dpo/delta": -0.010049469769001007,
      "fcm_dpo/margin": 27.355573654174805,
      "fcm_dpo/q_t": 0.37495267391204834,
      "grad_norm": 29.576772689819336,
      "learning_rate": 4.92133019126601e-07,
      "logits/chosen": -0.6743392944335938,
      "logits/rejected": -0.6560527682304382,
      "logps/chosen": -100.41120910644531,
      "logps/ref_chosen": -73.51019287109375,
      "logps/ref_rejected": -102.977294921875,
      "logps/rejected": -157.23387145996094,
      "loss": 1.0134,
      "margin_dpo/margin_mean": 27.355573654174805,
      "margin_dpo/margin_std": 37.5633430480957,
      "step": 119
    },
    {
      "epoch": 0.1762114537444934,
      "fcm_dpo/beta": 0.02168981544673443,
      "fcm_dpo/delta": -0.2090209424495697,
      "fcm_dpo/margin": 36.529258728027344,
      "fcm_dpo/q_t": 0.3369954228401184,
      "grad_norm": 27.668405532836914,
      "learning_rate": 4.918104238142103e-07,
      "logits/chosen": -0.6739026308059692,
      "logits/rejected": -0.6356397867202759,
      "logps/chosen": -105.38459777832031,
      "logps/ref_chosen": -76.78083801269531,
      "logps/ref_rejected": -108.02374267578125,
      "logps/rejected": -173.15676879882812,
      "loss": 0.9002,
      "margin_dpo/margin_mean": 36.529258728027344,
      "margin_dpo/margin_std": 40.862762451171875,
      "step": 120
    },
    {
      "epoch": 0.1776798825256975,
      "fcm_dpo/beta": 0.020538385957479477,
      "fcm_dpo/delta": -0.2580873370170593,
      "fcm_dpo/margin": 40.62395477294922,
      "fcm_dpo/q_t": 0.33450600504875183,
      "grad_norm": 27.22615623474121,
      "learning_rate": 4.91481456572267e-07,
      "logits/chosen": -0.6262691020965576,
      "logits/rejected": -0.6172099113464355,
      "logps/chosen": -89.005126953125,
      "logps/ref_chosen": -61.789894104003906,
      "logps/ref_rejected": -109.99456787109375,
      "logps/rejected": -177.83375549316406,
      "loss": 0.9049,
      "margin_dpo/margin_mean": 40.623958587646484,
      "margin_dpo/margin_std": 47.878326416015625,
      "step": 121
    },
    {
      "epoch": 0.17914831130690162,
      "fcm_dpo/beta": 0.01918221451342106,
      "fcm_dpo/delta": -0.4117882251739502,
      "fcm_dpo/margin": 50.65019989013672,
      "fcm_dpo/q_t": 0.29864490032196045,
      "grad_norm": 28.19658660888672,
      "learning_rate": 4.911461260693638e-07,
      "logits/chosen": -0.6220955848693848,
      "logits/rejected": -0.6330730319023132,
      "logps/chosen": -71.84508514404297,
      "logps/ref_chosen": -46.9022102355957,
      "logps/ref_rejected": -106.71418762207031,
      "logps/rejected": -182.30726623535156,
      "loss": 0.7772,
      "margin_dpo/margin_mean": 50.65019989013672,
      "margin_dpo/margin_std": 44.97718048095703,
      "step": 122
    },
    {
      "epoch": 0.18061674008810572,
      "fcm_dpo/beta": 0.018608931452035904,
      "fcm_dpo/delta": -0.027324281632900238,
      "fcm_dpo/margin": 33.586708068847656,
      "fcm_dpo/q_t": 0.374822199344635,
      "grad_norm": 26.818113327026367,
      "learning_rate": 4.908044411417711e-07,
      "logits/chosen": -0.612564206123352,
      "logits/rejected": -0.5866974592208862,
      "logps/chosen": -88.52189636230469,
      "logps/ref_chosen": -61.33863830566406,
      "logps/ref_rejected": -87.775390625,
      "logps/rejected": -148.5453643798828,
      "loss": 1.0431,
      "margin_dpo/margin_mean": 33.586708068847656,
      "margin_dpo/margin_std": 51.27476501464844,
      "step": 123
    },
    {
      "epoch": 0.18208516886930984,
      "fcm_dpo/beta": 0.017872150987386703,
      "fcm_dpo/delta": -0.27416178584098816,
      "fcm_dpo/margin": 47.546592712402344,
      "fcm_dpo/q_t": 0.34260714054107666,
      "grad_norm": 28.516874313354492,
      "learning_rate": 4.904564107932048e-07,
      "logits/chosen": -0.6066104173660278,
      "logits/rejected": -0.6043754816055298,
      "logps/chosen": -101.30255126953125,
      "logps/ref_chosen": -71.44833374023438,
      "logps/ref_rejected": -117.58056640625,
      "logps/rejected": -194.98138427734375,
      "loss": 0.9423,
      "margin_dpo/margin_mean": 47.546592712402344,
      "margin_dpo/margin_std": 64.18199157714844,
      "step": 124
    },
    {
      "epoch": 0.18355359765051396,
      "fcm_dpo/beta": 0.017282985150814056,
      "fcm_dpo/delta": -0.17058980464935303,
      "fcm_dpo/margin": 43.85457229614258,
      "fcm_dpo/q_t": 0.3455929458141327,
      "grad_norm": 24.23063087463379,
      "learning_rate": 4.90102044194588e-07,
      "logits/chosen": -0.5887873768806458,
      "logits/rejected": -0.5830005407333374,
      "logps/chosen": -74.79535675048828,
      "logps/ref_chosen": -50.136940002441406,
      "logps/ref_rejected": -83.98861694335938,
      "logps/rejected": -152.50160217285156,
      "loss": 0.9197,
      "margin_dpo/margin_mean": 43.85457229614258,
      "margin_dpo/margin_std": 50.49039077758789,
      "step": 125
    },
    {
      "epoch": 0.18502202643171806,
      "fcm_dpo/beta": 0.016843030229210854,
      "fcm_dpo/delta": -0.08583886921405792,
      "fcm_dpo/margin": 40.36263656616211,
      "fcm_dpo/q_t": 0.35755184292793274,
      "grad_norm": 26.442380905151367,
      "learning_rate": 4.897413506838102e-07,
      "logits/chosen": -0.5875859260559082,
      "logits/rejected": -0.5705457925796509,
      "logps/chosen": -82.85122680664062,
      "logps/ref_chosen": -55.66706848144531,
      "logps/ref_rejected": -98.1297607421875,
      "logps/rejected": -165.67654418945312,
      "loss": 0.9522,
      "margin_dpo/margin_mean": 40.36263656616211,
      "margin_dpo/margin_std": 47.85084533691406,
      "step": 126
    },
    {
      "epoch": 0.18649045521292218,
      "fcm_dpo/beta": 0.0169706791639328,
      "fcm_dpo/delta": 0.061831459403038025,
      "fcm_dpo/margin": 31.902957916259766,
      "fcm_dpo/q_t": 0.38108593225479126,
      "grad_norm": 26.799118041992188,
      "learning_rate": 4.89374339765481e-07,
      "logits/chosen": -0.576623797416687,
      "logits/rejected": -0.5499337911605835,
      "logps/chosen": -82.35685729980469,
      "logps/ref_chosen": -56.55467987060547,
      "logps/ref_rejected": -76.7957763671875,
      "logps/rejected": -134.50091552734375,
      "loss": 1.0288,
      "margin_dpo/margin_mean": 31.902957916259766,
      "margin_dpo/margin_std": 41.576568603515625,
      "step": 127
    },
    {
      "epoch": 0.18795888399412627,
      "fcm_dpo/beta": 0.017191898077726364,
      "fcm_dpo/delta": 0.03887506574392319,
      "fcm_dpo/margin": 32.724117279052734,
      "fcm_dpo/q_t": 0.3826625347137451,
      "grad_norm": 38.53647994995117,
      "learning_rate": 4.890010211106795e-07,
      "logits/chosen": -0.5986074209213257,
      "logits/rejected": -0.5656411051750183,
      "logps/chosen": -86.5602035522461,
      "logps/ref_chosen": -58.12095642089844,
      "logps/ref_rejected": -76.43896484375,
      "logps/rejected": -137.60232543945312,
      "loss": 1.0607,
      "margin_dpo/margin_mean": 32.72411346435547,
      "margin_dpo/margin_std": 49.83234405517578,
      "step": 128
    },
    {
      "epoch": 0.1894273127753304,
      "fcm_dpo/beta": 0.017184410244226456,
      "fcm_dpo/delta": 0.005901549011468887,
      "fcm_dpo/margin": 34.58758544921875,
      "fcm_dpo/q_t": 0.3824169635772705,
      "grad_norm": 28.598745346069336,
      "learning_rate": 4.88621404556699e-07,
      "logits/chosen": -0.6040947437286377,
      "logits/rejected": -0.5859910845756531,
      "logps/chosen": -102.27650451660156,
      "logps/ref_chosen": -66.91637420654297,
      "logps/ref_rejected": -96.6422119140625,
      "logps/rejected": -166.58993530273438,
      "loss": 1.0624,
      "margin_dpo/margin_mean": 34.58758544921875,
      "margin_dpo/margin_std": 55.73461151123047,
      "step": 129
    },
    {
      "epoch": 0.19089574155653452,
      "fcm_dpo/beta": 0.016697879880666733,
      "fcm_dpo/delta": -0.2792219817638397,
      "fcm_dpo/margin": 51.21843338012695,
      "fcm_dpo/q_t": 0.3288246989250183,
      "grad_norm": 22.612878799438477,
      "learning_rate": 4.882355001067891e-07,
      "logits/chosen": -0.5792099237442017,
      "logits/rejected": -0.5733057260513306,
      "logps/chosen": -71.08717346191406,
      "logps/ref_chosen": -44.66685104370117,
      "logps/ref_rejected": -82.78165435791016,
      "logps/rejected": -160.42041015625,
      "loss": 0.8943,
      "margin_dpo/margin_mean": 51.21843338012695,
      "margin_dpo/margin_std": 56.39968490600586,
      "step": 130
    },
    {
      "epoch": 0.19236417033773862,
      "fcm_dpo/beta": 0.015755565837025642,
      "fcm_dpo/delta": -0.18772238492965698,
      "fcm_dpo/margin": 49.05485916137695,
      "fcm_dpo/q_t": 0.3336232304573059,
      "grad_norm": 26.170124053955078,
      "learning_rate": 4.878433179298909e-07,
      "logits/chosen": -0.5605712532997131,
      "logits/rejected": -0.564369261264801,
      "logps/chosen": -66.87252044677734,
      "logps/ref_chosen": -44.924591064453125,
      "logps/ref_rejected": -88.44401550292969,
      "logps/rejected": -159.44680786132812,
      "loss": 0.8818,
      "margin_dpo/margin_mean": 49.05485916137695,
      "margin_dpo/margin_std": 49.06263732910156,
      "step": 131
    },
    {
      "epoch": 0.19383259911894274,
      "fcm_dpo/beta": 0.015453611500561237,
      "fcm_dpo/delta": -0.05859919637441635,
      "fcm_dpo/margin": 42.35327911376953,
      "fcm_dpo/q_t": 0.36758387088775635,
      "grad_norm": 25.90082359313965,
      "learning_rate": 4.874448683603694e-07,
      "logits/chosen": -0.6056761741638184,
      "logits/rejected": -0.5935735702514648,
      "logps/chosen": -89.5750732421875,
      "logps/ref_chosen": -59.00108337402344,
      "logps/ref_rejected": -87.89215087890625,
      "logps/rejected": -160.81942749023438,
      "loss": 0.9903,
      "margin_dpo/margin_mean": 42.35327911376953,
      "margin_dpo/margin_std": 57.95384979248047,
      "step": 132
    },
    {
      "epoch": 0.19530102790014683,
      "fcm_dpo/beta": 0.01545734517276287,
      "fcm_dpo/delta": 0.0167352557182312,
      "fcm_dpo/margin": 37.79522705078125,
      "fcm_dpo/q_t": 0.38009342551231384,
      "grad_norm": 27.229202270507812,
      "learning_rate": 4.870401618977415e-07,
      "logits/chosen": -0.5968517065048218,
      "logits/rejected": -0.579937219619751,
      "logps/chosen": -102.51626586914062,
      "logps/ref_chosen": -66.60449981689453,
      "logps/ref_rejected": -96.33355712890625,
      "logps/rejected": -170.04055786132812,
      "loss": 1.0233,
      "margin_dpo/margin_mean": 37.795230865478516,
      "margin_dpo/margin_std": 53.25979995727539,
      "step": 133
    },
    {
      "epoch": 0.19676945668135096,
      "fcm_dpo/beta": 0.01544961053878069,
      "fcm_dpo/delta": -0.046767231076955795,
      "fcm_dpo/margin": 41.66034698486328,
      "fcm_dpo/q_t": 0.3632383346557617,
      "grad_norm": 24.29633903503418,
      "learning_rate": 4.866292092063986e-07,
      "logits/chosen": -0.562139630317688,
      "logits/rejected": -0.5372625589370728,
      "logps/chosen": -80.47048950195312,
      "logps/ref_chosen": -52.06925582885742,
      "logps/ref_rejected": -87.6545181274414,
      "logps/rejected": -157.71609497070312,
      "loss": 0.9512,
      "margin_dpo/margin_mean": 41.66035079956055,
      "margin_dpo/margin_std": 47.288726806640625,
      "step": 134
    },
    {
      "epoch": 0.19823788546255505,
      "fcm_dpo/beta": 0.014866790734231472,
      "fcm_dpo/delta": -0.21857857704162598,
      "fcm_dpo/margin": 53.88473892211914,
      "fcm_dpo/q_t": 0.3398352265357971,
      "grad_norm": 25.403532028198242,
      "learning_rate": 4.862120211153265e-07,
      "logits/chosen": -0.6001534461975098,
      "logits/rejected": -0.627263069152832,
      "logps/chosen": -82.99242401123047,
      "logps/ref_chosen": -50.353858947753906,
      "logps/ref_rejected": -115.97975158691406,
      "logps/rejected": -202.5030517578125,
      "loss": 0.9102,
      "margin_dpo/margin_mean": 53.88473892211914,
      "margin_dpo/margin_std": 62.925498962402344,
      "step": 135
    },
    {
      "epoch": 0.19970631424375918,
      "fcm_dpo/beta": 0.014702379703521729,
      "fcm_dpo/delta": 0.05534332990646362,
      "fcm_dpo/margin": 37.198089599609375,
      "fcm_dpo/q_t": 0.3941587805747986,
      "grad_norm": 27.017377853393555,
      "learning_rate": 4.857886086178193e-07,
      "logits/chosen": -0.5803062915802002,
      "logits/rejected": -0.5640683174133301,
      "logps/chosen": -103.38967895507812,
      "logps/ref_chosen": -65.072509765625,
      "logps/ref_rejected": -96.32122802734375,
      "logps/rejected": -171.83648681640625,
      "loss": 1.0679,
      "margin_dpo/margin_mean": 37.198089599609375,
      "margin_dpo/margin_std": 59.631011962890625,
      "step": 136
    },
    {
      "epoch": 0.2011747430249633,
      "fcm_dpo/beta": 0.014293761923909187,
      "fcm_dpo/delta": -0.24502620100975037,
      "fcm_dpo/margin": 57.65531921386719,
      "fcm_dpo/q_t": 0.3446000814437866,
      "grad_norm": 23.374046325683594,
      "learning_rate": 4.853589828711902e-07,
      "logits/chosen": -0.5490569472312927,
      "logits/rejected": -0.5711492300033569,
      "logps/chosen": -87.5726089477539,
      "logps/ref_chosen": -48.759117126464844,
      "logps/ref_rejected": -113.86376953125,
      "logps/rejected": -210.33258056640625,
      "loss": 0.9452,
      "margin_dpo/margin_mean": 57.65531921386719,
      "margin_dpo/margin_std": 76.925048828125,
      "step": 137
    },
    {
      "epoch": 0.2026431718061674,
      "fcm_dpo/beta": 0.014147358015179634,
      "fcm_dpo/delta": -0.0190581101924181,
      "fcm_dpo/margin": 43.66788864135742,
      "fcm_dpo/q_t": 0.3662746548652649,
      "grad_norm": 26.790586471557617,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": -0.5593977570533752,
      "logits/rejected": -0.5456332564353943,
      "logps/chosen": -101.01687622070312,
      "logps/ref_chosen": -60.519649505615234,
      "logps/ref_rejected": -93.19694519042969,
      "logps/rejected": -177.362060546875,
      "loss": 0.9758,
      "margin_dpo/margin_mean": 43.667884826660156,
      "margin_dpo/margin_std": 51.35163879394531,
      "step": 138
    },
    {
      "epoch": 0.20411160058737152,
      "fcm_dpo/beta": 0.01379378791898489,
      "fcm_dpo/delta": -0.16102235019207,
      "fcm_dpo/margin": 54.313804626464844,
      "fcm_dpo/q_t": 0.34360840916633606,
      "grad_norm": 22.88560676574707,
      "learning_rate": 4.844811370781446e-07,
      "logits/chosen": -0.5527390241622925,
      "logits/rejected": -0.5373610258102417,
      "logps/chosen": -79.2032470703125,
      "logps/ref_chosen": -46.89138412475586,
      "logps/ref_rejected": -79.72798156738281,
      "logps/rejected": -166.3536376953125,
      "loss": 0.906,
      "margin_dpo/margin_mean": 54.313804626464844,
      "margin_dpo/margin_std": 59.772552490234375,
      "step": 139
    },
    {
      "epoch": 0.2055800293685756,
      "fcm_dpo/beta": 0.013468625023961067,
      "fcm_dpo/delta": -0.0790109634399414,
      "fcm_dpo/margin": 50.007423400878906,
      "fcm_dpo/q_t": 0.36189499497413635,
      "grad_norm": 24.64659309387207,
      "learning_rate": 4.840329401637809e-07,
      "logits/chosen": -0.5458736419677734,
      "logits/rejected": -0.5282671451568604,
      "logps/chosen": -98.45331573486328,
      "logps/ref_chosen": -58.97471618652344,
      "logps/ref_rejected": -83.28410339355469,
      "logps/rejected": -172.77012634277344,
      "loss": 0.9788,
      "margin_dpo/margin_mean": 50.007423400878906,
      "margin_dpo/margin_std": 64.80023193359375,
      "step": 140
    },
    {
      "epoch": 0.20704845814977973,
      "fcm_dpo/beta": 0.013357013463973999,
      "fcm_dpo/delta": -0.04534909874200821,
      "fcm_dpo/margin": 48.098968505859375,
      "fcm_dpo/q_t": 0.3626646399497986,
      "grad_norm": 28.8856258392334,
      "learning_rate": 4.83578576263792e-07,
      "logits/chosen": -0.5459233522415161,
      "logits/rejected": -0.5275824069976807,
      "logps/chosen": -116.89370727539062,
      "logps/ref_chosen": -75.07566833496094,
      "logps/ref_rejected": -98.1922607421875,
      "logps/rejected": -188.10928344726562,
      "loss": 0.9921,
      "margin_dpo/margin_mean": 48.098968505859375,
      "margin_dpo/margin_std": 62.48729705810547,
      "step": 141
    },
    {
      "epoch": 0.20851688693098386,
      "fcm_dpo/beta": 0.01312628760933876,
      "fcm_dpo/delta": -0.13432356715202332,
      "fcm_dpo/margin": 55.20783233642578,
      "fcm_dpo/q_t": 0.35684800148010254,
      "grad_norm": 28.54540252685547,
      "learning_rate": 4.83118057351089e-07,
      "logits/chosen": -0.5353626608848572,
      "logits/rejected": -0.5301727056503296,
      "logps/chosen": -103.46934509277344,
      "logps/ref_chosen": -58.027931213378906,
      "logps/ref_rejected": -94.58222961425781,
      "logps/rejected": -195.23147583007812,
      "loss": 0.9917,
      "margin_dpo/margin_mean": 55.20783233642578,
      "margin_dpo/margin_std": 75.06365203857422,
      "step": 142
    },
    {
      "epoch": 0.20998531571218795,
      "fcm_dpo/beta": 0.013182537630200386,
      "fcm_dpo/delta": 0.14479340612888336,
      "fcm_dpo/margin": 35.0554313659668,
      "fcm_dpo/q_t": 0.4067147374153137,
      "grad_norm": 31.684499740600586,
      "learning_rate": 4.826513955607734e-07,
      "logits/chosen": -0.5001394152641296,
      "logits/rejected": -0.48260974884033203,
      "logps/chosen": -107.36073303222656,
      "logps/ref_chosen": -57.59645080566406,
      "logps/ref_rejected": -78.99957275390625,
      "logps/rejected": -163.81927490234375,
      "loss": 1.1328,
      "margin_dpo/margin_mean": 35.05542755126953,
      "margin_dpo/margin_std": 64.47492980957031,
      "step": 143
    },
    {
      "epoch": 0.21145374449339208,
      "fcm_dpo/beta": 0.0133123230189085,
      "fcm_dpo/delta": -0.005581766366958618,
      "fcm_dpo/margin": 45.4619255065918,
      "fcm_dpo/q_t": 0.3696960508823395,
      "grad_norm": 24.83319854736328,
      "learning_rate": 4.821786031898176e-07,
      "logits/chosen": -0.5548040866851807,
      "logits/rejected": -0.5310955047607422,
      "logps/chosen": -100.95870971679688,
      "logps/ref_chosen": -59.90636444091797,
      "logps/ref_rejected": -82.00025939941406,
      "logps/rejected": -168.51454162597656,
      "loss": 0.9842,
      "margin_dpo/margin_mean": 45.46192932128906,
      "margin_dpo/margin_std": 54.452301025390625,
      "step": 144
    },
    {
      "epoch": 0.21292217327459617,
      "fcm_dpo/beta": 0.013245116919279099,
      "fcm_dpo/delta": -0.03191668167710304,
      "fcm_dpo/margin": 47.55194854736328,
      "fcm_dpo/q_t": 0.3665177524089813,
      "grad_norm": 29.93503189086914,
      "learning_rate": 4.816996926967401e-07,
      "logits/chosen": -0.5175589323043823,
      "logits/rejected": -0.48851436376571655,
      "logps/chosen": -98.98822021484375,
      "logps/ref_chosen": -56.60066604614258,
      "logps/ref_rejected": -77.86631774902344,
      "logps/rejected": -167.80581665039062,
      "loss": 0.9815,
      "margin_dpo/margin_mean": 47.55194854736328,
      "margin_dpo/margin_std": 58.7222900390625,
      "step": 145
    },
    {
      "epoch": 0.2143906020558003,
      "fcm_dpo/beta": 0.013389000669121742,
      "fcm_dpo/delta": 0.0990617573261261,
      "fcm_dpo/margin": 37.79475021362305,
      "fcm_dpo/q_t": 0.3926282227039337,
      "grad_norm": 36.985809326171875,
      "learning_rate": 4.812146767012779e-07,
      "logits/chosen": -0.5096883177757263,
      "logits/rejected": -0.4698811173439026,
      "logps/chosen": -124.57284545898438,
      "logps/ref_chosen": -66.00045013427734,
      "logps/ref_rejected": -81.70278930664062,
      "logps/rejected": -178.0699462890625,
      "loss": 1.1025,
      "margin_dpo/margin_mean": 37.79475021362305,
      "margin_dpo/margin_std": 62.888580322265625,
      "step": 146
    },
    {
      "epoch": 0.21585903083700442,
      "fcm_dpo/beta": 0.013290293514728546,
      "fcm_dpo/delta": -0.0870274156332016,
      "fcm_dpo/margin": 51.21372985839844,
      "fcm_dpo/q_t": 0.36140725016593933,
      "grad_norm": 26.684223175048828,
      "learning_rate": 4.807235679840536e-07,
      "logits/chosen": -0.5257313251495361,
      "logits/rejected": -0.49477505683898926,
      "logps/chosen": -96.1661605834961,
      "logps/ref_chosen": -53.405487060546875,
      "logps/ref_rejected": -71.39060974121094,
      "logps/rejected": -165.36502075195312,
      "loss": 0.9801,
      "margin_dpo/margin_mean": 51.21372985839844,
      "margin_dpo/margin_std": 67.14807891845703,
      "step": 147
    },
    {
      "epoch": 0.2173274596182085,
      "fcm_dpo/beta": 0.01351265050470829,
      "fcm_dpo/delta": 0.03601977974176407,
      "fcm_dpo/margin": 41.63152313232422,
      "fcm_dpo/q_t": 0.3840796947479248,
      "grad_norm": 26.11017608642578,
      "learning_rate": 4.802263794862384e-07,
      "logits/chosen": -0.5603736639022827,
      "logits/rejected": -0.5478102564811707,
      "logps/chosen": -106.5193862915039,
      "logps/ref_chosen": -64.93708038330078,
      "logps/ref_rejected": -103.09384155273438,
      "logps/rejected": -186.30767822265625,
      "loss": 1.0332,
      "margin_dpo/margin_mean": 41.63152313232422,
      "margin_dpo/margin_std": 55.343299865722656,
      "step": 148
    },
    {
      "epoch": 0.21879588839941264,
      "fcm_dpo/beta": 0.013042960315942764,
      "fcm_dpo/delta": -0.0962439477443695,
      "fcm_dpo/margin": 52.64115905761719,
      "fcm_dpo/q_t": 0.35281527042388916,
      "grad_norm": 24.8011417388916,
      "learning_rate": 4.797231243092118e-07,
      "logits/chosen": -0.590922474861145,
      "logits/rejected": -0.5712438225746155,
      "logps/chosen": -99.46907043457031,
      "logps/ref_chosen": -58.47376251220703,
      "logps/ref_rejected": -99.31474304199219,
      "logps/rejected": -192.95120239257812,
      "loss": 0.9376,
      "margin_dpo/margin_mean": 52.64115905761719,
      "margin_dpo/margin_std": 57.46501922607422,
      "step": 149
    },
    {
      "epoch": 0.22026431718061673,
      "fcm_dpo/beta": 0.012883552350103855,
      "fcm_dpo/delta": -0.049887098371982574,
      "fcm_dpo/margin": 49.994728088378906,
      "fcm_dpo/q_t": 0.37187010049819946,
      "grad_norm": 24.784076690673828,
      "learning_rate": 4.792138157142157e-07,
      "logits/chosen": -0.557221531867981,
      "logits/rejected": -0.5565592050552368,
      "logps/chosen": -82.40767669677734,
      "logps/ref_chosen": -45.705810546875,
      "logps/ref_rejected": -83.34759521484375,
      "logps/rejected": -170.044189453125,
      "loss": 0.9916,
      "margin_dpo/margin_mean": 49.994728088378906,
      "margin_dpo/margin_std": 68.09803771972656,
      "step": 150
    },
    {
      "epoch": 0.22173274596182085,
      "fcm_dpo/beta": 0.012838941067457199,
      "fcm_dpo/delta": -0.08010067045688629,
      "fcm_dpo/margin": 52.55384063720703,
      "fcm_dpo/q_t": 0.35465726256370544,
      "grad_norm": 29.252914428710938,
      "learning_rate": 4.786984671220053e-07,
      "logits/chosen": -0.6052649021148682,
      "logits/rejected": -0.569302499294281,
      "logps/chosen": -114.0267333984375,
      "logps/ref_chosen": -70.57083129882812,
      "logps/ref_rejected": -100.46382141113281,
      "logps/rejected": -196.47357177734375,
      "loss": 0.9399,
      "margin_dpo/margin_mean": 52.55384063720703,
      "margin_dpo/margin_std": 58.15677261352539,
      "step": 151
    },
    {
      "epoch": 0.22320117474302498,
      "fcm_dpo/beta": 0.012523850426077843,
      "fcm_dpo/delta": -0.16435271501541138,
      "fcm_dpo/margin": 60.06843185424805,
      "fcm_dpo/q_t": 0.34519827365875244,
      "grad_norm": 23.900650024414062,
      "learning_rate": 4.78177092112495e-07,
      "logits/chosen": -0.588593602180481,
      "logits/rejected": -0.5805606842041016,
      "logps/chosen": -98.1216049194336,
      "logps/ref_chosen": -60.16438674926758,
      "logps/ref_rejected": -106.14045715332031,
      "logps/rejected": -204.16610717773438,
      "loss": 0.918,
      "margin_dpo/margin_mean": 60.06843185424805,
      "margin_dpo/margin_std": 66.94328308105469,
      "step": 152
    },
    {
      "epoch": 0.22466960352422907,
      "fcm_dpo/beta": 0.012331589125096798,
      "fcm_dpo/delta": -0.03408785164356232,
      "fcm_dpo/margin": 51.24233627319336,
      "fcm_dpo/q_t": 0.37102195620536804,
      "grad_norm": 21.188068389892578,
      "learning_rate": 4.776497044244016e-07,
      "logits/chosen": -0.576537013053894,
      "logits/rejected": -0.5661931037902832,
      "logps/chosen": -94.53947448730469,
      "logps/ref_chosen": -56.315277099609375,
      "logps/ref_rejected": -85.65583801269531,
      "logps/rejected": -175.12237548828125,
      "loss": 1.0004,
      "margin_dpo/margin_mean": 51.24233627319336,
      "margin_dpo/margin_std": 70.23153686523438,
      "step": 153
    },
    {
      "epoch": 0.2261380323054332,
      "fcm_dpo/beta": 0.012307348661124706,
      "fcm_dpo/delta": -0.028648383915424347,
      "fcm_dpo/margin": 50.88260269165039,
      "fcm_dpo/q_t": 0.37250685691833496,
      "grad_norm": 26.30771255493164,
      "learning_rate": 4.771163179548808e-07,
      "logits/chosen": -0.5932099223136902,
      "logits/rejected": -0.5894089937210083,
      "logps/chosen": -107.8296127319336,
      "logps/ref_chosen": -62.74256896972656,
      "logps/ref_rejected": -104.24420166015625,
      "logps/rejected": -200.21383666992188,
      "loss": 1.0289,
      "margin_dpo/margin_mean": 50.88260269165039,
      "margin_dpo/margin_std": 72.57513427734375,
      "step": 154
    },
    {
      "epoch": 0.2276064610866373,
      "fcm_dpo/beta": 0.012188572436571121,
      "fcm_dpo/delta": -0.041290249675512314,
      "fcm_dpo/margin": 52.38173294067383,
      "fcm_dpo/q_t": 0.3654828369617462,
      "grad_norm": 24.021343231201172,
      "learning_rate": 4.7657694675916247e-07,
      "logits/chosen": -0.6054894924163818,
      "logits/rejected": -0.581174373626709,
      "logps/chosen": -101.20684814453125,
      "logps/ref_chosen": -60.65318298339844,
      "logps/ref_rejected": -77.49220275878906,
      "logps/rejected": -170.42758178710938,
      "loss": 0.9842,
      "margin_dpo/margin_mean": 52.38173294067383,
      "margin_dpo/margin_std": 66.93553161621094,
      "step": 155
    },
    {
      "epoch": 0.2290748898678414,
      "fcm_dpo/beta": 0.012476801872253418,
      "fcm_dpo/delta": 0.20647725462913513,
      "fcm_dpo/margin": 32.21952819824219,
      "fcm_dpo/q_t": 0.41537606716156006,
      "grad_norm": 38.838287353515625,
      "learning_rate": 4.7603160505017893e-07,
      "logits/chosen": -0.531362771987915,
      "logits/rejected": -0.5084357857704163,
      "logps/chosen": -126.44786071777344,
      "logps/ref_chosen": -69.49188232421875,
      "logps/ref_rejected": -77.16929626464844,
      "logps/rejected": -166.3448028564453,
      "loss": 1.2212,
      "margin_dpo/margin_mean": 32.21952819824219,
      "margin_dpo/margin_std": 76.02751159667969,
      "step": 156
    },
    {
      "epoch": 0.2305433186490455,
      "fcm_dpo/beta": 0.012192957103252411,
      "fcm_dpo/delta": -0.18633562326431274,
      "fcm_dpo/margin": 63.14281463623047,
      "fcm_dpo/q_t": 0.33702823519706726,
      "grad_norm": 27.627710342407227,
      "learning_rate": 4.7548030719819154e-07,
      "logits/chosen": -0.5252609252929688,
      "logits/rejected": -0.5241682529449463,
      "logps/chosen": -113.81539916992188,
      "logps/ref_chosen": -61.368438720703125,
      "logps/ref_rejected": -107.64636993408203,
      "logps/rejected": -223.23614501953125,
      "loss": 0.9149,
      "margin_dpo/margin_mean": 63.14281463623047,
      "margin_dpo/margin_std": 69.51899719238281,
      "step": 157
    },
    {
      "epoch": 0.23201174743024963,
      "fcm_dpo/beta": 0.011821776628494263,
      "fcm_dpo/delta": -0.1891898512840271,
      "fcm_dpo/margin": 65.51558685302734,
      "fcm_dpo/q_t": 0.35599738359451294,
      "grad_norm": 23.29702377319336,
      "learning_rate": 4.7492306773041136e-07,
      "logits/chosen": -0.5330479145050049,
      "logits/rejected": -0.5438076257705688,
      "logps/chosen": -108.17401123046875,
      "logps/ref_chosen": -57.612918853759766,
      "logps/ref_rejected": -113.6946792602539,
      "logps/rejected": -229.7713623046875,
      "loss": 0.9692,
      "margin_dpo/margin_mean": 65.51559448242188,
      "margin_dpo/margin_std": 92.29185485839844,
      "step": 158
    },
    {
      "epoch": 0.23348017621145375,
      "fcm_dpo/beta": 0.011839054524898529,
      "fcm_dpo/delta": 0.03908521309494972,
      "fcm_dpo/margin": 47.499595642089844,
      "fcm_dpo/q_t": 0.38455915451049805,
      "grad_norm": 30.979955673217773,
      "learning_rate": 4.743599013306165e-07,
      "logits/chosen": -0.5508168935775757,
      "logits/rejected": -0.5111355781555176,
      "logps/chosen": -139.3695526123047,
      "logps/ref_chosen": -81.56034851074219,
      "logps/ref_rejected": -88.89871215820312,
      "logps/rejected": -194.20751953125,
      "loss": 1.0618,
      "margin_dpo/margin_mean": 47.499595642089844,
      "margin_dpo/margin_std": 73.20787811279297,
      "step": 159
    },
    {
      "epoch": 0.23494860499265785,
      "fcm_dpo/beta": 0.011559359729290009,
      "fcm_dpo/delta": -0.1065782979130745,
      "fcm_dpo/margin": 60.404293060302734,
      "fcm_dpo/q_t": 0.365913450717926,
      "grad_norm": 25.069011688232422,
      "learning_rate": 4.737908228387656e-07,
      "logits/chosen": -0.5436577796936035,
      "logits/rejected": -0.5286530256271362,
      "logps/chosen": -123.36406707763672,
      "logps/ref_chosen": -65.73088073730469,
      "logps/ref_rejected": -97.21781921386719,
      "logps/rejected": -215.25531005859375,
      "loss": 1.0081,
      "margin_dpo/margin_mean": 60.404296875,
      "margin_dpo/margin_std": 88.11878967285156,
      "step": 160
    },
    {
      "epoch": 0.23641703377386197,
      "fcm_dpo/beta": 0.01151471957564354,
      "fcm_dpo/delta": -0.025925662368535995,
      "fcm_dpo/margin": 54.21880340576172,
      "fcm_dpo/q_t": 0.3725304901599884,
      "grad_norm": 25.731287002563477,
      "learning_rate": 4.7321584725060594e-07,
      "logits/chosen": -0.5345560312271118,
      "logits/rejected": -0.5234242677688599,
      "logps/chosen": -102.83772277832031,
      "logps/ref_chosen": -52.43647003173828,
      "logps/ref_rejected": -83.43095397949219,
      "logps/rejected": -188.05099487304688,
      "loss": 1.0083,
      "margin_dpo/margin_mean": 54.21880340576172,
      "margin_dpo/margin_std": 74.05145263671875,
      "step": 161
    },
    {
      "epoch": 0.23788546255506607,
      "fcm_dpo/beta": 0.011299725621938705,
      "fcm_dpo/delta": -0.03639139235019684,
      "fcm_dpo/margin": 55.859214782714844,
      "fcm_dpo/q_t": 0.37161916494369507,
      "grad_norm": 23.54488182067871,
      "learning_rate": 4.7263498971727905e-07,
      "logits/chosen": -0.5820114612579346,
      "logits/rejected": -0.5577338933944702,
      "logps/chosen": -109.61683654785156,
      "logps/ref_chosen": -62.6105842590332,
      "logps/ref_rejected": -89.39057922363281,
      "logps/rejected": -192.25604248046875,
      "loss": 1.0073,
      "margin_dpo/margin_mean": 55.859214782714844,
      "margin_dpo/margin_std": 76.0927734375,
      "step": 162
    },
    {
      "epoch": 0.2393538913362702,
      "fcm_dpo/beta": 0.01139851938933134,
      "fcm_dpo/delta": -0.009991994127631187,
      "fcm_dpo/margin": 53.46131134033203,
      "fcm_dpo/q_t": 0.374041885137558,
      "grad_norm": 23.255294799804688,
      "learning_rate": 4.720482655449212e-07,
      "logits/chosen": -0.5350059270858765,
      "logits/rejected": -0.510658323764801,
      "logps/chosen": -106.6119384765625,
      "logps/ref_chosen": -55.021629333496094,
      "logps/ref_rejected": -75.418212890625,
      "logps/rejected": -180.4698486328125,
      "loss": 1.0059,
      "margin_dpo/margin_mean": 53.46131134033203,
      "margin_dpo/margin_std": 71.67129516601562,
      "step": 163
    },
    {
      "epoch": 0.24082232011747431,
      "fcm_dpo/beta": 0.011088410392403603,
      "fcm_dpo/delta": -0.1139398068189621,
      "fcm_dpo/margin": 63.41717529296875,
      "fcm_dpo/q_t": 0.3512231409549713,
      "grad_norm": 25.359941482543945,
      "learning_rate": 4.714556901942599e-07,
      "logits/chosen": -0.5142285823822021,
      "logits/rejected": -0.4906259775161743,
      "logps/chosen": -104.08917236328125,
      "logps/ref_chosen": -55.64066696166992,
      "logps/ref_rejected": -79.66463470458984,
      "logps/rejected": -191.5303192138672,
      "loss": 0.9321,
      "margin_dpo/margin_mean": 63.41717529296875,
      "margin_dpo/margin_std": 71.19700622558594,
      "step": 164
    },
    {
      "epoch": 0.2422907488986784,
      "fcm_dpo/beta": 0.011331463232636452,
      "fcm_dpo/delta": 0.1354885995388031,
      "fcm_dpo/margin": 41.572532653808594,
      "fcm_dpo/q_t": 0.39925825595855713,
      "grad_norm": 34.06428527832031,
      "learning_rate": 4.708572792802069e-07,
      "logits/chosen": -0.5556213855743408,
      "logits/rejected": -0.5192099213600159,
      "logps/chosen": -114.64573669433594,
      "logps/ref_chosen": -61.310691833496094,
      "logps/ref_rejected": -73.67060852050781,
      "logps/rejected": -168.57818603515625,
      "loss": 1.094,
      "margin_dpo/margin_mean": 41.572532653808594,
      "margin_dpo/margin_std": 65.32736206054688,
      "step": 165
    },
    {
      "epoch": 0.24375917767988253,
      "fcm_dpo/beta": 0.010963313281536102,
      "fcm_dpo/delta": -0.2387712448835373,
      "fcm_dpo/margin": 74.51507568359375,
      "fcm_dpo/q_t": 0.3456820547580719,
      "grad_norm": 21.995134353637695,
      "learning_rate": 4.702530485714461e-07,
      "logits/chosen": -0.49838924407958984,
      "logits/rejected": -0.5080593824386597,
      "logps/chosen": -99.90086364746094,
      "logps/ref_chosen": -50.98360061645508,
      "logps/ref_rejected": -98.09512329101562,
      "logps/rejected": -221.5274658203125,
      "loss": 0.9183,
      "margin_dpo/margin_mean": 74.51507568359375,
      "margin_dpo/margin_std": 95.35562896728516,
      "step": 166
    },
    {
      "epoch": 0.24522760646108663,
      "fcm_dpo/beta": 0.010487915948033333,
      "fcm_dpo/delta": -0.2514822483062744,
      "fcm_dpo/margin": 79.16014862060547,
      "fcm_dpo/q_t": 0.3255418539047241,
      "grad_norm": 22.585697174072266,
      "learning_rate": 4.6964301399001877e-07,
      "logits/chosen": -0.4470037817955017,
      "logits/rejected": -0.4476546347141266,
      "logps/chosen": -99.91099548339844,
      "logps/ref_chosen": -50.424095153808594,
      "logps/ref_rejected": -96.03042602539062,
      "logps/rejected": -224.677490234375,
      "loss": 0.8589,
      "margin_dpo/margin_mean": 79.16014862060547,
      "margin_dpo/margin_std": 78.89952087402344,
      "step": 167
    },
    {
      "epoch": 0.24669603524229075,
      "fcm_dpo/beta": 0.010302037000656128,
      "fcm_dpo/delta": -0.02494889497756958,
      "fcm_dpo/margin": 60.4947395324707,
      "fcm_dpo/q_t": 0.36779898405075073,
      "grad_norm": 23.62080192565918,
      "learning_rate": 4.690271916109034e-07,
      "logits/chosen": -0.4938944876194,
      "logits/rejected": -0.4785892963409424,
      "logps/chosen": -104.25721740722656,
      "logps/ref_chosen": -49.462825775146484,
      "logps/ref_rejected": -75.30855560302734,
      "logps/rejected": -190.59768676757812,
      "loss": 0.9727,
      "margin_dpo/margin_mean": 60.49474334716797,
      "margin_dpo/margin_std": 72.37049865722656,
      "step": 168
    },
    {
      "epoch": 0.24816446402349487,
      "fcm_dpo/beta": 0.010289417579770088,
      "fcm_dpo/delta": 0.049492061138153076,
      "fcm_dpo/margin": 53.629798889160156,
      "fcm_dpo/q_t": 0.3923034071922302,
      "grad_norm": 26.301158905029297,
      "learning_rate": 4.6840559766159235e-07,
      "logits/chosen": -0.4954657554626465,
      "logits/rejected": -0.4744875431060791,
      "logps/chosen": -114.426513671875,
      "logps/ref_chosen": -59.803443908691406,
      "logps/ref_rejected": -83.34574890136719,
      "logps/rejected": -191.59860229492188,
      "loss": 1.0873,
      "margin_dpo/margin_mean": 53.629798889160156,
      "margin_dpo/margin_std": 91.4170913696289,
      "step": 169
    },
    {
      "epoch": 0.24963289280469897,
      "fcm_dpo/beta": 0.010223541408777237,
      "fcm_dpo/delta": -0.07429465651512146,
      "fcm_dpo/margin": 65.30518341064453,
      "fcm_dpo/q_t": 0.35696014761924744,
      "grad_norm": 24.374975204467773,
      "learning_rate": 4.6777824852166437e-07,
      "logits/chosen": -0.4555085003376007,
      "logits/rejected": -0.4390765428543091,
      "logps/chosen": -99.52091979980469,
      "logps/ref_chosen": -49.471771240234375,
      "logps/ref_rejected": -75.91734313964844,
      "logps/rejected": -191.27166748046875,
      "loss": 0.958,
      "margin_dpo/margin_mean": 65.30518341064453,
      "margin_dpo/margin_std": 75.40479278564453,
      "step": 170
    },
    {
      "epoch": 0.2511013215859031,
      "fcm_dpo/beta": 0.010339688509702682,
      "fcm_dpo/delta": 0.05246017128229141,
      "fcm_dpo/margin": 53.23592758178711,
      "fcm_dpo/q_t": 0.39151298999786377,
      "grad_norm": 31.557279586791992,
      "learning_rate": 4.6714516072235273e-07,
      "logits/chosen": -0.5044229030609131,
      "logits/rejected": -0.47634661197662354,
      "logps/chosen": -157.2343292236328,
      "logps/ref_chosen": -84.49931335449219,
      "logps/ref_rejected": -109.38209533691406,
      "logps/rejected": -235.35305786132812,
      "loss": 1.0862,
      "margin_dpo/margin_mean": 53.23592758178711,
      "margin_dpo/margin_std": 91.4041748046875,
      "step": 171
    },
    {
      "epoch": 0.2525697503671072,
      "fcm_dpo/beta": 0.010385138913989067,
      "fcm_dpo/delta": 0.024514978751540184,
      "fcm_dpo/margin": 55.545555114746094,
      "fcm_dpo/q_t": 0.3802782893180847,
      "grad_norm": 29.966943740844727,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": -0.5027618408203125,
      "logits/rejected": -0.47274085879325867,
      "logps/chosen": -136.3382568359375,
      "logps/ref_chosen": -68.65391540527344,
      "logps/ref_rejected": -85.43667602539062,
      "logps/rejected": -208.66656494140625,
      "loss": 1.0408,
      "margin_dpo/margin_mean": 55.54555892944336,
      "margin_dpo/margin_std": 81.37893676757812,
      "step": 172
    },
    {
      "epoch": 0.2540381791483113,
      "fcm_dpo/beta": 0.010488756000995636,
      "fcm_dpo/delta": 0.0158163383603096,
      "fcm_dpo/margin": 55.749046325683594,
      "fcm_dpo/q_t": 0.3747754693031311,
      "grad_norm": 29.66188621520996,
      "learning_rate": 4.6586183602616687e-07,
      "logits/chosen": -0.5061242580413818,
      "logits/rejected": -0.4692162275314331,
      "logps/chosen": -123.67233276367188,
      "logps/ref_chosen": -63.050880432128906,
      "logps/ref_rejected": -78.68392181396484,
      "logps/rejected": -195.054443359375,
      "loss": 0.9973,
      "margin_dpo/margin_mean": 55.74905014038086,
      "margin_dpo/margin_std": 69.4229736328125,
      "step": 173
    },
    {
      "epoch": 0.2555066079295154,
      "fcm_dpo/beta": 0.01044470351189375,
      "fcm_dpo/delta": -0.0565309077501297,
      "fcm_dpo/margin": 62.44271469116211,
      "fcm_dpo/q_t": 0.3679395318031311,
      "grad_norm": 31.072416305541992,
      "learning_rate": 4.652116329460919e-07,
      "logits/chosen": -0.42711180448532104,
      "logits/rejected": -0.44361674785614014,
      "logps/chosen": -113.0875015258789,
      "logps/ref_chosen": -53.36296844482422,
      "logps/ref_rejected": -101.91120910644531,
      "logps/rejected": -224.07846069335938,
      "loss": 0.9957,
      "margin_dpo/margin_mean": 62.44270706176758,
      "margin_dpo/margin_std": 83.03677368164062,
      "step": 174
    },
    {
      "epoch": 0.25697503671071953,
      "fcm_dpo/beta": 0.009896190837025642,
      "fcm_dpo/delta": -0.282113254070282,
      "fcm_dpo/margin": 86.63121032714844,
      "fcm_dpo/q_t": 0.32039257884025574,
      "grad_norm": 29.841856002807617,
      "learning_rate": 4.645557588393406e-07,
      "logits/chosen": -0.39522379636764526,
      "logits/rejected": -0.38075965642929077,
      "logps/chosen": -103.10018920898438,
      "logps/ref_chosen": -45.417762756347656,
      "logps/ref_rejected": -89.50579833984375,
      "logps/rejected": -233.8194580078125,
      "loss": 0.835,
      "margin_dpo/margin_mean": 86.63121032714844,
      "margin_dpo/margin_std": 82.89295196533203,
      "step": 175
    },
    {
      "epoch": 0.25844346549192365,
      "fcm_dpo/beta": 0.009663419798016548,
      "fcm_dpo/delta": -0.09544707089662552,
      "fcm_dpo/margin": 71.29586791992188,
      "fcm_dpo/q_t": 0.3592815101146698,
      "grad_norm": 28.798011779785156,
      "learning_rate": 4.638942309888058e-07,
      "logits/chosen": -0.39833515882492065,
      "logits/rejected": -0.41467368602752686,
      "logps/chosen": -110.1692123413086,
      "logps/ref_chosen": -50.452842712402344,
      "logps/ref_rejected": -95.5589599609375,
      "logps/rejected": -226.57119750976562,
      "loss": 0.9511,
      "margin_dpo/margin_mean": 71.29586791992188,
      "margin_dpo/margin_std": 86.2072525024414,
      "step": 176
    },
    {
      "epoch": 0.2599118942731278,
      "fcm_dpo/beta": 0.009497301653027534,
      "fcm_dpo/delta": -0.09194030612707138,
      "fcm_dpo/margin": 72.19395446777344,
      "fcm_dpo/q_t": 0.35893410444259644,
      "grad_norm": 32.332763671875,
      "learning_rate": 4.6322706682636137e-07,
      "logits/chosen": -0.4460568428039551,
      "logits/rejected": -0.4327634572982788,
      "logps/chosen": -128.06985473632812,
      "logps/ref_chosen": -61.216468811035156,
      "logps/ref_rejected": -95.89378356933594,
      "logps/rejected": -234.94113159179688,
      "loss": 0.9502,
      "margin_dpo/margin_mean": 72.19395446777344,
      "margin_dpo/margin_std": 87.10002136230469,
      "step": 177
    },
    {
      "epoch": 0.26138032305433184,
      "fcm_dpo/beta": 0.009019111283123493,
      "fcm_dpo/delta": -0.24799524247646332,
      "fcm_dpo/margin": 91.52853393554688,
      "fcm_dpo/q_t": 0.3365201950073242,
      "grad_norm": 31.862321853637695,
      "learning_rate": 4.6255428393240354e-07,
      "logits/chosen": -0.3157181143760681,
      "logits/rejected": -0.31319284439086914,
      "logps/chosen": -133.81484985351562,
      "logps/ref_chosen": -58.26478958129883,
      "logps/ref_rejected": -105.3653335571289,
      "logps/rejected": -272.44390869140625,
      "loss": 0.8982,
      "margin_dpo/margin_mean": 91.52853393554688,
      "margin_dpo/margin_std": 108.31314086914062,
      "step": 178
    },
    {
      "epoch": 0.26284875183553597,
      "fcm_dpo/beta": 0.008883455768227577,
      "fcm_dpo/delta": -0.01396089792251587,
      "fcm_dpo/margin": 68.92755889892578,
      "fcm_dpo/q_t": 0.37395423650741577,
      "grad_norm": 51.664894104003906,
      "learning_rate": 4.6187590003538724e-07,
      "logits/chosen": -0.3555856943130493,
      "logits/rejected": -0.35472238063812256,
      "logps/chosen": -136.07827758789062,
      "logps/ref_chosen": -61.05832290649414,
      "logps/ref_rejected": -90.52782440185547,
      "logps/rejected": -234.475341796875,
      "loss": 1.0389,
      "margin_dpo/margin_mean": 68.92755889892578,
      "margin_dpo/margin_std": 100.45736694335938,
      "step": 179
    },
    {
      "epoch": 0.2643171806167401,
      "fcm_dpo/beta": 0.008733205497264862,
      "fcm_dpo/delta": -0.13286282122135162,
      "fcm_dpo/margin": 82.76274108886719,
      "fcm_dpo/q_t": 0.34625113010406494,
      "grad_norm": 34.209205627441406,
      "learning_rate": 4.611919330113591e-07,
      "logits/chosen": -0.39520972967147827,
      "logits/rejected": -0.3850780725479126,
      "logps/chosen": -116.90325927734375,
      "logps/ref_chosen": -54.34272003173828,
      "logps/ref_rejected": -98.21183776855469,
      "logps/rejected": -243.5351104736328,
      "loss": 0.9237,
      "margin_dpo/margin_mean": 82.76274108886719,
      "margin_dpo/margin_std": 91.62629699707031,
      "step": 180
    },
    {
      "epoch": 0.2657856093979442,
      "fcm_dpo/beta": 0.008904640562832355,
      "fcm_dpo/delta": 0.14196932315826416,
      "fcm_dpo/margin": 52.12580871582031,
      "fcm_dpo/q_t": 0.4002281725406647,
      "grad_norm": 29.242713928222656,
      "learning_rate": 4.605024008834863e-07,
      "logits/chosen": -0.4259217381477356,
      "logits/rejected": -0.3925449252128601,
      "logps/chosen": -108.0588607788086,
      "logps/ref_chosen": -55.000457763671875,
      "logps/ref_rejected": -61.656166076660156,
      "logps/rejected": -166.8403778076172,
      "loss": 1.0943,
      "margin_dpo/margin_mean": 52.12580871582031,
      "margin_dpo/margin_std": 82.18958282470703,
      "step": 181
    },
    {
      "epoch": 0.26725403817914833,
      "fcm_dpo/beta": 0.008686801418662071,
      "fcm_dpo/delta": -0.16562983393669128,
      "fcm_dpo/margin": 86.55928802490234,
      "fcm_dpo/q_t": 0.3413204550743103,
      "grad_norm": 27.281949996948242,
      "learning_rate": 4.598073218215817e-07,
      "logits/chosen": -0.3973362445831299,
      "logits/rejected": -0.4009121060371399,
      "logps/chosen": -91.49468231201172,
      "logps/ref_chosen": -41.107852935791016,
      "logps/ref_rejected": -89.5215835571289,
      "logps/rejected": -226.4676971435547,
      "loss": 0.9226,
      "margin_dpo/margin_mean": 86.55928802490234,
      "margin_dpo/margin_std": 97.51657104492188,
      "step": 182
    },
    {
      "epoch": 0.2687224669603524,
      "fcm_dpo/beta": 0.008756536990404129,
      "fcm_dpo/delta": 0.15965726971626282,
      "fcm_dpo/margin": 50.974124908447266,
      "fcm_dpo/q_t": 0.40270137786865234,
      "grad_norm": 26.794544219970703,
      "learning_rate": 4.5910671414162484e-07,
      "logits/chosen": -0.4323645234107971,
      "logits/rejected": -0.41012632846832275,
      "logps/chosen": -132.4599151611328,
      "logps/ref_chosen": -57.52456283569336,
      "logps/ref_rejected": -75.97572326660156,
      "logps/rejected": -201.8852081298828,
      "loss": 1.0928,
      "margin_dpo/margin_mean": 50.97412872314453,
      "margin_dpo/margin_std": 70.7024917602539,
      "step": 183
    },
    {
      "epoch": 0.2701908957415565,
      "fcm_dpo/beta": 0.009184681810438633,
      "fcm_dpo/delta": 0.13521449267864227,
      "fcm_dpo/margin": 51.08043670654297,
      "fcm_dpo/q_t": 0.40079188346862793,
      "grad_norm": 24.206710815429688,
      "learning_rate": 4.5840059630527985e-07,
      "logits/chosen": -0.4838387966156006,
      "logits/rejected": -0.4702536165714264,
      "logps/chosen": -119.1988296508789,
      "logps/ref_chosen": -58.544952392578125,
      "logps/ref_rejected": -76.63406372070312,
      "logps/rejected": -188.36837768554688,
      "loss": 1.083,
      "margin_dpo/margin_mean": 51.080440521240234,
      "margin_dpo/margin_std": 76.5762939453125,
      "step": 184
    },
    {
      "epoch": 0.27165932452276065,
      "fcm_dpo/beta": 0.00946805626153946,
      "fcm_dpo/delta": 0.21153177320957184,
      "fcm_dpo/margin": 41.93297576904297,
      "fcm_dpo/q_t": 0.41936683654785156,
      "grad_norm": 26.689727783203125,
      "learning_rate": 4.5768898691940836e-07,
      "logits/chosen": -0.4485119581222534,
      "logits/rejected": -0.4117010831832886,
      "logps/chosen": -128.1175079345703,
      "logps/ref_chosen": -62.025848388671875,
      "logps/ref_rejected": -73.7625961303711,
      "logps/rejected": -181.7872314453125,
      "loss": 1.1579,
      "margin_dpo/margin_mean": 41.93297576904297,
      "margin_dpo/margin_std": 82.34846496582031,
      "step": 185
    },
    {
      "epoch": 0.27312775330396477,
      "fcm_dpo/beta": 0.009380245581269264,
      "fcm_dpo/delta": -0.12999294698238373,
      "fcm_dpo/margin": 76.79740905761719,
      "fcm_dpo/q_t": 0.3472128212451935,
      "grad_norm": 49.98681640625,
      "learning_rate": 4.5697190473557947e-07,
      "logits/chosen": -0.4748345613479614,
      "logits/rejected": -0.43923431634902954,
      "logps/chosen": -130.9991455078125,
      "logps/ref_chosen": -69.35346984863281,
      "logps/ref_rejected": -88.07244873046875,
      "logps/rejected": -226.51553344726562,
      "loss": 0.9126,
      "margin_dpo/margin_mean": 76.79740905761719,
      "margin_dpo/margin_std": 82.54960632324219,
      "step": 186
    },
    {
      "epoch": 0.2745961820851689,
      "fcm_dpo/beta": 0.00942191667854786,
      "fcm_dpo/delta": 0.014476820826530457,
      "fcm_dpo/margin": 62.11200714111328,
      "fcm_dpo/q_t": 0.3749355673789978,
      "grad_norm": 33.28017807006836,
      "learning_rate": 4.5624936864957555e-07,
      "logits/chosen": -0.4199049770832062,
      "logits/rejected": -0.41446995735168457,
      "logps/chosen": -114.2715072631836,
      "logps/ref_chosen": -52.7564582824707,
      "logps/ref_rejected": -81.96910095214844,
      "logps/rejected": -205.59616088867188,
      "loss": 1.0036,
      "margin_dpo/margin_mean": 62.11200714111328,
      "margin_dpo/margin_std": 77.76611328125,
      "step": 187
    },
    {
      "epoch": 0.27606461086637296,
      "fcm_dpo/beta": 0.009187942370772362,
      "fcm_dpo/delta": -0.12695354223251343,
      "fcm_dpo/margin": 78.13199615478516,
      "fcm_dpo/q_t": 0.35078778862953186,
      "grad_norm": 41.620758056640625,
      "learning_rate": 4.5552139770089454e-07,
      "logits/chosen": -0.4144795536994934,
      "logits/rejected": -0.4158819019794464,
      "logps/chosen": -107.33932495117188,
      "logps/ref_chosen": -49.415489196777344,
      "logps/ref_rejected": -89.54043579101562,
      "logps/rejected": -225.59625244140625,
      "loss": 0.9298,
      "margin_dpo/margin_mean": 78.13198852539062,
      "margin_dpo/margin_std": 89.57806396484375,
      "step": 188
    },
    {
      "epoch": 0.2775330396475771,
      "fcm_dpo/beta": 0.00912393070757389,
      "fcm_dpo/delta": -0.008689474314451218,
      "fcm_dpo/margin": 66.65364074707031,
      "fcm_dpo/q_t": 0.37750983238220215,
      "grad_norm": 41.65003967285156,
      "learning_rate": 4.5478801107224794e-07,
      "logits/chosen": -0.4112040400505066,
      "logits/rejected": -0.38720983266830444,
      "logps/chosen": -123.0237045288086,
      "logps/ref_chosen": -52.39896011352539,
      "logps/ref_rejected": -72.16735076904297,
      "logps/rejected": -209.44573974609375,
      "loss": 1.028,
      "margin_dpo/margin_mean": 66.65364074707031,
      "margin_dpo/margin_std": 99.14986419677734,
      "step": 189
    },
    {
      "epoch": 0.2790014684287812,
      "fcm_dpo/beta": 0.009091001003980637,
      "fcm_dpo/delta": -0.09938493371009827,
      "fcm_dpo/margin": 75.96780395507812,
      "fcm_dpo/q_t": 0.361076295375824,
      "grad_norm": 30.051088333129883,
      "learning_rate": 4.5404922808905543e-07,
      "logits/chosen": -0.43357422947883606,
      "logits/rejected": -0.4192691147327423,
      "logps/chosen": -141.0237579345703,
      "logps/ref_chosen": -64.68305969238281,
      "logps/ref_rejected": -102.55052185058594,
      "logps/rejected": -254.85903930664062,
      "loss": 0.9991,
      "margin_dpo/margin_mean": 75.96780395507812,
      "margin_dpo/margin_std": 101.82373809814453,
      "step": 190
    },
    {
      "epoch": 0.28046989720998533,
      "fcm_dpo/beta": 0.008413515985012054,
      "fcm_dpo/delta": -0.3635314106941223,
      "fcm_dpo/margin": 110.20065307617188,
      "fcm_dpo/q_t": 0.3166946768760681,
      "grad_norm": 25.67074203491211,
      "learning_rate": 4.5330506821893565e-07,
      "logits/chosen": -0.35954105854034424,
      "logits/rejected": -0.33676308393478394,
      "logps/chosen": -138.35675048828125,
      "logps/ref_chosen": -68.65887451171875,
      "logps/ref_rejected": -110.1396713256836,
      "logps/rejected": -290.0382080078125,
      "loss": 0.8336,
      "margin_dpo/margin_mean": 110.20065307617188,
      "margin_dpo/margin_std": 117.54473876953125,
      "step": 191
    },
    {
      "epoch": 0.28193832599118945,
      "fcm_dpo/beta": 0.008292671293020248,
      "fcm_dpo/delta": -0.028325140476226807,
      "fcm_dpo/margin": 75.54448699951172,
      "fcm_dpo/q_t": 0.37382009625434875,
      "grad_norm": 34.30679702758789,
      "learning_rate": 4.5255555107119336e-07,
      "logits/chosen": -0.37166693806648254,
      "logits/rejected": -0.37043970823287964,
      "logps/chosen": -160.6333770751953,
      "logps/ref_chosen": -69.72691345214844,
      "logps/ref_rejected": -103.32135009765625,
      "logps/rejected": -269.77227783203125,
      "loss": 1.0202,
      "margin_dpo/margin_mean": 75.54447937011719,
      "margin_dpo/margin_std": 108.79905700683594,
      "step": 192
    },
    {
      "epoch": 0.2834067547723935,
      "fcm_dpo/beta": 0.008594648912549019,
      "fcm_dpo/delta": 0.2289629727602005,
      "fcm_dpo/margin": 43.941375732421875,
      "fcm_dpo/q_t": 0.4161950647830963,
      "grad_norm": 38.5386848449707,
      "learning_rate": 4.5180069639630236e-07,
      "logits/chosen": -0.37603700160980225,
      "logits/rejected": -0.35951176285743713,
      "logps/chosen": -151.70782470703125,
      "logps/ref_chosen": -60.19049835205078,
      "logps/ref_rejected": -76.40755462646484,
      "logps/rejected": -211.86627197265625,
      "loss": 1.2035,
      "margin_dpo/margin_mean": 43.941375732421875,
      "margin_dpo/margin_std": 97.33943176269531,
      "step": 193
    },
    {
      "epoch": 0.28487518355359764,
      "fcm_dpo/beta": 0.008597111329436302,
      "fcm_dpo/delta": -0.016570590436458588,
      "fcm_dpo/margin": 71.57522583007812,
      "fcm_dpo/q_t": 0.36463361978530884,
      "grad_norm": 24.157445907592773,
      "learning_rate": 4.510405240853854e-07,
      "logits/chosen": -0.28482919931411743,
      "logits/rejected": -0.2655167579650879,
      "logps/chosen": -93.96827697753906,
      "logps/ref_chosen": -37.84037399291992,
      "logps/ref_rejected": -60.684783935546875,
      "logps/rejected": -188.38790893554688,
      "loss": 0.9556,
      "margin_dpo/margin_mean": 71.57522583007812,
      "margin_dpo/margin_std": 75.2847671508789,
      "step": 194
    },
    {
      "epoch": 0.28634361233480177,
      "fcm_dpo/beta": 0.008543387055397034,
      "fcm_dpo/delta": -0.04339263588190079,
      "fcm_dpo/margin": 74.96640014648438,
      "fcm_dpo/q_t": 0.361470490694046,
      "grad_norm": 31.8635311126709,
      "learning_rate": 4.5027505416968985e-07,
      "logits/chosen": -0.3245890140533447,
      "logits/rejected": -0.3413041830062866,
      "logps/chosen": -144.32359313964844,
      "logps/ref_chosen": -54.891571044921875,
      "logps/ref_rejected": -96.77095794677734,
      "logps/rejected": -261.1694030761719,
      "loss": 0.9508,
      "margin_dpo/margin_mean": 74.96640014648438,
      "margin_dpo/margin_std": 82.75733947753906,
      "step": 195
    },
    {
      "epoch": 0.2878120411160059,
      "fcm_dpo/beta": 0.008355004712939262,
      "fcm_dpo/delta": -0.11748833954334259,
      "fcm_dpo/margin": 84.77229309082031,
      "fcm_dpo/q_t": 0.352541983127594,
      "grad_norm": 28.523649215698242,
      "learning_rate": 4.495043068200599e-07,
      "logits/chosen": -0.3666142225265503,
      "logits/rejected": -0.34821170568466187,
      "logps/chosen": -121.3379898071289,
      "logps/ref_chosen": -53.245243072509766,
      "logps/ref_rejected": -76.05294799804688,
      "logps/rejected": -228.91798400878906,
      "loss": 0.9446,
      "margin_dpo/margin_mean": 84.77229309082031,
      "margin_dpo/margin_std": 98.6243896484375,
      "step": 196
    },
    {
      "epoch": 0.28928046989721,
      "fcm_dpo/beta": 0.00841631181538105,
      "fcm_dpo/delta": 0.04314948618412018,
      "fcm_dpo/margin": 66.4241714477539,
      "fcm_dpo/q_t": 0.3789867162704468,
      "grad_norm": 36.85988235473633,
      "learning_rate": 4.4872830234640493e-07,
      "logits/chosen": -0.34630054235458374,
      "logits/rejected": -0.33658844232559204,
      "logps/chosen": -127.1322250366211,
      "logps/ref_chosen": -60.42033386230469,
      "logps/ref_rejected": -77.20890808105469,
      "logps/rejected": -210.344970703125,
      "loss": 1.0009,
      "margin_dpo/margin_mean": 66.4241714477539,
      "margin_dpo/margin_std": 79.97479248046875,
      "step": 197
    },
    {
      "epoch": 0.2907488986784141,
      "fcm_dpo/beta": 0.008350390940904617,
      "fcm_dpo/delta": -0.068976491689682,
      "fcm_dpo/margin": 79.5688705444336,
      "fcm_dpo/q_t": 0.3639752268791199,
      "grad_norm": 31.69214630126953,
      "learning_rate": 4.479470611971645e-07,
      "logits/chosen": -0.36272114515304565,
      "logits/rejected": -0.36416107416152954,
      "logps/chosen": -129.2322998046875,
      "logps/ref_chosen": -55.03618621826172,
      "logps/ref_rejected": -97.24325561523438,
      "logps/rejected": -251.00823974609375,
      "loss": 0.9641,
      "margin_dpo/margin_mean": 79.5688705444336,
      "margin_dpo/margin_std": 100.84001922607422,
      "step": 198
    },
    {
      "epoch": 0.2922173274596182,
      "fcm_dpo/beta": 0.008206663653254509,
      "fcm_dpo/delta": -0.027660571038722992,
      "fcm_dpo/margin": 76.12931060791016,
      "fcm_dpo/q_t": 0.3673062324523926,
      "grad_norm": 35.12301254272461,
      "learning_rate": 4.471606039587695e-07,
      "logits/chosen": -0.3977452516555786,
      "logits/rejected": -0.38206660747528076,
      "logps/chosen": -128.30673217773438,
      "logps/ref_chosen": -56.828826904296875,
      "logps/ref_rejected": -84.64820861816406,
      "logps/rejected": -232.25543212890625,
      "loss": 0.9921,
      "margin_dpo/margin_mean": 76.12931060791016,
      "margin_dpo/margin_std": 95.28636169433594,
      "step": 199
    },
    {
      "epoch": 0.2936857562408223,
      "fcm_dpo/beta": 0.008154574781656265,
      "fcm_dpo/delta": -0.05656225234270096,
      "fcm_dpo/margin": 80.00682830810547,
      "fcm_dpo/q_t": 0.3651021122932434,
      "grad_norm": 30.504737854003906,
      "learning_rate": 4.4636895135509966e-07,
      "logits/chosen": -0.3527507185935974,
      "logits/rejected": -0.3269976079463959,
      "logps/chosen": -120.8394775390625,
      "logps/ref_chosen": -53.06706237792969,
      "logps/ref_rejected": -80.60843658447266,
      "logps/rejected": -228.38766479492188,
      "loss": 0.9914,
      "margin_dpo/margin_mean": 80.00682830810547,
      "margin_dpo/margin_std": 106.2452621459961,
      "step": 200
    },
    {
      "epoch": 0.2936857562408223,
      "eval_fcm_dpo/beta": 0.00816048588603735,
      "eval_logits/chosen": -0.4719357490539551,
      "eval_logits/rejected": -0.45578357577323914,
      "eval_logps/chosen": -172.25338745117188,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -225.0161590576172,
      "eval_loss": 0.6008790135383606,
      "eval_margin_dpo/margin_mean": 45.01588439941406,
      "eval_margin_dpo/margin_std": 98.482421875,
      "eval_runtime": 39.3122,
      "eval_samples_per_second": 59.498,
      "eval_steps_per_second": 1.882,
      "step": 200
    },
    {
      "epoch": 0.29515418502202645,
      "fcm_dpo/beta": 0.008100075647234917,
      "fcm_dpo/delta": -0.0565091148018837,
      "fcm_dpo/margin": 80.59832000732422,
      "fcm_dpo/q_t": 0.3638731837272644,
      "grad_norm": 30.036048889160156,
      "learning_rate": 4.455721242469372e-07,
      "logits/chosen": -0.4626866281032562,
      "logits/rejected": -0.4563639163970947,
      "logps/chosen": -146.43814086914062,
      "logps/ref_chosen": -75.4022216796875,
      "logps/ref_rejected": -114.80821990966797,
      "logps/rejected": -266.44244384765625,
      "loss": 0.9815,
      "margin_dpo/margin_mean": 80.59831237792969,
      "margin_dpo/margin_std": 103.2762222290039,
      "step": 201
    },
    {
      "epoch": 0.2966226138032305,
      "fcm_dpo/beta": 0.00825195387005806,
      "fcm_dpo/delta": 0.13680000603199005,
      "fcm_dpo/margin": 56.866981506347656,
      "fcm_dpo/q_t": 0.40256237983703613,
      "grad_norm": 26.29439353942871,
      "learning_rate": 4.4477014363141755e-07,
      "logits/chosen": -0.3789522051811218,
      "logits/rejected": -0.3867080807685852,
      "logps/chosen": -124.50201416015625,
      "logps/ref_chosen": -50.101318359375,
      "logps/ref_rejected": -86.98503112792969,
      "logps/rejected": -218.25270080566406,
      "loss": 1.1171,
      "margin_dpo/margin_mean": 56.866981506347656,
      "margin_dpo/margin_std": 99.0474853515625,
      "step": 202
    },
    {
      "epoch": 0.29809104258443464,
      "fcm_dpo/beta": 0.008316997438669205,
      "fcm_dpo/delta": 0.0254436694085598,
      "fcm_dpo/margin": 69.2591781616211,
      "fcm_dpo/q_t": 0.3762577772140503,
      "grad_norm": 31.99873924255371,
      "learning_rate": 4.439630306414758e-07,
      "logits/chosen": -0.4376029968261719,
      "logits/rejected": -0.4240657389163971,
      "logps/chosen": -134.08056640625,
      "logps/ref_chosen": -60.60969543457031,
      "logps/ref_rejected": -85.89596557617188,
      "logps/rejected": -228.62603759765625,
      "loss": 0.9972,
      "margin_dpo/margin_mean": 69.2591781616211,
      "margin_dpo/margin_std": 84.98655700683594,
      "step": 203
    },
    {
      "epoch": 0.29955947136563876,
      "fcm_dpo/beta": 0.00838047917932272,
      "fcm_dpo/delta": 0.04098369926214218,
      "fcm_dpo/margin": 66.98109436035156,
      "fcm_dpo/q_t": 0.3864881992340088,
      "grad_norm": 29.694320678710938,
      "learning_rate": 4.431508065452897e-07,
      "logits/chosen": -0.47066575288772583,
      "logits/rejected": -0.4281098246574402,
      "logps/chosen": -162.51052856445312,
      "logps/ref_chosen": -80.16496276855469,
      "logps/ref_rejected": -87.69590759277344,
      "logps/rejected": -237.0225830078125,
      "loss": 1.0491,
      "margin_dpo/margin_mean": 66.98109436035156,
      "margin_dpo/margin_std": 100.66291046142578,
      "step": 204
    },
    {
      "epoch": 0.3010279001468429,
      "fcm_dpo/beta": 0.008169597014784813,
      "fcm_dpo/delta": -0.11707717180252075,
      "fcm_dpo/margin": 86.33795166015625,
      "fcm_dpo/q_t": 0.350330114364624,
      "grad_norm": 24.040218353271484,
      "learning_rate": 4.4233349274571974e-07,
      "logits/chosen": -0.38784509897232056,
      "logits/rejected": -0.35509926080703735,
      "logps/chosen": -142.8284454345703,
      "logps/ref_chosen": -59.384735107421875,
      "logps/ref_rejected": -85.12505340576172,
      "logps/rejected": -254.90672302246094,
      "loss": 0.9583,
      "margin_dpo/margin_mean": 86.33795166015625,
      "margin_dpo/margin_std": 101.34950256347656,
      "step": 205
    },
    {
      "epoch": 0.302496328928047,
      "fcm_dpo/beta": 0.007971297949552536,
      "fcm_dpo/delta": -0.16260316967964172,
      "fcm_dpo/margin": 93.97215270996094,
      "fcm_dpo/q_t": 0.33909979462623596,
      "grad_norm": 34.61493682861328,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": -0.3233247995376587,
      "logits/rejected": -0.32719945907592773,
      "logps/chosen": -125.01007843017578,
      "logps/ref_chosen": -46.964500427246094,
      "logps/ref_rejected": -98.9534912109375,
      "logps/rejected": -270.9712219238281,
      "loss": 0.9038,
      "margin_dpo/margin_mean": 93.97215270996094,
      "margin_dpo/margin_std": 98.51936340332031,
      "step": 206
    },
    {
      "epoch": 0.3039647577092511,
      "fcm_dpo/beta": 0.00767237413674593,
      "fcm_dpo/delta": -0.24557989835739136,
      "fcm_dpo/margin": 107.59861755371094,
      "fcm_dpo/q_t": 0.33342668414115906,
      "grad_norm": 28.77750587463379,
      "learning_rate": 4.4068368231789365e-07,
      "logits/chosen": -0.40268674492836,
      "logits/rejected": -0.37191277742385864,
      "logps/chosen": -121.83184814453125,
      "logps/ref_chosen": -56.05625915527344,
      "logps/ref_rejected": -84.44779968261719,
      "logps/rejected": -257.822021484375,
      "loss": 0.8758,
      "margin_dpo/margin_mean": 107.59861755371094,
      "margin_dpo/margin_std": 117.77052307128906,
      "step": 207
    },
    {
      "epoch": 0.3054331864904552,
      "fcm_dpo/beta": 0.007520717103034258,
      "fcm_dpo/delta": -0.02344403602182865,
      "fcm_dpo/margin": 82.68033599853516,
      "fcm_dpo/q_t": 0.36858728528022766,
      "grad_norm": 29.981359481811523,
      "learning_rate": 4.398512291636768e-07,
      "logits/chosen": -0.42680084705352783,
      "logits/rejected": -0.4093056321144104,
      "logps/chosen": -167.66326904296875,
      "logps/ref_chosen": -67.06761169433594,
      "logps/ref_rejected": -94.28689575195312,
      "logps/rejected": -277.5628662109375,
      "loss": 1.007,
      "margin_dpo/margin_mean": 82.68034362792969,
      "margin_dpo/margin_std": 110.93801879882812,
      "step": 208
    },
    {
      "epoch": 0.3069016152716593,
      "fcm_dpo/beta": 0.0075776162557303905,
      "fcm_dpo/delta": 0.04492693394422531,
      "fcm_dpo/margin": 73.58404541015625,
      "fcm_dpo/q_t": 0.3805572986602783,
      "grad_norm": 32.00458908081055,
      "learning_rate": 4.3901377325300857e-07,
      "logits/chosen": -0.33511584997177124,
      "logits/rejected": -0.3224318027496338,
      "logps/chosen": -138.41427612304688,
      "logps/ref_chosen": -56.18169403076172,
      "logps/ref_rejected": -80.94152069091797,
      "logps/rejected": -236.75814819335938,
      "loss": 1.0361,
      "margin_dpo/margin_mean": 73.58403778076172,
      "margin_dpo/margin_std": 99.81734466552734,
      "step": 209
    },
    {
      "epoch": 0.30837004405286345,
      "fcm_dpo/beta": 0.007613741792738438,
      "fcm_dpo/delta": -0.010615115985274315,
      "fcm_dpo/margin": 80.08528137207031,
      "fcm_dpo/q_t": 0.3702564239501953,
      "grad_norm": 22.40015411376953,
      "learning_rate": 4.381713366536311e-07,
      "logits/chosen": -0.3724694550037384,
      "logits/rejected": -0.36007583141326904,
      "logps/chosen": -120.55526733398438,
      "logps/ref_chosen": -46.371822357177734,
      "logps/ref_rejected": -76.68162536621094,
      "logps/rejected": -230.95034790039062,
      "loss": 0.984,
      "margin_dpo/margin_mean": 80.08528137207031,
      "margin_dpo/margin_std": 96.79129028320312,
      "step": 210
    },
    {
      "epoch": 0.30983847283406757,
      "fcm_dpo/beta": 0.0076542324386537075,
      "fcm_dpo/delta": 0.0667615756392479,
      "fcm_dpo/margin": 70.13720703125,
      "fcm_dpo/q_t": 0.38842254877090454,
      "grad_norm": 34.49944305419922,
      "learning_rate": 4.373239415645323e-07,
      "logits/chosen": -0.4161061644554138,
      "logits/rejected": -0.37069594860076904,
      "logps/chosen": -185.52003479003906,
      "logps/ref_chosen": -78.93235778808594,
      "logps/ref_rejected": -86.82098388671875,
      "logps/rejected": -263.5458984375,
      "loss": 1.0509,
      "margin_dpo/margin_mean": 70.13721466064453,
      "margin_dpo/margin_std": 100.4876937866211,
      "step": 211
    },
    {
      "epoch": 0.31130690161527164,
      "fcm_dpo/beta": 0.007427205331623554,
      "fcm_dpo/delta": -0.15895992517471313,
      "fcm_dpo/margin": 100.10624694824219,
      "fcm_dpo/q_t": 0.3454575836658478,
      "grad_norm": 28.596296310424805,
      "learning_rate": 4.3647161031536086e-07,
      "logits/chosen": -0.3710683584213257,
      "logits/rejected": -0.35650646686553955,
      "logps/chosen": -147.09378051757812,
      "logps/ref_chosen": -58.19701385498047,
      "logps/ref_rejected": -103.05785369873047,
      "logps/rejected": -292.06085205078125,
      "loss": 0.9345,
      "margin_dpo/margin_mean": 100.10624694824219,
      "margin_dpo/margin_std": 112.30523681640625,
      "step": 212
    },
    {
      "epoch": 0.31277533039647576,
      "fcm_dpo/beta": 0.007264791056513786,
      "fcm_dpo/delta": -0.1239488273859024,
      "fcm_dpo/margin": 98.2468490600586,
      "fcm_dpo/q_t": 0.34874916076660156,
      "grad_norm": 25.72871208190918,
      "learning_rate": 4.3561436536583774e-07,
      "logits/chosen": -0.41955453157424927,
      "logits/rejected": -0.3891031742095947,
      "logps/chosen": -149.52923583984375,
      "logps/ref_chosen": -67.51271057128906,
      "logps/ref_rejected": -93.91471862792969,
      "logps/rejected": -274.1781005859375,
      "loss": 0.9295,
      "margin_dpo/margin_mean": 98.24685668945312,
      "margin_dpo/margin_std": 108.88845825195312,
      "step": 213
    },
    {
      "epoch": 0.3142437591776799,
      "fcm_dpo/beta": 0.007254892028868198,
      "fcm_dpo/delta": -0.009785597212612629,
      "fcm_dpo/margin": 83.96929168701172,
      "fcm_dpo/q_t": 0.37200796604156494,
      "grad_norm": 26.340700149536133,
      "learning_rate": 4.3475222930516473e-07,
      "logits/chosen": -0.3503088653087616,
      "logits/rejected": -0.35779234766960144,
      "logps/chosen": -118.85914611816406,
      "logps/ref_chosen": -41.604888916015625,
      "logps/ref_rejected": -77.51741027832031,
      "logps/rejected": -238.740966796875,
      "loss": 0.9962,
      "margin_dpo/margin_mean": 83.96929931640625,
      "margin_dpo/margin_std": 107.99652099609375,
      "step": 214
    },
    {
      "epoch": 0.315712187958884,
      "fcm_dpo/beta": 0.007164125796407461,
      "fcm_dpo/delta": -0.04974304512143135,
      "fcm_dpo/margin": 90.17284393310547,
      "fcm_dpo/q_t": 0.3607136309146881,
      "grad_norm": 27.573640823364258,
      "learning_rate": 4.3388522485142885e-07,
      "logits/chosen": -0.3955865502357483,
      "logits/rejected": -0.3844960331916809,
      "logps/chosen": -139.45066833496094,
      "logps/ref_chosen": -53.279266357421875,
      "logps/ref_rejected": -89.96464538574219,
      "logps/rejected": -266.30889892578125,
      "loss": 0.9501,
      "margin_dpo/margin_mean": 90.1728515625,
      "margin_dpo/margin_std": 99.1728515625,
      "step": 215
    },
    {
      "epoch": 0.31718061674008813,
      "fcm_dpo/beta": 0.007151792757213116,
      "fcm_dpo/delta": -0.04989144578576088,
      "fcm_dpo/margin": 90.40126037597656,
      "fcm_dpo/q_t": 0.36354175209999084,
      "grad_norm": 33.67525863647461,
      "learning_rate": 4.330133748510036e-07,
      "logits/chosen": -0.3702380061149597,
      "logits/rejected": -0.3506859540939331,
      "logps/chosen": -137.24215698242188,
      "logps/ref_chosen": -48.887794494628906,
      "logps/ref_rejected": -77.19892883300781,
      "logps/rejected": -255.95455932617188,
      "loss": 0.9844,
      "margin_dpo/margin_mean": 90.40126037597656,
      "margin_dpo/margin_std": 113.781982421875,
      "step": 216
    },
    {
      "epoch": 0.3186490455212922,
      "fcm_dpo/beta": 0.006948284804821014,
      "fcm_dpo/delta": -0.1394684761762619,
      "fcm_dpo/margin": 104.92652893066406,
      "fcm_dpo/q_t": 0.34340259432792664,
      "grad_norm": 25.96721649169922,
      "learning_rate": 4.3213670227794757e-07,
      "logits/chosen": -0.3827437162399292,
      "logits/rejected": -0.3740556240081787,
      "logps/chosen": -134.2442626953125,
      "logps/ref_chosen": -49.845306396484375,
      "logps/ref_rejected": -100.07832336425781,
      "logps/rejected": -289.40380859375,
      "loss": 0.897,
      "margin_dpo/margin_mean": 104.92652893066406,
      "margin_dpo/margin_std": 105.56159210205078,
      "step": 217
    },
    {
      "epoch": 0.3201174743024963,
      "fcm_dpo/beta": 0.0069186268374323845,
      "fcm_dpo/delta": 0.01962290331721306,
      "fcm_dpo/margin": 84.04899597167969,
      "fcm_dpo/q_t": 0.37689316272735596,
      "grad_norm": 29.522064208984375,
      "learning_rate": 4.3125523023339815e-07,
      "logits/chosen": -0.4043159484863281,
      "logits/rejected": -0.3937210440635681,
      "logps/chosen": -144.95455932617188,
      "logps/ref_chosen": -58.576683044433594,
      "logps/ref_rejected": -87.84639739990234,
      "logps/rejected": -258.2732849121094,
      "loss": 1.008,
      "margin_dpo/margin_mean": 84.04900360107422,
      "margin_dpo/margin_std": 107.98806762695312,
      "step": 218
    },
    {
      "epoch": 0.32158590308370044,
      "fcm_dpo/beta": 0.007075803354382515,
      "fcm_dpo/delta": 0.09832102805376053,
      "fcm_dpo/margin": 71.45269775390625,
      "fcm_dpo/q_t": 0.3926897644996643,
      "grad_norm": 25.61137580871582,
      "learning_rate": 4.303689819449636e-07,
      "logits/chosen": -0.4170767664909363,
      "logits/rejected": -0.4031888246536255,
      "logps/chosen": -152.66976928710938,
      "logps/ref_chosen": -61.083858489990234,
      "logps/ref_rejected": -85.83042907714844,
      "logps/rejected": -248.86904907226562,
      "loss": 1.0769,
      "margin_dpo/margin_mean": 71.45269775390625,
      "margin_dpo/margin_std": 110.11042785644531,
      "step": 219
    },
    {
      "epoch": 0.32305433186490456,
      "fcm_dpo/beta": 0.007226555608212948,
      "fcm_dpo/delta": 0.1684371680021286,
      "fcm_dpo/margin": 60.78836441040039,
      "fcm_dpo/q_t": 0.40072399377822876,
      "grad_norm": 25.96778106689453,
      "learning_rate": 4.2947798076611047e-07,
      "logits/chosen": -0.40701720118522644,
      "logits/rejected": -0.3823140263557434,
      "logps/chosen": -173.3245849609375,
      "logps/ref_chosen": -70.03128051757812,
      "logps/ref_rejected": -87.68551635742188,
      "logps/rejected": -251.76718139648438,
      "loss": 1.0723,
      "margin_dpo/margin_mean": 60.788368225097656,
      "margin_dpo/margin_std": 78.86666870117188,
      "step": 220
    },
    {
      "epoch": 0.3245227606461087,
      "fcm_dpo/beta": 0.007043336518108845,
      "fcm_dpo/delta": -0.2806098461151123,
      "fcm_dpo/margin": 121.70394897460938,
      "fcm_dpo/q_t": 0.3166281580924988,
      "grad_norm": 29.857473373413086,
      "learning_rate": 4.285822501755485e-07,
      "logits/chosen": -0.36656078696250916,
      "logits/rejected": -0.3734094500541687,
      "logps/chosen": -148.55584716796875,
      "logps/ref_chosen": -52.15470886230469,
      "logps/ref_rejected": -106.46768188476562,
      "logps/rejected": -324.57275390625,
      "loss": 0.8286,
      "margin_dpo/margin_mean": 121.70394897460938,
      "margin_dpo/margin_std": 108.79747009277344,
      "step": 221
    },
    {
      "epoch": 0.32599118942731276,
      "fcm_dpo/beta": 0.00688206497579813,
      "fcm_dpo/delta": -0.058866240084171295,
      "fcm_dpo/margin": 95.16747283935547,
      "fcm_dpo/q_t": 0.3594551086425781,
      "grad_norm": 27.359058380126953,
      "learning_rate": 4.276818137766118e-07,
      "logits/chosen": -0.4119839668273926,
      "logits/rejected": -0.40599650144577026,
      "logps/chosen": -164.09307861328125,
      "logps/ref_chosen": -60.971099853515625,
      "logps/ref_rejected": -100.00115203857422,
      "logps/rejected": -298.29058837890625,
      "loss": 0.9574,
      "margin_dpo/margin_mean": 95.16747283935547,
      "margin_dpo/margin_std": 109.49444580078125,
      "step": 222
    },
    {
      "epoch": 0.3274596182085169,
      "fcm_dpo/beta": 0.006874423939734697,
      "fcm_dpo/delta": 0.045850180089473724,
      "fcm_dpo/margin": 80.98518371582031,
      "fcm_dpo/q_t": 0.38237255811691284,
      "grad_norm": 31.334300994873047,
      "learning_rate": 4.2677669529663686e-07,
      "logits/chosen": -0.3320853114128113,
      "logits/rejected": -0.31920111179351807,
      "logps/chosen": -163.82130432128906,
      "logps/ref_chosen": -52.64057540893555,
      "logps/ref_rejected": -82.82502746582031,
      "logps/rejected": -274.9909362792969,
      "loss": 1.0635,
      "margin_dpo/margin_mean": 80.98518371582031,
      "margin_dpo/margin_std": 123.70716857910156,
      "step": 223
    },
    {
      "epoch": 0.328928046989721,
      "fcm_dpo/beta": 0.006771251559257507,
      "fcm_dpo/delta": -0.05909021943807602,
      "fcm_dpo/margin": 96.40702056884766,
      "fcm_dpo/q_t": 0.3656306862831116,
      "grad_norm": 32.546329498291016,
      "learning_rate": 4.2586691858633747e-07,
      "logits/chosen": -0.36080002784729004,
      "logits/rejected": -0.3402705192565918,
      "logps/chosen": -143.6527862548828,
      "logps/ref_chosen": -48.59541320800781,
      "logps/ref_rejected": -77.11648559570312,
      "logps/rejected": -268.58087158203125,
      "loss": 0.9886,
      "margin_dpo/margin_mean": 96.40702056884766,
      "margin_dpo/margin_std": 124.457275390625,
      "step": 224
    },
    {
      "epoch": 0.3303964757709251,
      "fcm_dpo/beta": 0.006611201912164688,
      "fcm_dpo/delta": -0.19814401865005493,
      "fcm_dpo/margin": 118.33515930175781,
      "fcm_dpo/q_t": 0.3383178114891052,
      "grad_norm": 26.60202407836914,
      "learning_rate": 4.249525076191759e-07,
      "logits/chosen": -0.35223451256752014,
      "logits/rejected": -0.33783799409866333,
      "logps/chosen": -169.36512756347656,
      "logps/ref_chosen": -58.000465393066406,
      "logps/ref_rejected": -99.90291595458984,
      "logps/rejected": -329.60272216796875,
      "loss": 0.9074,
      "margin_dpo/margin_mean": 118.33515930175781,
      "margin_dpo/margin_std": 133.42843627929688,
      "step": 225
    },
    {
      "epoch": 0.33186490455212925,
      "fcm_dpo/beta": 0.006490599364042282,
      "fcm_dpo/delta": 0.004586532711982727,
      "fcm_dpo/margin": 91.46955871582031,
      "fcm_dpo/q_t": 0.37888455390930176,
      "grad_norm": 27.73929214477539,
      "learning_rate": 4.2403348649073167e-07,
      "logits/chosen": -0.389433890581131,
      "logits/rejected": -0.3457120954990387,
      "logps/chosen": -155.97816467285156,
      "logps/ref_chosen": -58.898799896240234,
      "logps/ref_rejected": -78.68775939941406,
      "logps/rejected": -267.2366943359375,
      "loss": 1.0272,
      "margin_dpo/margin_mean": 91.46955871582031,
      "margin_dpo/margin_std": 127.21261596679688,
      "step": 226
    },
    {
      "epoch": 0.3333333333333333,
      "fcm_dpo/beta": 0.00636161956936121,
      "fcm_dpo/delta": -0.16426458954811096,
      "fcm_dpo/margin": 117.93624877929688,
      "fcm_dpo/q_t": 0.34007027745246887,
      "grad_norm": 27.874391555786133,
      "learning_rate": 4.2310987941806615e-07,
      "logits/chosen": -0.37095752358436584,
      "logits/rejected": -0.35500913858413696,
      "logps/chosen": -170.41358947753906,
      "logps/ref_chosen": -59.072181701660156,
      "logps/ref_rejected": -99.41236877441406,
      "logps/rejected": -328.6900329589844,
      "loss": 0.9131,
      "margin_dpo/margin_mean": 117.93626403808594,
      "margin_dpo/margin_std": 129.8984832763672,
      "step": 227
    },
    {
      "epoch": 0.33480176211453744,
      "fcm_dpo/beta": 0.006433088798075914,
      "fcm_dpo/delta": 0.07818908244371414,
      "fcm_dpo/margin": 81.71975708007812,
      "fcm_dpo/q_t": 0.38696879148483276,
      "grad_norm": 28.652111053466797,
      "learning_rate": 4.2218171073908463e-07,
      "logits/chosen": -0.3662306070327759,
      "logits/rejected": -0.3457154631614685,
      "logps/chosen": -179.63031005859375,
      "logps/ref_chosen": -65.89128875732422,
      "logps/ref_rejected": -91.04875183105469,
      "logps/rejected": -286.50750732421875,
      "loss": 1.0584,
      "margin_dpo/margin_mean": 81.71975708007812,
      "margin_dpo/margin_std": 119.07015228271484,
      "step": 228
    },
    {
      "epoch": 0.33627019089574156,
      "fcm_dpo/beta": 0.006459444761276245,
      "fcm_dpo/delta": 0.005232607014477253,
      "fcm_dpo/margin": 92.12015533447266,
      "fcm_dpo/q_t": 0.3724709153175354,
      "grad_norm": 33.15553665161133,
      "learning_rate": 4.212490049118951e-07,
      "logits/chosen": -0.44420498609542847,
      "logits/rejected": -0.412958562374115,
      "logps/chosen": -181.48049926757812,
      "logps/ref_chosen": -70.70637512207031,
      "logps/ref_rejected": -84.52741241455078,
      "logps/rejected": -287.42169189453125,
      "loss": 1.0091,
      "margin_dpo/margin_mean": 92.12016296386719,
      "margin_dpo/margin_std": 121.14804077148438,
      "step": 229
    },
    {
      "epoch": 0.3377386196769457,
      "fcm_dpo/beta": 0.006237386725842953,
      "fcm_dpo/delta": -0.20489397644996643,
      "fcm_dpo/margin": 126.29830169677734,
      "fcm_dpo/q_t": 0.33031266927719116,
      "grad_norm": 31.86959457397461,
      "learning_rate": 4.203117865141635e-07,
      "logits/chosen": -0.32878124713897705,
      "logits/rejected": -0.3298182487487793,
      "logps/chosen": -130.9337158203125,
      "logps/ref_chosen": -39.282005310058594,
      "logps/ref_rejected": -85.62191009521484,
      "logps/rejected": -303.5719299316406,
      "loss": 0.8704,
      "margin_dpo/margin_mean": 126.29830932617188,
      "margin_dpo/margin_std": 123.00277709960938,
      "step": 230
    },
    {
      "epoch": 0.3392070484581498,
      "fcm_dpo/beta": 0.0062300898134708405,
      "fcm_dpo/delta": 0.04776204377412796,
      "fcm_dpo/margin": 89.06995391845703,
      "fcm_dpo/q_t": 0.3806319832801819,
      "grad_norm": 26.72955894470215,
      "learning_rate": 4.1937008024246625e-07,
      "logits/chosen": -0.36490845680236816,
      "logits/rejected": -0.33270594477653503,
      "logps/chosen": -161.59432983398438,
      "logps/ref_chosen": -63.27644348144531,
      "logps/ref_rejected": -74.1239013671875,
      "logps/rejected": -261.5117492675781,
      "loss": 1.0029,
      "margin_dpo/margin_mean": 89.06995391845703,
      "margin_dpo/margin_std": 107.24581909179688,
      "step": 231
    },
    {
      "epoch": 0.3406754772393539,
      "fcm_dpo/beta": 0.006363061256706715,
      "fcm_dpo/delta": 0.136710524559021,
      "fcm_dpo/margin": 73.8527603149414,
      "fcm_dpo/q_t": 0.40357333421707153,
      "grad_norm": 31.16952896118164,
      "learning_rate": 4.1842391091163933e-07,
      "logits/chosen": -0.3764330744743347,
      "logits/rejected": -0.3467271029949188,
      "logps/chosen": -187.44082641601562,
      "logps/ref_chosen": -70.74876403808594,
      "logps/ref_rejected": -83.97706604003906,
      "logps/rejected": -274.52191162109375,
      "loss": 1.092,
      "margin_dpo/margin_mean": 73.8527603149414,
      "margin_dpo/margin_std": 118.7332763671875,
      "step": 232
    },
    {
      "epoch": 0.342143906020558,
      "fcm_dpo/beta": 0.006316785700619221,
      "fcm_dpo/delta": -0.11919762194156647,
      "fcm_dpo/margin": 112.51909637451172,
      "fcm_dpo/q_t": 0.3590048551559448,
      "grad_norm": 28.269271850585938,
      "learning_rate": 4.174733034541245e-07,
      "logits/chosen": -0.3798902630805969,
      "logits/rejected": -0.3772153854370117,
      "logps/chosen": -157.00439453125,
      "logps/ref_chosen": -54.8829345703125,
      "logps/ref_rejected": -107.4800796508789,
      "logps/rejected": -322.12060546875,
      "loss": 0.9786,
      "margin_dpo/margin_mean": 112.51909637451172,
      "margin_dpo/margin_std": 150.15170288085938,
      "step": 233
    },
    {
      "epoch": 0.3436123348017621,
      "fcm_dpo/beta": 0.006136967800557613,
      "fcm_dpo/delta": -0.0946032702922821,
      "fcm_dpo/margin": 111.787109375,
      "fcm_dpo/q_t": 0.3509105145931244,
      "grad_norm": 38.759918212890625,
      "learning_rate": 4.165182829193126e-07,
      "logits/chosen": -0.3641296625137329,
      "logits/rejected": -0.39079636335372925,
      "logps/chosen": -142.52503967285156,
      "logps/ref_chosen": -44.094520568847656,
      "logps/ref_rejected": -100.00663757324219,
      "logps/rejected": -310.2242431640625,
      "loss": 0.9207,
      "margin_dpo/margin_mean": 111.787109375,
      "margin_dpo/margin_std": 113.17666625976562,
      "step": 234
    },
    {
      "epoch": 0.34508076358296624,
      "fcm_dpo/beta": 0.0063125560991466045,
      "fcm_dpo/delta": 0.14529383182525635,
      "fcm_dpo/margin": 72.99315643310547,
      "fcm_dpo/q_t": 0.40060219168663025,
      "grad_norm": 33.466941833496094,
      "learning_rate": 4.1555887447288255e-07,
      "logits/chosen": -0.4195903241634369,
      "logits/rejected": -0.39497965574264526,
      "logps/chosen": -181.33181762695312,
      "logps/ref_chosen": -62.237911224365234,
      "logps/ref_rejected": -90.39506530761719,
      "logps/rejected": -282.48211669921875,
      "loss": 1.0955,
      "margin_dpo/margin_mean": 72.9931640625,
      "margin_dpo/margin_std": 114.59500122070312,
      "step": 235
    },
    {
      "epoch": 0.3465491923641703,
      "fcm_dpo/beta": 0.006248786114156246,
      "fcm_dpo/delta": -0.09669020771980286,
      "fcm_dpo/margin": 110.42259216308594,
      "fcm_dpo/q_t": 0.3470776677131653,
      "grad_norm": 50.5585823059082,
      "learning_rate": 4.1459510339613946e-07,
      "logits/chosen": -0.36349114775657654,
      "logits/rejected": -0.3634676933288574,
      "logps/chosen": -136.66470336914062,
      "logps/ref_chosen": -49.34136199951172,
      "logps/ref_rejected": -103.51162719726562,
      "logps/rejected": -301.257568359375,
      "loss": 0.8949,
      "margin_dpo/margin_mean": 110.42259216308594,
      "margin_dpo/margin_std": 99.5032958984375,
      "step": 236
    },
    {
      "epoch": 0.34801762114537443,
      "fcm_dpo/beta": 0.006244904361665249,
      "fcm_dpo/delta": 0.023870917037129402,
      "fcm_dpo/margin": 92.47233581542969,
      "fcm_dpo/q_t": 0.37653428316116333,
      "grad_norm": 30.452253341674805,
      "learning_rate": 4.136269950853473e-07,
      "logits/chosen": -0.35491400957107544,
      "logits/rejected": -0.34882035851478577,
      "logps/chosen": -170.31382751464844,
      "logps/ref_chosen": -54.168121337890625,
      "logps/ref_rejected": -94.78036499023438,
      "logps/rejected": -303.3984069824219,
      "loss": 1.0111,
      "margin_dpo/margin_mean": 92.47232818603516,
      "margin_dpo/margin_std": 120.4630126953125,
      "step": 237
    },
    {
      "epoch": 0.34948604992657856,
      "fcm_dpo/beta": 0.006248504854738712,
      "fcm_dpo/delta": 0.04543028026819229,
      "fcm_dpo/margin": 89.06282043457031,
      "fcm_dpo/q_t": 0.3825730085372925,
      "grad_norm": 31.341205596923828,
      "learning_rate": 4.126545750510605e-07,
      "logits/chosen": -0.3699679374694824,
      "logits/rejected": -0.3880308270454407,
      "logps/chosen": -163.38356018066406,
      "logps/ref_chosen": -53.973121643066406,
      "logps/ref_rejected": -89.41795349121094,
      "logps/rejected": -287.8912048339844,
      "loss": 1.0143,
      "margin_dpo/margin_mean": 89.06282043457031,
      "margin_dpo/margin_std": 114.02665710449219,
      "step": 238
    },
    {
      "epoch": 0.3509544787077827,
      "fcm_dpo/beta": 0.0062086135149002075,
      "fcm_dpo/delta": -0.038354501128196716,
      "fcm_dpo/margin": 102.07293701171875,
      "fcm_dpo/q_t": 0.36239856481552124,
      "grad_norm": 28.831066131591797,
      "learning_rate": 4.116778689174514e-07,
      "logits/chosen": -0.34534239768981934,
      "logits/rejected": -0.32214316725730896,
      "logps/chosen": -178.08087158203125,
      "logps/ref_chosen": -58.09782409667969,
      "logps/ref_rejected": -93.59294128417969,
      "logps/rejected": -315.64892578125,
      "loss": 0.9607,
      "margin_dpo/margin_mean": 102.07293701171875,
      "margin_dpo/margin_std": 110.16291809082031,
      "step": 239
    },
    {
      "epoch": 0.3524229074889868,
      "fcm_dpo/beta": 0.006256358698010445,
      "fcm_dpo/delta": 0.03319564089179039,
      "fcm_dpo/margin": 90.84452056884766,
      "fcm_dpo/q_t": 0.37809598445892334,
      "grad_norm": 34.58857727050781,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -0.3797963261604309,
      "logits/rejected": -0.3545226454734802,
      "logps/chosen": -188.4025421142578,
      "logps/ref_chosen": -60.6144905090332,
      "logps/ref_rejected": -74.1185302734375,
      "logps/rejected": -292.7510986328125,
      "loss": 1.0351,
      "margin_dpo/margin_mean": 90.84451293945312,
      "margin_dpo/margin_std": 124.30812072753906,
      "step": 240
    },
    {
      "epoch": 0.35389133627019087,
      "fcm_dpo/beta": 0.006106827408075333,
      "fcm_dpo/delta": -0.2318933755159378,
      "fcm_dpo/margin": 133.21487426757812,
      "fcm_dpo/q_t": 0.33365797996520996,
      "grad_norm": 27.41342544555664,
      "learning_rate": 4.097117014129903e-07,
      "logits/chosen": -0.42394572496414185,
      "logits/rejected": -0.39460113644599915,
      "logps/chosen": -171.47601318359375,
      "logps/ref_chosen": -66.091064453125,
      "logps/ref_rejected": -88.06088256835938,
      "logps/rejected": -326.66070556640625,
      "loss": 0.8839,
      "margin_dpo/margin_mean": 133.21487426757812,
      "margin_dpo/margin_std": 142.80030822753906,
      "step": 241
    },
    {
      "epoch": 0.355359765051395,
      "fcm_dpo/beta": 0.006066991947591305,
      "fcm_dpo/delta": 0.01960517093539238,
      "fcm_dpo/margin": 95.71859741210938,
      "fcm_dpo/q_t": 0.3771246075630188,
      "grad_norm": 29.23126983642578,
      "learning_rate": 4.087222918524807e-07,
      "logits/chosen": -0.3212859630584717,
      "logits/rejected": -0.2920798659324646,
      "logps/chosen": -186.41012573242188,
      "logps/ref_chosen": -67.86392974853516,
      "logps/ref_rejected": -83.36033630371094,
      "logps/rejected": -297.6251220703125,
      "loss": 1.0017,
      "margin_dpo/margin_mean": 95.71859741210938,
      "margin_dpo/margin_std": 121.07659912109375,
      "step": 242
    },
    {
      "epoch": 0.3568281938325991,
      "fcm_dpo/beta": 0.005917678587138653,
      "fcm_dpo/delta": -0.11988495290279388,
      "fcm_dpo/margin": 120.15345764160156,
      "fcm_dpo/q_t": 0.34703272581100464,
      "grad_norm": 27.846588134765625,
      "learning_rate": 4.07728699811968e-07,
      "logits/chosen": -0.36647382378578186,
      "logits/rejected": -0.3263128399848938,
      "logps/chosen": -183.94918823242188,
      "logps/ref_chosen": -63.0842399597168,
      "logps/ref_rejected": -76.33563232421875,
      "logps/rejected": -317.35406494140625,
      "loss": 0.9108,
      "margin_dpo/margin_mean": 120.1534652709961,
      "margin_dpo/margin_std": 125.44760131835938,
      "step": 243
    },
    {
      "epoch": 0.35829662261380324,
      "fcm_dpo/beta": 0.005800644867122173,
      "fcm_dpo/delta": -0.08758918195962906,
      "fcm_dpo/margin": 117.43812561035156,
      "fcm_dpo/q_t": 0.3503507375717163,
      "grad_norm": 31.091215133666992,
      "learning_rate": 4.067309514735267e-07,
      "logits/chosen": -0.42268094420433044,
      "logits/rejected": -0.4104187488555908,
      "logps/chosen": -163.81822204589844,
      "logps/ref_chosen": -61.140689849853516,
      "logps/ref_rejected": -94.89193725585938,
      "logps/rejected": -315.00762939453125,
      "loss": 0.9077,
      "margin_dpo/margin_mean": 117.43812561035156,
      "margin_dpo/margin_std": 110.86767578125,
      "step": 244
    },
    {
      "epoch": 0.35976505139500736,
      "fcm_dpo/beta": 0.00573451304808259,
      "fcm_dpo/delta": 0.01733044534921646,
      "fcm_dpo/margin": 101.28717803955078,
      "fcm_dpo/q_t": 0.3786167502403259,
      "grad_norm": 33.05402755737305,
      "learning_rate": 4.057290731287531e-07,
      "logits/chosen": -0.40215417742729187,
      "logits/rejected": -0.3637341856956482,
      "logps/chosen": -191.681396484375,
      "logps/ref_chosen": -67.26228332519531,
      "logps/ref_rejected": -87.64010620117188,
      "logps/rejected": -313.3464050292969,
      "loss": 1.0265,
      "margin_dpo/margin_mean": 101.28717041015625,
      "margin_dpo/margin_std": 134.58218383789062,
      "step": 245
    },
    {
      "epoch": 0.36123348017621143,
      "fcm_dpo/beta": 0.005821553990244865,
      "fcm_dpo/delta": 0.027677321806550026,
      "fcm_dpo/margin": 98.57552337646484,
      "fcm_dpo/q_t": 0.3820421099662781,
      "grad_norm": 26.930212020874023,
      "learning_rate": 4.047230911780736e-07,
      "logits/chosen": -0.4357486963272095,
      "logits/rejected": -0.394938200712204,
      "logps/chosen": -191.74822998046875,
      "logps/ref_chosen": -66.69696807861328,
      "logps/ref_rejected": -84.34634399414062,
      "logps/rejected": -307.97314453125,
      "loss": 1.0254,
      "margin_dpo/margin_mean": 98.57553100585938,
      "margin_dpo/margin_std": 138.3719482421875,
      "step": 246
    },
    {
      "epoch": 0.36270190895741555,
      "fcm_dpo/beta": 0.005621997639536858,
      "fcm_dpo/delta": -0.23428848385810852,
      "fcm_dpo/margin": 144.79278564453125,
      "fcm_dpo/q_t": 0.33090120553970337,
      "grad_norm": 26.57573890686035,
      "learning_rate": 4.0371303213004814e-07,
      "logits/chosen": -0.3413492441177368,
      "logits/rejected": -0.33773866295814514,
      "logps/chosen": -205.19589233398438,
      "logps/ref_chosen": -56.6053466796875,
      "logps/ref_rejected": -106.29326629638672,
      "logps/rejected": -399.6766357421875,
      "loss": 0.8815,
      "margin_dpo/margin_mean": 144.79278564453125,
      "margin_dpo/margin_std": 153.0501708984375,
      "step": 247
    },
    {
      "epoch": 0.3641703377386197,
      "fcm_dpo/beta": 0.005500612780451775,
      "fcm_dpo/delta": -0.0632907822728157,
      "fcm_dpo/margin": 119.71734619140625,
      "fcm_dpo/q_t": 0.35248565673828125,
      "grad_norm": 40.54594039916992,
      "learning_rate": 4.0269892260067197e-07,
      "logits/chosen": -0.3447737991809845,
      "logits/rejected": -0.358917772769928,
      "logps/chosen": -171.75363159179688,
      "logps/ref_chosen": -44.043216705322266,
      "logps/ref_rejected": -91.85687255859375,
      "logps/rejected": -339.2846374511719,
      "loss": 0.908,
      "margin_dpo/margin_mean": 119.71734619140625,
      "margin_dpo/margin_std": 105.92500305175781,
      "step": 248
    },
    {
      "epoch": 0.3656387665198238,
      "fcm_dpo/beta": 0.005672304891049862,
      "fcm_dpo/delta": 0.20714300870895386,
      "fcm_dpo/margin": 70.76122283935547,
      "fcm_dpo/q_t": 0.4150072932243347,
      "grad_norm": 52.06844711303711,
      "learning_rate": 4.0168078931267426e-07,
      "logits/chosen": -0.3665584921836853,
      "logits/rejected": -0.3416873514652252,
      "logps/chosen": -225.1993408203125,
      "logps/ref_chosen": -62.442352294921875,
      "logps/ref_rejected": -80.46806335449219,
      "logps/rejected": -313.98626708984375,
      "loss": 1.1559,
      "margin_dpo/margin_mean": 70.76122283935547,
      "margin_dpo/margin_std": 133.9126739501953,
      "step": 249
    },
    {
      "epoch": 0.3671071953010279,
      "fcm_dpo/beta": 0.005674234591424465,
      "fcm_dpo/delta": -0.10982278734445572,
      "fcm_dpo/margin": 123.74378967285156,
      "fcm_dpo/q_t": 0.34339314699172974,
      "grad_norm": 34.09236526489258,
      "learning_rate": 4.006586590948141e-07,
      "logits/chosen": -0.38642430305480957,
      "logits/rejected": -0.33151566982269287,
      "logps/chosen": -202.39520263671875,
      "logps/ref_chosen": -65.63668823242188,
      "logps/ref_rejected": -73.87184143066406,
      "logps/rejected": -334.3741455078125,
      "loss": 0.8893,
      "margin_dpo/margin_mean": 123.74378967285156,
      "margin_dpo/margin_std": 108.41996765136719,
      "step": 250
    },
    {
      "epoch": 0.368575624082232,
      "fcm_dpo/beta": 0.005667783785611391,
      "fcm_dpo/delta": 0.09314079582691193,
      "fcm_dpo/margin": 90.27076721191406,
      "fcm_dpo/q_t": 0.3912964463233948,
      "grad_norm": 54.34296798706055,
      "learning_rate": 3.9963255888117325e-07,
      "logits/chosen": -0.33801060914993286,
      "logits/rejected": -0.3055931329727173,
      "logps/chosen": -211.67941284179688,
      "logps/ref_chosen": -57.182716369628906,
      "logps/ref_rejected": -77.66343688964844,
      "logps/rejected": -322.430908203125,
      "loss": 1.0705,
      "margin_dpo/margin_mean": 90.27076721191406,
      "margin_dpo/margin_std": 133.482421875,
      "step": 251
    },
    {
      "epoch": 0.3700440528634361,
      "fcm_dpo/beta": 0.005632590502500534,
      "fcm_dpo/delta": -0.06894432753324509,
      "fcm_dpo/margin": 117.80094909667969,
      "fcm_dpo/q_t": 0.34721171855926514,
      "grad_norm": 34.09019088745117,
      "learning_rate": 3.9860251571044666e-07,
      "logits/chosen": -0.4143436849117279,
      "logits/rejected": -0.3699311912059784,
      "logps/chosen": -211.8092041015625,
      "logps/ref_chosen": -71.68563842773438,
      "logps/ref_rejected": -84.75799560546875,
      "logps/rejected": -342.6824951171875,
      "loss": 0.9072,
      "margin_dpo/margin_mean": 117.80094909667969,
      "margin_dpo/margin_std": 102.10981750488281,
      "step": 252
    },
    {
      "epoch": 0.37151248164464024,
      "fcm_dpo/beta": 0.0056702555157244205,
      "fcm_dpo/delta": -0.024740278720855713,
      "fcm_dpo/margin": 109.68414306640625,
      "fcm_dpo/q_t": 0.3643852472305298,
      "grad_norm": 29.00065040588379,
      "learning_rate": 3.9756855672522986e-07,
      "logits/chosen": -0.40758126974105835,
      "logits/rejected": -0.3990533649921417,
      "logps/chosen": -192.28323364257812,
      "logps/ref_chosen": -69.1339340209961,
      "logps/ref_rejected": -98.70252990722656,
      "logps/rejected": -331.5359802246094,
      "loss": 0.9822,
      "margin_dpo/margin_mean": 109.68414306640625,
      "margin_dpo/margin_std": 129.01275634765625,
      "step": 253
    },
    {
      "epoch": 0.37298091042584436,
      "fcm_dpo/beta": 0.005641551688313484,
      "fcm_dpo/delta": 0.04004208371043205,
      "fcm_dpo/margin": 99.6583251953125,
      "fcm_dpo/q_t": 0.3856009840965271,
      "grad_norm": 27.635013580322266,
      "learning_rate": 3.965307091713037e-07,
      "logits/chosen": -0.41606834530830383,
      "logits/rejected": -0.397452175617218,
      "logps/chosen": -183.28872680664062,
      "logps/ref_chosen": -54.154998779296875,
      "logps/ref_rejected": -90.30764770507812,
      "logps/rejected": -319.0997009277344,
      "loss": 1.0509,
      "margin_dpo/margin_mean": 99.6583251953125,
      "margin_dpo/margin_std": 151.13661193847656,
      "step": 254
    },
    {
      "epoch": 0.3744493392070485,
      "fcm_dpo/beta": 0.005625682417303324,
      "fcm_dpo/delta": 0.0007314831018447876,
      "fcm_dpo/margin": 106.40800476074219,
      "fcm_dpo/q_t": 0.37160855531692505,
      "grad_norm": 29.136993408203125,
      "learning_rate": 3.954890003969163e-07,
      "logits/chosen": -0.3470228314399719,
      "logits/rejected": -0.3381271958351135,
      "logps/chosen": -193.8470458984375,
      "logps/ref_chosen": -57.14167022705078,
      "logps/ref_rejected": -90.2085952758789,
      "logps/rejected": -333.32196044921875,
      "loss": 1.0155,
      "margin_dpo/margin_mean": 106.40800476074219,
      "margin_dpo/margin_std": 140.4176483154297,
      "step": 255
    },
    {
      "epoch": 0.37591776798825255,
      "fcm_dpo/beta": 0.005624064244329929,
      "fcm_dpo/delta": -0.03853630647063255,
      "fcm_dpo/margin": 113.09561157226562,
      "fcm_dpo/q_t": 0.3635770082473755,
      "grad_norm": 43.678157806396484,
      "learning_rate": 3.944434578520628e-07,
      "logits/chosen": -0.3604806661605835,
      "logits/rejected": -0.3640385866165161,
      "logps/chosen": -175.4744110107422,
      "logps/ref_chosen": -55.163490295410156,
      "logps/ref_rejected": -92.56291961669922,
      "logps/rejected": -325.969482421875,
      "loss": 0.9694,
      "margin_dpo/margin_mean": 113.09561157226562,
      "margin_dpo/margin_std": 133.815185546875,
      "step": 256
    },
    {
      "epoch": 0.37738619676945667,
      "fcm_dpo/beta": 0.005505690351128578,
      "fcm_dpo/delta": -0.06898978352546692,
      "fcm_dpo/margin": 120.23841857910156,
      "fcm_dpo/q_t": 0.35991615056991577,
      "grad_norm": 27.866411209106445,
      "learning_rate": 3.933941090877615e-07,
      "logits/chosen": -0.3533473610877991,
      "logits/rejected": -0.33796215057373047,
      "logps/chosen": -166.825439453125,
      "logps/ref_chosen": -49.42369842529297,
      "logps/ref_rejected": -79.53791809082031,
      "logps/rejected": -317.1780700683594,
      "loss": 0.9576,
      "margin_dpo/margin_mean": 120.23841857910156,
      "margin_dpo/margin_std": 135.40008544921875,
      "step": 257
    },
    {
      "epoch": 0.3788546255506608,
      "fcm_dpo/beta": 0.005546920001506805,
      "fcm_dpo/delta": -0.001775261014699936,
      "fcm_dpo/margin": 108.45994567871094,
      "fcm_dpo/q_t": 0.36892908811569214,
      "grad_norm": 33.50674819946289,
      "learning_rate": 3.923409817553284e-07,
      "logits/chosen": -0.35097846388816833,
      "logits/rejected": -0.34521350264549255,
      "logps/chosen": -207.83615112304688,
      "logps/ref_chosen": -59.384124755859375,
      "logps/ref_rejected": -95.99010467529297,
      "logps/rejected": -352.902099609375,
      "loss": 1.0141,
      "margin_dpo/margin_mean": 108.45994567871094,
      "margin_dpo/margin_std": 145.33441162109375,
      "step": 258
    },
    {
      "epoch": 0.3803230543318649,
      "fcm_dpo/beta": 0.005560112651437521,
      "fcm_dpo/delta": 0.057319387793540955,
      "fcm_dpo/margin": 98.13949584960938,
      "fcm_dpo/q_t": 0.38358962535858154,
      "grad_norm": 33.96443557739258,
      "learning_rate": 3.9128410360564793e-07,
      "logits/chosen": -0.41265004873275757,
      "logits/rejected": -0.40723663568496704,
      "logps/chosen": -191.66282653808594,
      "logps/ref_chosen": -52.828346252441406,
      "logps/ref_rejected": -89.191650390625,
      "logps/rejected": -326.1656494140625,
      "loss": 1.0362,
      "margin_dpo/margin_mean": 98.13948822021484,
      "margin_dpo/margin_std": 132.66073608398438,
      "step": 259
    },
    {
      "epoch": 0.38179148311306904,
      "fcm_dpo/beta": 0.005574359092861414,
      "fcm_dpo/delta": -0.05944538116455078,
      "fcm_dpo/margin": 117.58509826660156,
      "fcm_dpo/q_t": 0.35981056094169617,
      "grad_norm": 32.101783752441406,
      "learning_rate": 3.9022350248844246e-07,
      "logits/chosen": -0.37826740741729736,
      "logits/rejected": -0.3950265347957611,
      "logps/chosen": -189.1026611328125,
      "logps/ref_chosen": -47.41767501831055,
      "logps/ref_rejected": -95.08978271484375,
      "logps/rejected": -354.35986328125,
      "loss": 0.9407,
      "margin_dpo/margin_mean": 117.58509826660156,
      "margin_dpo/margin_std": 128.20523071289062,
      "step": 260
    },
    {
      "epoch": 0.3832599118942731,
      "fcm_dpo/beta": 0.005468597635626793,
      "fcm_dpo/delta": -0.06998618692159653,
      "fcm_dpo/margin": 121.63386535644531,
      "fcm_dpo/q_t": 0.35736051201820374,
      "grad_norm": 23.966012954711914,
      "learning_rate": 3.891592063515376e-07,
      "logits/chosen": -0.319491982460022,
      "logits/rejected": -0.3146633505821228,
      "logps/chosen": -190.02244567871094,
      "logps/ref_chosen": -53.03137969970703,
      "logps/ref_rejected": -88.51494598388672,
      "logps/rejected": -347.139892578125,
      "loss": 0.9529,
      "margin_dpo/margin_mean": 121.63386535644531,
      "margin_dpo/margin_std": 140.09190368652344,
      "step": 261
    },
    {
      "epoch": 0.38472834067547723,
      "fcm_dpo/beta": 0.005526003893464804,
      "fcm_dpo/delta": 0.0573844239115715,
      "fcm_dpo/margin": 98.6690902709961,
      "fcm_dpo/q_t": 0.37783992290496826,
      "grad_norm": 33.454994201660156,
      "learning_rate": 3.880912432401264e-07,
      "logits/chosen": -0.2890551686286926,
      "logits/rejected": -0.26014506816864014,
      "logps/chosen": -221.21661376953125,
      "logps/ref_chosen": -59.620140075683594,
      "logps/ref_rejected": -86.41853332519531,
      "logps/rejected": -346.68408203125,
      "loss": 1.0033,
      "margin_dpo/margin_mean": 98.6690902709961,
      "margin_dpo/margin_std": 113.80564880371094,
      "step": 262
    },
    {
      "epoch": 0.38619676945668135,
      "fcm_dpo/beta": 0.005366258788853884,
      "fcm_dpo/delta": -0.16116134822368622,
      "fcm_dpo/margin": 139.3936309814453,
      "fcm_dpo/q_t": 0.3375219702720642,
      "grad_norm": 30.789506912231445,
      "learning_rate": 3.870196412960302e-07,
      "logits/chosen": -0.3338819146156311,
      "logits/rejected": -0.30775582790374756,
      "logps/chosen": -204.53103637695312,
      "logps/ref_chosen": -59.42094421386719,
      "logps/ref_rejected": -96.85720825195312,
      "logps/rejected": -381.3609619140625,
      "loss": 0.8897,
      "margin_dpo/margin_mean": 139.3936309814453,
      "margin_dpo/margin_std": 136.70474243164062,
      "step": 263
    },
    {
      "epoch": 0.3876651982378855,
      "fcm_dpo/beta": 0.0052995504811406136,
      "fcm_dpo/delta": -0.014157909899950027,
      "fcm_dpo/margin": 115.54821014404297,
      "fcm_dpo/q_t": 0.3663901686668396,
      "grad_norm": 33.63625717163086,
      "learning_rate": 3.8594442875695665e-07,
      "logits/chosen": -0.3557894825935364,
      "logits/rejected": -0.3422415256500244,
      "logps/chosen": -212.91517639160156,
      "logps/ref_chosen": -62.722084045410156,
      "logps/ref_rejected": -93.85620880126953,
      "logps/rejected": -359.5975341796875,
      "loss": 0.9682,
      "margin_dpo/margin_mean": 115.5482177734375,
      "margin_dpo/margin_std": 127.98780822753906,
      "step": 264
    },
    {
      "epoch": 0.3891336270190896,
      "fcm_dpo/beta": 0.005331607535481453,
      "fcm_dpo/delta": -0.0023941383697092533,
      "fcm_dpo/margin": 112.95767211914062,
      "fcm_dpo/q_t": 0.3753606081008911,
      "grad_norm": 48.9280891418457,
      "learning_rate": 3.848656339557562e-07,
      "logits/chosen": -0.31224972009658813,
      "logits/rejected": -0.2951231598854065,
      "logps/chosen": -235.91409301757812,
      "logps/ref_chosen": -61.971466064453125,
      "logps/ref_rejected": -88.02059936523438,
      "logps/rejected": -374.9208984375,
      "loss": 1.0295,
      "margin_dpo/margin_mean": 112.95767211914062,
      "margin_dpo/margin_std": 163.15493774414062,
      "step": 265
    },
    {
      "epoch": 0.39060205580029367,
      "fcm_dpo/beta": 0.0053364490158855915,
      "fcm_dpo/delta": 0.014164826832711697,
      "fcm_dpo/margin": 109.93594360351562,
      "fcm_dpo/q_t": 0.37322813272476196,
      "grad_norm": 41.273826599121094,
      "learning_rate": 3.8378328531967507e-07,
      "logits/chosen": -0.343116819858551,
      "logits/rejected": -0.2906397581100464,
      "logps/chosen": -235.95025634765625,
      "logps/ref_chosen": -67.09967041015625,
      "logps/ref_rejected": -67.97122192382812,
      "logps/rejected": -346.75775146484375,
      "loss": 0.9942,
      "margin_dpo/margin_mean": 109.93594360351562,
      "margin_dpo/margin_std": 133.6815185546875,
      "step": 266
    },
    {
      "epoch": 0.3920704845814978,
      "fcm_dpo/beta": 0.005292531102895737,
      "fcm_dpo/delta": -0.037379078567028046,
      "fcm_dpo/margin": 119.896484375,
      "fcm_dpo/q_t": 0.36082565784454346,
      "grad_norm": 37.10322189331055,
      "learning_rate": 3.8269741136960646e-07,
      "logits/chosen": -0.3480778932571411,
      "logits/rejected": -0.30662471055984497,
      "logps/chosen": -230.77191162109375,
      "logps/ref_chosen": -68.97075653076172,
      "logps/ref_rejected": -90.16844940185547,
      "logps/rejected": -371.8660888671875,
      "loss": 0.9604,
      "margin_dpo/margin_mean": 119.896484375,
      "margin_dpo/margin_std": 134.846435546875,
      "step": 267
    },
    {
      "epoch": 0.3935389133627019,
      "fcm_dpo/beta": 0.0052706534042954445,
      "fcm_dpo/delta": -0.005634669214487076,
      "fcm_dpo/margin": 114.72981262207031,
      "fcm_dpo/q_t": 0.37070363759994507,
      "grad_norm": 36.76900100708008,
      "learning_rate": 3.8160804071933894e-07,
      "logits/chosen": -0.3364071846008301,
      "logits/rejected": -0.34323662519454956,
      "logps/chosen": -227.5570526123047,
      "logps/ref_chosen": -55.90031051635742,
      "logps/ref_rejected": -101.64763641357422,
      "logps/rejected": -388.0341796875,
      "loss": 0.9952,
      "margin_dpo/margin_mean": 114.72982025146484,
      "margin_dpo/margin_std": 145.24905395507812,
      "step": 268
    },
    {
      "epoch": 0.39500734214390604,
      "fcm_dpo/beta": 0.005157118663191795,
      "fcm_dpo/delta": -0.17022442817687988,
      "fcm_dpo/margin": 146.7906494140625,
      "fcm_dpo/q_t": 0.3414715826511383,
      "grad_norm": 42.1302375793457,
      "learning_rate": 3.8051520207480204e-07,
      "logits/chosen": -0.3691613972187042,
      "logits/rejected": -0.3452231287956238,
      "logps/chosen": -251.450439453125,
      "logps/ref_chosen": -70.03955841064453,
      "logps/ref_rejected": -107.34937286376953,
      "logps/rejected": -435.5509033203125,
      "loss": 0.925,
      "margin_dpo/margin_mean": 146.7906494140625,
      "margin_dpo/margin_std": 168.05130004882812,
      "step": 269
    },
    {
      "epoch": 0.3964757709251101,
      "fcm_dpo/beta": 0.005182851571589708,
      "fcm_dpo/delta": 0.05792323499917984,
      "fcm_dpo/margin": 105.11456298828125,
      "fcm_dpo/q_t": 0.3828323185443878,
      "grad_norm": 37.189781188964844,
      "learning_rate": 3.794189242333106e-07,
      "logits/chosen": -0.41337841749191284,
      "logits/rejected": -0.40365487337112427,
      "logps/chosen": -232.35296630859375,
      "logps/ref_chosen": -69.53347778320312,
      "logps/ref_rejected": -109.92864990234375,
      "logps/rejected": -377.8627014160156,
      "loss": 1.0389,
      "margin_dpo/margin_mean": 105.11456298828125,
      "margin_dpo/margin_std": 145.3082275390625,
      "step": 270
    },
    {
      "epoch": 0.39794419970631423,
      "fcm_dpo/beta": 0.0051218001171946526,
      "fcm_dpo/delta": -0.057347435504198074,
      "fcm_dpo/margin": 127.55348205566406,
      "fcm_dpo/q_t": 0.35683873295783997,
      "grad_norm": 39.590396881103516,
      "learning_rate": 3.7831923608280514e-07,
      "logits/chosen": -0.34910643100738525,
      "logits/rejected": -0.3228174149990082,
      "logps/chosen": -206.6493377685547,
      "logps/ref_chosen": -56.76456832885742,
      "logps/ref_rejected": -92.51383972167969,
      "logps/rejected": -369.95208740234375,
      "loss": 0.9256,
      "margin_dpo/margin_mean": 127.55348205566406,
      "margin_dpo/margin_std": 125.99525451660156,
      "step": 271
    },
    {
      "epoch": 0.39941262848751835,
      "fcm_dpo/beta": 0.005027889274060726,
      "fcm_dpo/delta": -0.1585131734609604,
      "fcm_dpo/margin": 148.55596923828125,
      "fcm_dpo/q_t": 0.33690333366394043,
      "grad_norm": 36.002960205078125,
      "learning_rate": 3.772161666010912e-07,
      "logits/chosen": -0.331684410572052,
      "logits/rejected": -0.3400617837905884,
      "logps/chosen": -184.74874877929688,
      "logps/ref_chosen": -49.497154235839844,
      "logps/ref_rejected": -105.54279327392578,
      "logps/rejected": -389.350341796875,
      "loss": 0.8841,
      "margin_dpo/margin_mean": 148.55596923828125,
      "margin_dpo/margin_std": 140.95240783691406,
      "step": 272
    },
    {
      "epoch": 0.4008810572687225,
      "fcm_dpo/beta": 0.004825687035918236,
      "fcm_dpo/delta": -0.1409759819507599,
      "fcm_dpo/margin": 151.05177307128906,
      "fcm_dpo/q_t": 0.3416686952114105,
      "grad_norm": 34.01618576049805,
      "learning_rate": 3.761097448550755e-07,
      "logits/chosen": -0.3630342483520508,
      "logits/rejected": -0.3401733636856079,
      "logps/chosen": -218.07992553710938,
      "logps/ref_chosen": -62.97539520263672,
      "logps/ref_rejected": -92.49858093261719,
      "logps/rejected": -398.6549072265625,
      "loss": 0.892,
      "margin_dpo/margin_mean": 151.05178833007812,
      "margin_dpo/margin_std": 145.12493896484375,
      "step": 273
    },
    {
      "epoch": 0.4023494860499266,
      "fcm_dpo/beta": 0.004842686001211405,
      "fcm_dpo/delta": 0.012656617909669876,
      "fcm_dpo/margin": 121.4140625,
      "fcm_dpo/q_t": 0.3711016774177551,
      "grad_norm": 29.5135555267334,
      "learning_rate": 3.75e-07,
      "logits/chosen": -0.2929585874080658,
      "logits/rejected": -0.26695334911346436,
      "logps/chosen": -246.96438598632812,
      "logps/ref_chosen": -55.66770935058594,
      "logps/ref_rejected": -77.33308410644531,
      "logps/rejected": -390.0438232421875,
      "loss": 0.9834,
      "margin_dpo/margin_mean": 121.4140625,
      "margin_dpo/margin_std": 138.95596313476562,
      "step": 274
    },
    {
      "epoch": 0.40381791483113066,
      "fcm_dpo/beta": 0.0048539480194449425,
      "fcm_dpo/delta": 0.006926249712705612,
      "fcm_dpo/margin": 122.2330551147461,
      "fcm_dpo/q_t": 0.3715580701828003,
      "grad_norm": 31.085887908935547,
      "learning_rate": 3.738869612786737e-07,
      "logits/chosen": -0.3700248599052429,
      "logits/rejected": -0.3734614849090576,
      "logps/chosen": -204.39918518066406,
      "logps/ref_chosen": -48.594703674316406,
      "logps/ref_rejected": -93.30369567871094,
      "logps/rejected": -371.34124755859375,
      "loss": 0.995,
      "margin_dpo/margin_mean": 122.2330551147461,
      "margin_dpo/margin_std": 148.9960479736328,
      "step": 275
    },
    {
      "epoch": 0.4052863436123348,
      "fcm_dpo/beta": 0.004785975441336632,
      "fcm_dpo/delta": -0.05713774263858795,
      "fcm_dpo/margin": 136.44924926757812,
      "fcm_dpo/q_t": 0.3614548146724701,
      "grad_norm": 38.087745666503906,
      "learning_rate": 3.7277065802070204e-07,
      "logits/chosen": -0.3316674828529358,
      "logits/rejected": -0.3020857274532318,
      "logps/chosen": -219.54293823242188,
      "logps/ref_chosen": -56.57740783691406,
      "logps/ref_rejected": -70.36566925048828,
      "logps/rejected": -369.78045654296875,
      "loss": 0.9619,
      "margin_dpo/margin_mean": 136.44924926757812,
      "margin_dpo/margin_std": 161.114990234375,
      "step": 276
    },
    {
      "epoch": 0.4067547723935389,
      "fcm_dpo/beta": 0.0047689881175756454,
      "fcm_dpo/delta": -0.04522915184497833,
      "fcm_dpo/margin": 134.67898559570312,
      "fcm_dpo/q_t": 0.362157940864563,
      "grad_norm": 36.099918365478516,
      "learning_rate": 3.71651119641714e-07,
      "logits/chosen": -0.3248156011104584,
      "logits/rejected": -0.3068217635154724,
      "logps/chosen": -242.65780639648438,
      "logps/ref_chosen": -56.27156066894531,
      "logps/ref_rejected": -92.88127136230469,
      "logps/rejected": -413.946533203125,
      "loss": 0.9632,
      "margin_dpo/margin_mean": 134.67898559570312,
      "margin_dpo/margin_std": 155.36151123046875,
      "step": 277
    },
    {
      "epoch": 0.40822320117474303,
      "fcm_dpo/beta": 0.004655085504055023,
      "fcm_dpo/delta": -0.10972905904054642,
      "fcm_dpo/margin": 150.7223358154297,
      "fcm_dpo/q_t": 0.3516245484352112,
      "grad_norm": 29.135597229003906,
      "learning_rate": 3.705283756425872e-07,
      "logits/chosen": -0.3341606557369232,
      "logits/rejected": -0.336972713470459,
      "logps/chosen": -221.07388305664062,
      "logps/ref_chosen": -52.94194030761719,
      "logps/ref_rejected": -91.25357818603516,
      "logps/rejected": -410.10784912109375,
      "loss": 0.9254,
      "margin_dpo/margin_mean": 150.72235107421875,
      "margin_dpo/margin_std": 164.86074829101562,
      "step": 278
    },
    {
      "epoch": 0.40969162995594716,
      "fcm_dpo/beta": 0.004513373598456383,
      "fcm_dpo/delta": -0.13143345713615417,
      "fcm_dpo/margin": 159.43524169921875,
      "fcm_dpo/q_t": 0.35075873136520386,
      "grad_norm": 37.86472702026367,
      "learning_rate": 3.6940245560867e-07,
      "logits/chosen": -0.2502022981643677,
      "logits/rejected": -0.24658802151679993,
      "logps/chosen": -250.07838439941406,
      "logps/ref_chosen": -48.641319274902344,
      "logps/ref_rejected": -87.8514404296875,
      "logps/rejected": -448.7237548828125,
      "loss": 0.9479,
      "margin_dpo/margin_mean": 159.4352569580078,
      "margin_dpo/margin_std": 191.3142852783203,
      "step": 279
    },
    {
      "epoch": 0.4111600587371512,
      "fcm_dpo/beta": 0.00445135310292244,
      "fcm_dpo/delta": -0.09706270694732666,
      "fcm_dpo/margin": 155.06472778320312,
      "fcm_dpo/q_t": 0.3463428020477295,
      "grad_norm": 40.227508544921875,
      "learning_rate": 3.6827338920900253e-07,
      "logits/chosen": -0.28654709458351135,
      "logits/rejected": -0.28727859258651733,
      "logps/chosen": -259.53009033203125,
      "logps/ref_chosen": -58.797122955322266,
      "logps/ref_rejected": -98.61885070800781,
      "logps/rejected": -454.41656494140625,
      "loss": 0.9052,
      "margin_dpo/margin_mean": 155.0647430419922,
      "margin_dpo/margin_std": 146.40179443359375,
      "step": 280
    },
    {
      "epoch": 0.41262848751835535,
      "fcm_dpo/beta": 0.004433557391166687,
      "fcm_dpo/delta": -0.03812451288104057,
      "fcm_dpo/margin": 143.29185485839844,
      "fcm_dpo/q_t": 0.3562266230583191,
      "grad_norm": 30.542402267456055,
      "learning_rate": 3.6714120619553435e-07,
      "logits/chosen": -0.34395909309387207,
      "logits/rejected": -0.3141595721244812,
      "logps/chosen": -239.7939453125,
      "logps/ref_chosen": -55.488521575927734,
      "logps/ref_rejected": -80.88258361816406,
      "logps/rejected": -408.4798583984375,
      "loss": 0.9469,
      "margin_dpo/margin_mean": 143.29185485839844,
      "margin_dpo/margin_std": 147.6947021484375,
      "step": 281
    },
    {
      "epoch": 0.41409691629955947,
      "fcm_dpo/beta": 0.004496179986745119,
      "fcm_dpo/delta": 0.11047537624835968,
      "fcm_dpo/margin": 109.85005187988281,
      "fcm_dpo/q_t": 0.39475515484809875,
      "grad_norm": 32.4976806640625,
      "learning_rate": 3.660059364023408e-07,
      "logits/chosen": -0.40445345640182495,
      "logits/rejected": -0.37312695384025574,
      "logps/chosen": -266.0508728027344,
      "logps/ref_chosen": -73.07014465332031,
      "logps/ref_rejected": -95.35098266601562,
      "logps/rejected": -398.1817626953125,
      "loss": 1.0544,
      "margin_dpo/margin_mean": 109.85005187988281,
      "margin_dpo/margin_std": 156.09814453125,
      "step": 282
    },
    {
      "epoch": 0.4155653450807636,
      "fcm_dpo/beta": 0.004433175548911095,
      "fcm_dpo/delta": -0.12632399797439575,
      "fcm_dpo/margin": 161.82952880859375,
      "fcm_dpo/q_t": 0.341400146484375,
      "grad_norm": 44.17348098754883,
      "learning_rate": 3.6486760974483685e-07,
      "logits/chosen": -0.3220086991786957,
      "logits/rejected": -0.3167068660259247,
      "logps/chosen": -252.05734252929688,
      "logps/ref_chosen": -61.89844512939453,
      "logps/ref_rejected": -96.98655700683594,
      "logps/rejected": -448.9749755859375,
      "loss": 0.8904,
      "margin_dpo/margin_mean": 161.8295135498047,
      "margin_dpo/margin_std": 149.40249633789062,
      "step": 283
    },
    {
      "epoch": 0.4170337738619677,
      "fcm_dpo/beta": 0.004331875126808882,
      "fcm_dpo/delta": -0.0721510797739029,
      "fcm_dpo/margin": 154.01272583007812,
      "fcm_dpo/q_t": 0.35365206003189087,
      "grad_norm": 28.796356201171875,
      "learning_rate": 3.6372625621898863e-07,
      "logits/chosen": -0.39249300956726074,
      "logits/rejected": -0.37775149941444397,
      "logps/chosen": -239.20748901367188,
      "logps/ref_chosen": -58.4355354309082,
      "logps/ref_rejected": -93.46926879882812,
      "logps/rejected": -428.2539367675781,
      "loss": 0.9158,
      "margin_dpo/margin_mean": 154.01272583007812,
      "margin_dpo/margin_std": 150.7726593017578,
      "step": 284
    },
    {
      "epoch": 0.4185022026431718,
      "fcm_dpo/beta": 0.004361903294920921,
      "fcm_dpo/delta": 0.014079738408327103,
      "fcm_dpo/margin": 134.28880310058594,
      "fcm_dpo/q_t": 0.3673364818096161,
      "grad_norm": 34.644283294677734,
      "learning_rate": 3.625819059005228e-07,
      "logits/chosen": -0.3506457209587097,
      "logits/rejected": -0.32794976234436035,
      "logps/chosen": -276.463134765625,
      "logps/ref_chosen": -66.23219299316406,
      "logps/ref_rejected": -99.1268310546875,
      "logps/rejected": -443.6466064453125,
      "loss": 0.9694,
      "margin_dpo/margin_mean": 134.28878784179688,
      "margin_dpo/margin_std": 137.147216796875,
      "step": 285
    },
    {
      "epoch": 0.4199706314243759,
      "fcm_dpo/beta": 0.004304747097194195,
      "fcm_dpo/delta": -0.060156404972076416,
      "fcm_dpo/margin": 152.447265625,
      "fcm_dpo/q_t": 0.3558097183704376,
      "grad_norm": 35.37562561035156,
      "learning_rate": 3.614345889441346e-07,
      "logits/chosen": -0.3511343002319336,
      "logits/rejected": -0.32436245679855347,
      "logps/chosen": -274.97412109375,
      "logps/ref_chosen": -72.95100402832031,
      "logps/ref_rejected": -88.58845520019531,
      "logps/rejected": -443.058837890625,
      "loss": 0.9432,
      "margin_dpo/margin_mean": 152.447265625,
      "margin_dpo/margin_std": 162.77908325195312,
      "step": 286
    },
    {
      "epoch": 0.42143906020558003,
      "fcm_dpo/beta": 0.0043206894770264626,
      "fcm_dpo/delta": 0.0441654808819294,
      "fcm_dpo/margin": 129.17588806152344,
      "fcm_dpo/q_t": 0.3783441185951233,
      "grad_norm": 31.608234405517578,
      "learning_rate": 3.6028433558269275e-07,
      "logits/chosen": -0.31515616178512573,
      "logits/rejected": -0.28630921244621277,
      "logps/chosen": -264.4310607910156,
      "logps/ref_chosen": -61.54115295410156,
      "logps/ref_rejected": -77.69607543945312,
      "logps/rejected": -409.7618408203125,
      "loss": 1.0024,
      "margin_dpo/margin_mean": 129.17587280273438,
      "margin_dpo/margin_std": 156.08370971679688,
      "step": 287
    },
    {
      "epoch": 0.42290748898678415,
      "fcm_dpo/beta": 0.00423096539452672,
      "fcm_dpo/delta": -0.06944943964481354,
      "fcm_dpo/margin": 156.4290771484375,
      "fcm_dpo/q_t": 0.3512691259384155,
      "grad_norm": 27.641883850097656,
      "learning_rate": 3.5913117612644327e-07,
      "logits/chosen": -0.32602715492248535,
      "logits/rejected": -0.3050415515899658,
      "logps/chosen": -260.18255615234375,
      "logps/ref_chosen": -56.661224365234375,
      "logps/ref_rejected": -87.33570098876953,
      "logps/rejected": -447.2861022949219,
      "loss": 0.914,
      "margin_dpo/margin_mean": 156.4290771484375,
      "margin_dpo/margin_std": 137.47991943359375,
      "step": 288
    },
    {
      "epoch": 0.4243759177679883,
      "fcm_dpo/beta": 0.0041911243461072445,
      "fcm_dpo/delta": -0.14678727090358734,
      "fcm_dpo/margin": 175.65328979492188,
      "fcm_dpo/q_t": 0.34263110160827637,
      "grad_norm": 36.062095642089844,
      "learning_rate": 3.5797514096221024e-07,
      "logits/chosen": -0.26659348607063293,
      "logits/rejected": -0.25999683141708374,
      "logps/chosen": -253.7037353515625,
      "logps/ref_chosen": -45.23039245605469,
      "logps/ref_rejected": -87.64266967773438,
      "logps/rejected": -471.769287109375,
      "loss": 0.8987,
      "margin_dpo/margin_mean": 175.65328979492188,
      "margin_dpo/margin_std": 176.63510131835938,
      "step": 289
    },
    {
      "epoch": 0.42584434654919234,
      "fcm_dpo/beta": 0.004051923286169767,
      "fcm_dpo/delta": -0.12010594457387924,
      "fcm_dpo/margin": 175.46475219726562,
      "fcm_dpo/q_t": 0.35006120800971985,
      "grad_norm": 32.5576286315918,
      "learning_rate": 3.568162605525952e-07,
      "logits/chosen": -0.24551498889923096,
      "logits/rejected": -0.262775182723999,
      "logps/chosen": -283.1232604980469,
      "logps/ref_chosen": -55.47149658203125,
      "logps/ref_rejected": -116.70857238769531,
      "logps/rejected": -519.8250732421875,
      "loss": 0.9297,
      "margin_dpo/margin_mean": 175.46475219726562,
      "margin_dpo/margin_std": 197.48675537109375,
      "step": 290
    },
    {
      "epoch": 0.42731277533039647,
      "fcm_dpo/beta": 0.004028604365885258,
      "fcm_dpo/delta": -0.06118813902139664,
      "fcm_dpo/margin": 163.03326416015625,
      "fcm_dpo/q_t": 0.35449501872062683,
      "grad_norm": 26.545326232910156,
      "learning_rate": 3.5565456543517485e-07,
      "logits/chosen": -0.30797719955444336,
      "logits/rejected": -0.2923903167247772,
      "logps/chosen": -260.06689453125,
      "logps/ref_chosen": -63.26036834716797,
      "logps/ref_rejected": -89.29708862304688,
      "logps/rejected": -449.1368713378906,
      "loss": 0.9354,
      "margin_dpo/margin_mean": 163.03326416015625,
      "margin_dpo/margin_std": 164.8643798828125,
      "step": 291
    },
    {
      "epoch": 0.4287812041116006,
      "fcm_dpo/beta": 0.003977569285780191,
      "fcm_dpo/delta": -0.05237273871898651,
      "fcm_dpo/margin": 163.1212615966797,
      "fcm_dpo/q_t": 0.3588453233242035,
      "grad_norm": 34.14498519897461,
      "learning_rate": 3.5449008622169583e-07,
      "logits/chosen": -0.262368768453598,
      "logits/rejected": -0.23994597792625427,
      "logps/chosen": -277.8984375,
      "logps/ref_chosen": -53.91852951049805,
      "logps/ref_rejected": -89.96138000488281,
      "logps/rejected": -477.06256103515625,
      "loss": 0.9758,
      "margin_dpo/margin_mean": 163.12124633789062,
      "margin_dpo/margin_std": 198.7785186767578,
      "step": 292
    },
    {
      "epoch": 0.4302496328928047,
      "fcm_dpo/beta": 0.004008471965789795,
      "fcm_dpo/delta": 0.07511234283447266,
      "fcm_dpo/margin": 131.8068389892578,
      "fcm_dpo/q_t": 0.38762548565864563,
      "grad_norm": 37.924625396728516,
      "learning_rate": 3.5332285359726846e-07,
      "logits/chosen": -0.3276376724243164,
      "logits/rejected": -0.3103697896003723,
      "logps/chosen": -269.2470703125,
      "logps/ref_chosen": -60.376033782958984,
      "logps/ref_rejected": -77.85244750976562,
      "logps/rejected": -418.53033447265625,
      "loss": 1.0488,
      "margin_dpo/margin_mean": 131.8068389892578,
      "margin_dpo/margin_std": 185.1345672607422,
      "step": 293
    },
    {
      "epoch": 0.43171806167400884,
      "fcm_dpo/beta": 0.0040005166083574295,
      "fcm_dpo/delta": 0.008330628275871277,
      "fcm_dpo/margin": 147.9698486328125,
      "fcm_dpo/q_t": 0.37461739778518677,
      "grad_norm": 28.923660278320312,
      "learning_rate": 3.5215289831955786e-07,
      "logits/chosen": -0.293660044670105,
      "logits/rejected": -0.2946998178958893,
      "logps/chosen": -237.821533203125,
      "logps/ref_chosen": -48.0875358581543,
      "logps/ref_rejected": -81.89698791503906,
      "logps/rejected": -419.600830078125,
      "loss": 0.9972,
      "margin_dpo/margin_mean": 147.9698486328125,
      "margin_dpo/margin_std": 184.6291961669922,
      "step": 294
    },
    {
      "epoch": 0.4331864904552129,
      "fcm_dpo/beta": 0.0040347035974264145,
      "fcm_dpo/delta": 0.02709888480603695,
      "fcm_dpo/margin": 142.37374877929688,
      "fcm_dpo/q_t": 0.3751312494277954,
      "grad_norm": 33.199710845947266,
      "learning_rate": 3.509802512179737e-07,
      "logits/chosen": -0.3318241834640503,
      "logits/rejected": -0.3298850655555725,
      "logps/chosen": -268.1130065917969,
      "logps/ref_chosen": -49.92467498779297,
      "logps/ref_rejected": -87.45632934570312,
      "logps/rejected": -448.01837158203125,
      "loss": 1.0026,
      "margin_dpo/margin_mean": 142.3737335205078,
      "margin_dpo/margin_std": 173.57199096679688,
      "step": 295
    },
    {
      "epoch": 0.434654919236417,
      "fcm_dpo/beta": 0.0041085234843194485,
      "fcm_dpo/delta": 0.1538042277097702,
      "fcm_dpo/margin": 110.34952545166016,
      "fcm_dpo/q_t": 0.40237492322921753,
      "grad_norm": 50.468746185302734,
      "learning_rate": 3.498049431928577e-07,
      "logits/chosen": -0.38077211380004883,
      "logits/rejected": -0.3572128415107727,
      "logps/chosen": -313.87554931640625,
      "logps/ref_chosen": -65.49124145507812,
      "logps/ref_rejected": -93.08908081054688,
      "logps/rejected": -451.82293701171875,
      "loss": 1.1194,
      "margin_dpo/margin_mean": 110.34951782226562,
      "margin_dpo/margin_std": 187.090576171875,
      "step": 296
    },
    {
      "epoch": 0.43612334801762115,
      "fcm_dpo/beta": 0.004196628928184509,
      "fcm_dpo/delta": 0.063119076192379,
      "fcm_dpo/margin": 128.75506591796875,
      "fcm_dpo/q_t": 0.3794586956501007,
      "grad_norm": 28.69700813293457,
      "learning_rate": 3.486270052146694e-07,
      "logits/chosen": -0.3626411557197571,
      "logits/rejected": -0.3594970703125,
      "logps/chosen": -282.9952697753906,
      "logps/ref_chosen": -56.476951599121094,
      "logps/ref_rejected": -95.1385498046875,
      "logps/rejected": -450.41192626953125,
      "loss": 1.0008,
      "margin_dpo/margin_mean": 128.7550506591797,
      "margin_dpo/margin_std": 144.84683227539062,
      "step": 297
    },
    {
      "epoch": 0.43759177679882527,
      "fcm_dpo/beta": 0.004173451103270054,
      "fcm_dpo/delta": -0.09957937896251678,
      "fcm_dpo/margin": 165.98265075683594,
      "fcm_dpo/q_t": 0.3572956323623657,
      "grad_norm": 29.900131225585938,
      "learning_rate": 3.474464683231698e-07,
      "logits/chosen": -0.4068795442581177,
      "logits/rejected": -0.42597341537475586,
      "logps/chosen": -299.30560302734375,
      "logps/ref_chosen": -67.32516479492188,
      "logps/ref_rejected": -116.66217041015625,
      "logps/rejected": -514.625244140625,
      "loss": 0.954,
      "margin_dpo/margin_mean": 165.98265075683594,
      "margin_dpo/margin_std": 202.80752563476562,
      "step": 298
    },
    {
      "epoch": 0.4390602055800294,
      "fcm_dpo/beta": 0.00413914117962122,
      "fcm_dpo/delta": -0.04581880569458008,
      "fcm_dpo/margin": 155.09854125976562,
      "fcm_dpo/q_t": 0.3642904758453369,
      "grad_norm": 38.76698684692383,
      "learning_rate": 3.462633636266041e-07,
      "logits/chosen": -0.3764153718948364,
      "logits/rejected": -0.3700462579727173,
      "logps/chosen": -246.14369201660156,
      "logps/ref_chosen": -48.96209716796875,
      "logps/ref_rejected": -84.32823944091797,
      "logps/rejected": -436.6083984375,
      "loss": 0.9702,
      "margin_dpo/margin_mean": 155.0985565185547,
      "margin_dpo/margin_std": 182.65501403808594,
      "step": 299
    },
    {
      "epoch": 0.44052863436123346,
      "fcm_dpo/beta": 0.004044054076075554,
      "fcm_dpo/delta": -0.1144830510020256,
      "fcm_dpo/margin": 174.70237731933594,
      "fcm_dpo/q_t": 0.3509829640388489,
      "grad_norm": 42.70845413208008,
      "learning_rate": 3.4507772230088147e-07,
      "logits/chosen": -0.3672061562538147,
      "logits/rejected": -0.36297306418418884,
      "logps/chosen": -328.4141845703125,
      "logps/ref_chosen": -59.07371139526367,
      "logps/ref_rejected": -95.9664535522461,
      "logps/rejected": -540.00927734375,
      "loss": 0.9647,
      "margin_dpo/margin_mean": 174.70237731933594,
      "margin_dpo/margin_std": 214.1061248779297,
      "step": 300
    },
    {
      "epoch": 0.4419970631424376,
      "fcm_dpo/beta": 0.003927802667021751,
      "fcm_dpo/delta": -0.099482960999012,
      "fcm_dpo/margin": 176.13919067382812,
      "fcm_dpo/q_t": 0.3530727028846741,
      "grad_norm": 30.08980369567871,
      "learning_rate": 3.4388957558875316e-07,
      "logits/chosen": -0.40564534068107605,
      "logits/rejected": -0.3970520496368408,
      "logps/chosen": -284.28485107421875,
      "logps/ref_chosen": -57.249366760253906,
      "logps/ref_rejected": -92.35354614257812,
      "logps/rejected": -495.5281982421875,
      "loss": 0.9267,
      "margin_dpo/margin_mean": 176.1392059326172,
      "margin_dpo/margin_std": 191.83755493164062,
      "step": 301
    },
    {
      "epoch": 0.4434654919236417,
      "fcm_dpo/beta": 0.0039380136877298355,
      "fcm_dpo/delta": 0.05164004862308502,
      "fcm_dpo/margin": 139.933349609375,
      "fcm_dpo/q_t": 0.3805837631225586,
      "grad_norm": 39.73493957519531,
      "learning_rate": 3.426989547989902e-07,
      "logits/chosen": -0.4200012683868408,
      "logits/rejected": -0.4234361946582794,
      "logps/chosen": -265.1363830566406,
      "logps/ref_chosen": -51.197994232177734,
      "logps/ref_rejected": -97.22636413574219,
      "logps/rejected": -451.0981140136719,
      "loss": 1.0152,
      "margin_dpo/margin_mean": 139.933349609375,
      "margin_dpo/margin_std": 175.3145751953125,
      "step": 302
    },
    {
      "epoch": 0.44493392070484583,
      "fcm_dpo/beta": 0.003999094944447279,
      "fcm_dpo/delta": 0.06958974152803421,
      "fcm_dpo/margin": 133.57589721679688,
      "fcm_dpo/q_t": 0.386226624250412,
      "grad_norm": 40.41800308227539,
      "learning_rate": 3.4150589130555773e-07,
      "logits/chosen": -0.41884028911590576,
      "logits/rejected": -0.39589619636535645,
      "logps/chosen": -288.8157958984375,
      "logps/ref_chosen": -66.71394348144531,
      "logps/ref_rejected": -86.94542694091797,
      "logps/rejected": -442.6231994628906,
      "loss": 1.0405,
      "margin_dpo/margin_mean": 133.57591247558594,
      "margin_dpo/margin_std": 183.11703491210938,
      "step": 303
    },
    {
      "epoch": 0.44640234948604995,
      "fcm_dpo/beta": 0.0040207491256296635,
      "fcm_dpo/delta": -0.03515169024467468,
      "fcm_dpo/margin": 157.3292694091797,
      "fcm_dpo/q_t": 0.35517168045043945,
      "grad_norm": 49.5794563293457,
      "learning_rate": 3.403104165467883e-07,
      "logits/chosen": -0.42296552658081055,
      "logits/rejected": -0.4050954580307007,
      "logps/chosen": -266.1893310546875,
      "logps/ref_chosen": -71.95069885253906,
      "logps/ref_rejected": -90.47203063964844,
      "logps/rejected": -442.0399169921875,
      "loss": 0.9141,
      "margin_dpo/margin_mean": 157.32925415039062,
      "margin_dpo/margin_std": 126.93194580078125,
      "step": 304
    },
    {
      "epoch": 0.447870778267254,
      "fcm_dpo/beta": 0.003973768092691898,
      "fcm_dpo/delta": 0.022460877895355225,
      "fcm_dpo/margin": 145.38482666015625,
      "fcm_dpo/q_t": 0.3763822913169861,
      "grad_norm": 28.328157424926758,
      "learning_rate": 3.391125620245535e-07,
      "logits/chosen": -0.42863526940345764,
      "logits/rejected": -0.40163588523864746,
      "logps/chosen": -273.1666259765625,
      "logps/ref_chosen": -66.79523468017578,
      "logps/ref_rejected": -92.75459289550781,
      "logps/rejected": -444.51080322265625,
      "loss": 1.0001,
      "margin_dpo/margin_mean": 145.38482666015625,
      "margin_dpo/margin_std": 177.77105712890625,
      "step": 305
    },
    {
      "epoch": 0.44933920704845814,
      "fcm_dpo/beta": 0.004037954844534397,
      "fcm_dpo/delta": 0.03192281723022461,
      "fcm_dpo/margin": 141.09954833984375,
      "fcm_dpo/q_t": 0.37558555603027344,
      "grad_norm": 32.04235076904297,
      "learning_rate": 3.3791235930343417e-07,
      "logits/chosen": -0.3904664218425751,
      "logits/rejected": -0.3591429889202118,
      "logps/chosen": -269.64520263671875,
      "logps/ref_chosen": -69.68389892578125,
      "logps/ref_rejected": -85.15919494628906,
      "logps/rejected": -426.2200622558594,
      "loss": 0.9881,
      "margin_dpo/margin_mean": 141.09954833984375,
      "margin_dpo/margin_std": 163.01104736328125,
      "step": 306
    },
    {
      "epoch": 0.45080763582966227,
      "fcm_dpo/beta": 0.004054022952914238,
      "fcm_dpo/delta": 0.060959137976169586,
      "fcm_dpo/margin": 133.7398681640625,
      "fcm_dpo/q_t": 0.3805789649486542,
      "grad_norm": 42.768367767333984,
      "learning_rate": 3.367098400098881e-07,
      "logits/chosen": -0.36754947900772095,
      "logits/rejected": -0.3443043828010559,
      "logps/chosen": -266.11358642578125,
      "logps/ref_chosen": -70.16542053222656,
      "logps/ref_rejected": -86.97230529785156,
      "logps/rejected": -416.6603698730469,
      "loss": 1.0175,
      "margin_dpo/margin_mean": 133.7398681640625,
      "margin_dpo/margin_std": 165.68634033203125,
      "step": 307
    },
    {
      "epoch": 0.4522760646108664,
      "fcm_dpo/beta": 0.004061142913997173,
      "fcm_dpo/delta": -0.046599872410297394,
      "fcm_dpo/margin": 158.4833526611328,
      "fcm_dpo/q_t": 0.3559141755104065,
      "grad_norm": 34.26539993286133,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": -0.3417690396308899,
      "logits/rejected": -0.32422229647636414,
      "logps/chosen": -250.93490600585938,
      "logps/ref_chosen": -55.2449951171875,
      "logps/ref_rejected": -79.37226104736328,
      "logps/rejected": -433.5455017089844,
      "loss": 0.9279,
      "margin_dpo/margin_mean": 158.48333740234375,
      "margin_dpo/margin_std": 150.41107177734375,
      "step": 308
    },
    {
      "epoch": 0.45374449339207046,
      "fcm_dpo/beta": 0.004053506068885326,
      "fcm_dpo/delta": -0.04356030374765396,
      "fcm_dpo/margin": 157.86776733398438,
      "fcm_dpo/q_t": 0.35849449038505554,
      "grad_norm": 36.413551330566406,
      "learning_rate": 3.3429797851573183e-07,
      "logits/chosen": -0.3285176157951355,
      "logits/rejected": -0.31304115056991577,
      "logps/chosen": -240.2936248779297,
      "logps/ref_chosen": -48.959083557128906,
      "logps/ref_rejected": -82.34072875976562,
      "logps/rejected": -431.54302978515625,
      "loss": 0.9503,
      "margin_dpo/margin_mean": 157.86776733398438,
      "margin_dpo/margin_std": 165.03907775878906,
      "step": 309
    },
    {
      "epoch": 0.4552129221732746,
      "fcm_dpo/beta": 0.004054386168718338,
      "fcm_dpo/delta": 0.03481416404247284,
      "fcm_dpo/margin": 139.72027587890625,
      "fcm_dpo/q_t": 0.3711395859718323,
      "grad_norm": 30.295249938964844,
      "learning_rate": 3.3308869986991487e-07,
      "logits/chosen": -0.35680243372917175,
      "logits/rejected": -0.33559316396713257,
      "logps/chosen": -279.739501953125,
      "logps/ref_chosen": -62.74177932739258,
      "logps/ref_rejected": -79.9302978515625,
      "logps/rejected": -436.6482849121094,
      "loss": 0.9662,
      "margin_dpo/margin_mean": 139.72027587890625,
      "margin_dpo/margin_std": 132.34963989257812,
      "step": 310
    },
    {
      "epoch": 0.4566813509544787,
      "fcm_dpo/beta": 0.003984658978879452,
      "fcm_dpo/delta": -0.08475086092948914,
      "fcm_dpo/margin": 170.3639373779297,
      "fcm_dpo/q_t": 0.3561347723007202,
      "grad_norm": 27.640430450439453,
      "learning_rate": 3.3187723175958346e-07,
      "logits/chosen": -0.33530759811401367,
      "logits/rejected": -0.30850982666015625,
      "logps/chosen": -278.0128479003906,
      "logps/ref_chosen": -53.02798080444336,
      "logps/ref_rejected": -77.43820190429688,
      "logps/rejected": -472.7869873046875,
      "loss": 0.9432,
      "margin_dpo/margin_mean": 170.3639373779297,
      "margin_dpo/margin_std": 193.53854370117188,
      "step": 311
    },
    {
      "epoch": 0.4581497797356828,
      "fcm_dpo/beta": 0.003978079650551081,
      "fcm_dpo/delta": -0.023921042680740356,
      "fcm_dpo/margin": 156.37979125976562,
      "fcm_dpo/q_t": 0.36489611864089966,
      "grad_norm": 29.57942008972168,
      "learning_rate": 3.306636061080487e-07,
      "logits/chosen": -0.27948933839797974,
      "logits/rejected": -0.261229932308197,
      "logps/chosen": -254.8425750732422,
      "logps/ref_chosen": -49.39221954345703,
      "logps/ref_rejected": -75.79280853271484,
      "logps/rejected": -437.62298583984375,
      "loss": 0.9616,
      "margin_dpo/margin_mean": 156.37979125976562,
      "margin_dpo/margin_std": 174.12229919433594,
      "step": 312
    },
    {
      "epoch": 0.45961820851688695,
      "fcm_dpo/beta": 0.0040033310651779175,
      "fcm_dpo/delta": 0.01650937646627426,
      "fcm_dpo/margin": 145.47708129882812,
      "fcm_dpo/q_t": 0.3724866807460785,
      "grad_norm": 33.476741790771484,
      "learning_rate": 3.2944785489547537e-07,
      "logits/chosen": -0.3220197558403015,
      "logits/rejected": -0.3162776529788971,
      "logps/chosen": -249.13385009765625,
      "logps/ref_chosen": -50.152740478515625,
      "logps/ref_rejected": -86.40620422363281,
      "logps/rejected": -430.8643798828125,
      "loss": 1.0053,
      "margin_dpo/margin_mean": 145.47706604003906,
      "margin_dpo/margin_std": 175.97396850585938,
      "step": 313
    },
    {
      "epoch": 0.461086637298091,
      "fcm_dpo/beta": 0.0039779325015842915,
      "fcm_dpo/delta": 0.053729020059108734,
      "fcm_dpo/margin": 137.9976806640625,
      "fcm_dpo/q_t": 0.38239920139312744,
      "grad_norm": 28.024948120117188,
      "learning_rate": 3.2823001015803857e-07,
      "logits/chosen": -0.37483978271484375,
      "logits/rejected": -0.37171846628189087,
      "logps/chosen": -271.1210632324219,
      "logps/ref_chosen": -57.237579345703125,
      "logps/ref_rejected": -97.5965347290039,
      "logps/rejected": -449.4776611328125,
      "loss": 1.0322,
      "margin_dpo/margin_mean": 137.99769592285156,
      "margin_dpo/margin_std": 185.14695739746094,
      "step": 314
    },
    {
      "epoch": 0.46255506607929514,
      "fcm_dpo/beta": 0.004043739754706621,
      "fcm_dpo/delta": 0.06402629613876343,
      "fcm_dpo/margin": 133.40902709960938,
      "fcm_dpo/q_t": 0.3819577395915985,
      "grad_norm": 34.34666442871094,
      "learning_rate": 3.270101039870797e-07,
      "logits/chosen": -0.25970029830932617,
      "logits/rejected": -0.2599654793739319,
      "logps/chosen": -258.28570556640625,
      "logps/ref_chosen": -49.06958770751953,
      "logps/ref_rejected": -85.68087768554688,
      "logps/rejected": -428.30596923828125,
      "loss": 1.0081,
      "margin_dpo/margin_mean": 133.40902709960938,
      "margin_dpo/margin_std": 158.57159423828125,
      "step": 315
    },
    {
      "epoch": 0.46402349486049926,
      "fcm_dpo/beta": 0.0039628720842301846,
      "fcm_dpo/delta": -0.11757272481918335,
      "fcm_dpo/margin": 178.52488708496094,
      "fcm_dpo/q_t": 0.34840020537376404,
      "grad_norm": 29.264596939086914,
      "learning_rate": 3.2578816852826086e-07,
      "logits/chosen": -0.30190056562423706,
      "logits/rejected": -0.3046305775642395,
      "logps/chosen": -262.7852783203125,
      "logps/ref_chosen": -54.26074981689453,
      "logps/ref_rejected": -101.2814712524414,
      "logps/rejected": -488.33087158203125,
      "loss": 0.9125,
      "margin_dpo/margin_mean": 178.52488708496094,
      "margin_dpo/margin_std": 183.82940673828125,
      "step": 316
    },
    {
      "epoch": 0.4654919236417034,
      "fcm_dpo/beta": 0.0038559194654226303,
      "fcm_dpo/delta": -0.21507573127746582,
      "fcm_dpo/margin": 207.04443359375,
      "fcm_dpo/q_t": 0.32492557168006897,
      "grad_norm": 30.406768798828125,
      "learning_rate": 3.2456423598071783e-07,
      "logits/chosen": -0.3299999237060547,
      "logits/rejected": -0.3157616853713989,
      "logps/chosen": -272.45648193359375,
      "logps/ref_chosen": -56.094207763671875,
      "logps/ref_rejected": -100.69905090332031,
      "logps/rejected": -524.105712890625,
      "loss": 0.8408,
      "margin_dpo/margin_mean": 207.04443359375,
      "margin_dpo/margin_std": 174.73843383789062,
      "step": 317
    },
    {
      "epoch": 0.4669603524229075,
      "fcm_dpo/beta": 0.0038027470000088215,
      "fcm_dpo/delta": -0.02990717813372612,
      "fcm_dpo/margin": 165.0573272705078,
      "fcm_dpo/q_t": 0.3614851236343384,
      "grad_norm": 39.84989929199219,
      "learning_rate": 3.233383385962115e-07,
      "logits/chosen": -0.39633870124816895,
      "logits/rejected": -0.3627668023109436,
      "logps/chosen": -288.56982421875,
      "logps/ref_chosen": -64.64569854736328,
      "logps/ref_rejected": -82.76425170898438,
      "logps/rejected": -471.7456970214844,
      "loss": 0.9456,
      "margin_dpo/margin_mean": 165.0573272705078,
      "margin_dpo/margin_std": 168.24526977539062,
      "step": 318
    },
    {
      "epoch": 0.4684287812041116,
      "fcm_dpo/beta": 0.0037455919664353132,
      "fcm_dpo/delta": -0.06087436154484749,
      "fcm_dpo/margin": 175.37954711914062,
      "fcm_dpo/q_t": 0.35514286160469055,
      "grad_norm": 32.62771987915039,
      "learning_rate": 3.2211050867827805e-07,
      "logits/chosen": -0.32384127378463745,
      "logits/rejected": -0.33828234672546387,
      "logps/chosen": -260.467529296875,
      "logps/ref_chosen": -49.383758544921875,
      "logps/ref_rejected": -113.90650939941406,
      "logps/rejected": -500.3698425292969,
      "loss": 0.9384,
      "margin_dpo/margin_mean": 175.37954711914062,
      "margin_dpo/margin_std": 184.76806640625,
      "step": 319
    },
    {
      "epoch": 0.4698972099853157,
      "fcm_dpo/beta": 0.0036399452947080135,
      "fcm_dpo/delta": -0.14344662427902222,
      "fcm_dpo/margin": 201.1544189453125,
      "fcm_dpo/q_t": 0.3414286971092224,
      "grad_norm": 35.252044677734375,
      "learning_rate": 3.208807785813777e-07,
      "logits/chosen": -0.34068095684051514,
      "logits/rejected": -0.34001022577285767,
      "logps/chosen": -272.8780212402344,
      "logps/ref_chosen": -59.50489044189453,
      "logps/ref_rejected": -97.66717529296875,
      "logps/rejected": -512.1947021484375,
      "loss": 0.8862,
      "margin_dpo/margin_mean": 201.1544189453125,
      "margin_dpo/margin_std": 194.9528045654297,
      "step": 320
    },
    {
      "epoch": 0.4713656387665198,
      "fcm_dpo/beta": 0.0035794866271317005,
      "fcm_dpo/delta": -0.025970421731472015,
      "fcm_dpo/margin": 173.90859985351562,
      "fcm_dpo/q_t": 0.3651740550994873,
      "grad_norm": 39.09996795654297,
      "learning_rate": 3.1964918071004217e-07,
      "logits/chosen": -0.3051578104496002,
      "logits/rejected": -0.29157111048698425,
      "logps/chosen": -323.6648254394531,
      "logps/ref_chosen": -61.548683166503906,
      "logps/ref_rejected": -91.64103698730469,
      "logps/rejected": -527.665771484375,
      "loss": 0.9826,
      "margin_dpo/margin_mean": 173.90859985351562,
      "margin_dpo/margin_std": 204.27359008789062,
      "step": 321
    },
    {
      "epoch": 0.47283406754772395,
      "fcm_dpo/beta": 0.003550457302480936,
      "fcm_dpo/delta": -0.07277655601501465,
      "fcm_dpo/margin": 187.88018798828125,
      "fcm_dpo/q_t": 0.3536107838153839,
      "grad_norm": 34.10807800292969,
      "learning_rate": 3.184157475180207e-07,
      "logits/chosen": -0.3065683841705322,
      "logits/rejected": -0.300385057926178,
      "logps/chosen": -284.23797607421875,
      "logps/ref_chosen": -57.29003143310547,
      "logps/ref_rejected": -95.74992370605469,
      "logps/rejected": -510.5780334472656,
      "loss": 0.9286,
      "margin_dpo/margin_mean": 187.88018798828125,
      "margin_dpo/margin_std": 189.9258270263672,
      "step": 322
    },
    {
      "epoch": 0.47430249632892807,
      "fcm_dpo/beta": 0.003559872740879655,
      "fcm_dpo/delta": 0.03238154202699661,
      "fcm_dpo/margin": 159.92843627929688,
      "fcm_dpo/q_t": 0.374622106552124,
      "grad_norm": 46.58732604980469,
      "learning_rate": 3.171805115074251e-07,
      "logits/chosen": -0.3481108546257019,
      "logits/rejected": -0.343014121055603,
      "logps/chosen": -301.1619873046875,
      "logps/ref_chosen": -51.23395919799805,
      "logps/ref_rejected": -75.06192016601562,
      "logps/rejected": -484.91839599609375,
      "loss": 1.0045,
      "margin_dpo/margin_mean": 159.92843627929688,
      "margin_dpo/margin_std": 189.8189697265625,
      "step": 323
    },
    {
      "epoch": 0.47577092511013214,
      "fcm_dpo/beta": 0.003606216050684452,
      "fcm_dpo/delta": -0.016858771443367004,
      "fcm_dpo/margin": 170.2562713623047,
      "fcm_dpo/q_t": 0.3706227242946625,
      "grad_norm": 61.91301345825195,
      "learning_rate": 3.1594350522787295e-07,
      "logits/chosen": -0.3303983509540558,
      "logits/rejected": -0.3054184913635254,
      "logps/chosen": -341.5509033203125,
      "logps/ref_chosen": -65.13516998291016,
      "logps/ref_rejected": -86.47750854492188,
      "logps/rejected": -533.1495361328125,
      "loss": 1.0022,
      "margin_dpo/margin_mean": 170.2562713623047,
      "margin_dpo/margin_std": 211.9681396484375,
      "step": 324
    },
    {
      "epoch": 0.47723935389133626,
      "fcm_dpo/beta": 0.003626979887485504,
      "fcm_dpo/delta": 0.10734298825263977,
      "fcm_dpo/margin": 137.27645874023438,
      "fcm_dpo/q_t": 0.38791322708129883,
      "grad_norm": 33.29164123535156,
      "learning_rate": 3.147047612756302e-07,
      "logits/chosen": -0.4132434129714966,
      "logits/rejected": -0.3767145872116089,
      "logps/chosen": -273.2174377441406,
      "logps/ref_chosen": -56.215599060058594,
      "logps/ref_rejected": -70.08592987060547,
      "logps/rejected": -424.3642272949219,
      "loss": 1.0305,
      "margin_dpo/margin_mean": 137.27645874023438,
      "margin_dpo/margin_std": 158.93292236328125,
      "step": 325
    },
    {
      "epoch": 0.4787077826725404,
      "fcm_dpo/beta": 0.0036813640035688877,
      "fcm_dpo/delta": 0.08160503208637238,
      "fcm_dpo/margin": 141.9955596923828,
      "fcm_dpo/q_t": 0.3813520669937134,
      "grad_norm": 49.46158981323242,
      "learning_rate": 3.134643122927519e-07,
      "logits/chosen": -0.42192065715789795,
      "logits/rejected": -0.38546887040138245,
      "logps/chosen": -309.13665771484375,
      "logps/ref_chosen": -72.72496032714844,
      "logps/ref_rejected": -79.8467788696289,
      "logps/rejected": -458.25408935546875,
      "loss": 0.9932,
      "margin_dpo/margin_mean": 141.9955596923828,
      "margin_dpo/margin_std": 141.18475341796875,
      "step": 326
    },
    {
      "epoch": 0.4801762114537445,
      "fcm_dpo/beta": 0.0036152596585452557,
      "fcm_dpo/delta": -0.12888285517692566,
      "fcm_dpo/margin": 198.74432373046875,
      "fcm_dpo/q_t": 0.34069618582725525,
      "grad_norm": 56.69047546386719,
      "learning_rate": 3.1222219096622264e-07,
      "logits/chosen": -0.3953353762626648,
      "logits/rejected": -0.3728201389312744,
      "logps/chosen": -278.8756103515625,
      "logps/ref_chosen": -69.13441467285156,
      "logps/ref_rejected": -111.93377685546875,
      "logps/rejected": -520.4193115234375,
      "loss": 0.8922,
      "margin_dpo/margin_mean": 198.74432373046875,
      "margin_dpo/margin_std": 184.82196044921875,
      "step": 327
    },
    {
      "epoch": 0.48164464023494863,
      "fcm_dpo/beta": 0.0036046169698238373,
      "fcm_dpo/delta": -0.01882966235280037,
      "fcm_dpo/margin": 171.35028076171875,
      "fcm_dpo/q_t": 0.3656262159347534,
      "grad_norm": 41.368961334228516,
      "learning_rate": 3.1097843002709427e-07,
      "logits/chosen": -0.3795957565307617,
      "logits/rejected": -0.3735014796257019,
      "logps/chosen": -278.96380615234375,
      "logps/ref_chosen": -59.68719482421875,
      "logps/ref_rejected": -90.85499572753906,
      "logps/rejected": -481.48187255859375,
      "loss": 0.963,
      "margin_dpo/margin_mean": 171.35028076171875,
      "margin_dpo/margin_std": 190.85531616210938,
      "step": 328
    },
    {
      "epoch": 0.4831130690161527,
      "fcm_dpo/beta": 0.0035419906489551067,
      "fcm_dpo/delta": -0.06768238544464111,
      "fcm_dpo/margin": 187.00729370117188,
      "fcm_dpo/q_t": 0.3545988202095032,
      "grad_norm": 28.033584594726562,
      "learning_rate": 3.0973306224962437e-07,
      "logits/chosen": -0.35659074783325195,
      "logits/rejected": -0.33611446619033813,
      "logps/chosen": -300.49114990234375,
      "logps/ref_chosen": -65.2461929321289,
      "logps/ref_rejected": -100.69770812988281,
      "logps/rejected": -522.949951171875,
      "loss": 0.9397,
      "margin_dpo/margin_mean": 187.00729370117188,
      "margin_dpo/margin_std": 195.09939575195312,
      "step": 329
    },
    {
      "epoch": 0.4845814977973568,
      "fcm_dpo/beta": 0.0035197772085666656,
      "fcm_dpo/delta": -0.04879715293645859,
      "fcm_dpo/margin": 183.4234619140625,
      "fcm_dpo/q_t": 0.3568430542945862,
      "grad_norm": 40.50788116455078,
      "learning_rate": 3.084861204504122e-07,
      "logits/chosen": -0.3393748104572296,
      "logits/rejected": -0.34215378761291504,
      "logps/chosen": -263.3137512207031,
      "logps/ref_chosen": -46.998348236083984,
      "logps/ref_rejected": -86.87684631347656,
      "logps/rejected": -486.61572265625,
      "loss": 0.9342,
      "margin_dpo/margin_mean": 183.4234619140625,
      "margin_dpo/margin_std": 184.44473266601562,
      "step": 330
    },
    {
      "epoch": 0.48604992657856094,
      "fcm_dpo/beta": 0.003468625945970416,
      "fcm_dpo/delta": -0.08099709451198578,
      "fcm_dpo/margin": 194.73370361328125,
      "fcm_dpo/q_t": 0.34509968757629395,
      "grad_norm": 34.197383880615234,
      "learning_rate": 3.072376374875335e-07,
      "logits/chosen": -0.33651411533355713,
      "logits/rejected": -0.32489413022994995,
      "logps/chosen": -278.44921875,
      "logps/ref_chosen": -50.52424621582031,
      "logps/ref_rejected": -89.01544189453125,
      "logps/rejected": -511.67413330078125,
      "loss": 0.8733,
      "margin_dpo/margin_mean": 194.73370361328125,
      "margin_dpo/margin_std": 140.58489990234375,
      "step": 331
    },
    {
      "epoch": 0.48751835535976507,
      "fcm_dpo/beta": 0.003518771380186081,
      "fcm_dpo/delta": 0.11069121956825256,
      "fcm_dpo/margin": 140.563232421875,
      "fcm_dpo/q_t": 0.38987618684768677,
      "grad_norm": 32.90526580810547,
      "learning_rate": 3.059876462596758e-07,
      "logits/chosen": -0.3346155285835266,
      "logits/rejected": -0.3171945810317993,
      "logps/chosen": -291.15472412109375,
      "logps/ref_chosen": -49.18028259277344,
      "logps/ref_rejected": -76.48515319824219,
      "logps/rejected": -459.0228271484375,
      "loss": 1.029,
      "margin_dpo/margin_mean": 140.563232421875,
      "margin_dpo/margin_std": 166.2570037841797,
      "step": 332
    },
    {
      "epoch": 0.4889867841409692,
      "fcm_dpo/beta": 0.0034761871211230755,
      "fcm_dpo/delta": -0.055231112986803055,
      "fcm_dpo/margin": 186.94793701171875,
      "fcm_dpo/q_t": 0.3612428307533264,
      "grad_norm": 31.33414649963379,
      "learning_rate": 3.0473617970527015e-07,
      "logits/chosen": -0.37555959820747375,
      "logits/rejected": -0.36939913034439087,
      "logps/chosen": -315.4451904296875,
      "logps/ref_chosen": -63.75574493408203,
      "logps/ref_rejected": -95.04411315917969,
      "logps/rejected": -533.6814575195312,
      "loss": 0.9718,
      "margin_dpo/margin_mean": 186.94793701171875,
      "margin_dpo/margin_std": 219.6304931640625,
      "step": 333
    },
    {
      "epoch": 0.49045521292217326,
      "fcm_dpo/beta": 0.003508550813421607,
      "fcm_dpo/delta": 0.022331170737743378,
      "fcm_dpo/margin": 165.01031494140625,
      "fcm_dpo/q_t": 0.3745434284210205,
      "grad_norm": 45.59312438964844,
      "learning_rate": 3.034832708016243e-07,
      "logits/chosen": -0.37708473205566406,
      "logits/rejected": -0.37125474214553833,
      "logps/chosen": -313.27618408203125,
      "logps/ref_chosen": -66.97975158691406,
      "logps/ref_rejected": -95.31692504882812,
      "logps/rejected": -506.6236877441406,
      "loss": 1.0114,
      "margin_dpo/margin_mean": 165.01031494140625,
      "margin_dpo/margin_std": 209.47122192382812,
      "step": 334
    },
    {
      "epoch": 0.4919236417033774,
      "fcm_dpo/beta": 0.003607717342674732,
      "fcm_dpo/delta": 0.14469006657600403,
      "fcm_dpo/margin": 127.66055297851562,
      "fcm_dpo/q_t": 0.4004567563533783,
      "grad_norm": 63.05012512207031,
      "learning_rate": 3.022289525640531e-07,
      "logits/chosen": -0.38800978660583496,
      "logits/rejected": -0.36463358998298645,
      "logps/chosen": -334.33935546875,
      "logps/ref_chosen": -62.54248046875,
      "logps/ref_rejected": -87.61770629882812,
      "logps/rejected": -487.07513427734375,
      "loss": 1.0921,
      "margin_dpo/margin_mean": 127.66055297851562,
      "margin_dpo/margin_std": 194.76068115234375,
      "step": 335
    },
    {
      "epoch": 0.4933920704845815,
      "fcm_dpo/beta": 0.0035587395541369915,
      "fcm_dpo/delta": -0.1215284913778305,
      "fcm_dpo/margin": 200.35911560058594,
      "fcm_dpo/q_t": 0.34965717792510986,
      "grad_norm": 34.31383514404297,
      "learning_rate": 3.009732580450086e-07,
      "logits/chosen": -0.3676879405975342,
      "logits/rejected": -0.36403754353523254,
      "logps/chosen": -313.09222412109375,
      "logps/ref_chosen": -54.53115463256836,
      "logps/ref_rejected": -104.40424346923828,
      "logps/rejected": -563.3244018554688,
      "loss": 0.9395,
      "margin_dpo/margin_mean": 200.35910034179688,
      "margin_dpo/margin_std": 232.01760864257812,
      "step": 336
    },
    {
      "epoch": 0.4948604992657856,
      "fcm_dpo/beta": 0.003459386760368943,
      "fcm_dpo/delta": -0.11083994805812836,
      "fcm_dpo/margin": 203.0913848876953,
      "fcm_dpo/q_t": 0.34715068340301514,
      "grad_norm": 32.61246109008789,
      "learning_rate": 2.9971622033320914e-07,
      "logits/chosen": -0.3618990480899811,
      "logits/rejected": -0.3507819175720215,
      "logps/chosen": -311.53759765625,
      "logps/ref_chosen": -65.12869262695312,
      "logps/ref_rejected": -101.72701263427734,
      "logps/rejected": -551.2273559570312,
      "loss": 0.9156,
      "margin_dpo/margin_mean": 203.09136962890625,
      "margin_dpo/margin_std": 209.97573852539062,
      "step": 337
    },
    {
      "epoch": 0.49632892804698975,
      "fcm_dpo/beta": 0.0034060748293995857,
      "fcm_dpo/delta": -0.08523920178413391,
      "fcm_dpo/margin": 199.4775848388672,
      "fcm_dpo/q_t": 0.34894174337387085,
      "grad_norm": 29.552736282348633,
      "learning_rate": 2.984578725527675e-07,
      "logits/chosen": -0.4156501591205597,
      "logits/rejected": -0.40091991424560547,
      "logps/chosen": -276.4804382324219,
      "logps/ref_chosen": -58.422706604003906,
      "logps/ref_rejected": -89.06854248046875,
      "logps/rejected": -506.6038513183594,
      "loss": 0.9049,
      "margin_dpo/margin_mean": 199.4775848388672,
      "margin_dpo/margin_std": 184.65594482421875,
      "step": 338
    },
    {
      "epoch": 0.4977973568281938,
      "fcm_dpo/beta": 0.0033988947980105877,
      "fcm_dpo/delta": -0.04927036911249161,
      "fcm_dpo/margin": 189.7136993408203,
      "fcm_dpo/q_t": 0.35729217529296875,
      "grad_norm": 32.228424072265625,
      "learning_rate": 2.9719824786231796e-07,
      "logits/chosen": -0.40810322761535645,
      "logits/rejected": -0.3912397623062134,
      "logps/chosen": -287.6553649902344,
      "logps/ref_chosen": -59.99531555175781,
      "logps/ref_rejected": -103.9109115600586,
      "logps/rejected": -521.28466796875,
      "loss": 0.9323,
      "margin_dpo/margin_mean": 189.7136993408203,
      "margin_dpo/margin_std": 180.0070343017578,
      "step": 339
    },
    {
      "epoch": 0.49926578560939794,
      "fcm_dpo/beta": 0.0033571631647646427,
      "fcm_dpo/delta": 0.030759550631046295,
      "fcm_dpo/margin": 169.98486328125,
      "fcm_dpo/q_t": 0.3748738765716553,
      "grad_norm": 34.929901123046875,
      "learning_rate": 2.959373794541426e-07,
      "logits/chosen": -0.3299640417098999,
      "logits/rejected": -0.3012125492095947,
      "logps/chosen": -285.945556640625,
      "logps/ref_chosen": -52.83022689819336,
      "logps/ref_rejected": -73.10723114013672,
      "logps/rejected": -476.2073974609375,
      "loss": 0.9978,
      "margin_dpo/margin_mean": 169.98486328125,
      "margin_dpo/margin_std": 201.46371459960938,
      "step": 340
    },
    {
      "epoch": 0.5007342143906021,
      "fcm_dpo/beta": 0.0033384612761437893,
      "fcm_dpo/delta": -0.0509408563375473,
      "fcm_dpo/margin": 193.81198120117188,
      "fcm_dpo/q_t": 0.3578472137451172,
      "grad_norm": 29.954818725585938,
      "learning_rate": 2.946753005532965e-07,
      "logits/chosen": -0.358737051486969,
      "logits/rejected": -0.3582429587841034,
      "logps/chosen": -274.4048767089844,
      "logps/ref_chosen": -47.899803161621094,
      "logps/ref_rejected": -101.80987548828125,
      "logps/rejected": -522.126953125,
      "loss": 0.9392,
      "margin_dpo/margin_mean": 193.81198120117188,
      "margin_dpo/margin_std": 198.76821899414062,
      "step": 341
    },
    {
      "epoch": 0.5022026431718062,
      "fcm_dpo/beta": 0.003372443839907646,
      "fcm_dpo/delta": 0.02195318415760994,
      "fcm_dpo/margin": 171.60679626464844,
      "fcm_dpo/q_t": 0.37457433342933655,
      "grad_norm": 34.212318420410156,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -0.38501685857772827,
      "logits/rejected": -0.35319679975509644,
      "logps/chosen": -302.15545654296875,
      "logps/ref_chosen": -71.99664306640625,
      "logps/ref_rejected": -92.58959197998047,
      "logps/rejected": -494.355224609375,
      "loss": 0.9967,
      "margin_dpo/margin_mean": 171.60679626464844,
      "margin_dpo/margin_std": 208.8232421875,
      "step": 342
    },
    {
      "epoch": 0.5036710719530103,
      "fcm_dpo/beta": 0.0033103451132774353,
      "fcm_dpo/delta": -0.0885283350944519,
      "fcm_dpo/margin": 206.1126708984375,
      "fcm_dpo/q_t": 0.3450053334236145,
      "grad_norm": 34.08480453491211,
      "learning_rate": 2.9214764433242476e-07,
      "logits/chosen": -0.3982047438621521,
      "logits/rejected": -0.3942341208457947,
      "logps/chosen": -267.8955383300781,
      "logps/ref_chosen": -54.405616760253906,
      "logps/ref_rejected": -111.04142761230469,
      "logps/rejected": -530.64404296875,
      "loss": 0.8828,
      "margin_dpo/margin_mean": 206.1126708984375,
      "margin_dpo/margin_std": 162.10682678222656,
      "step": 343
    },
    {
      "epoch": 0.5051395007342144,
      "fcm_dpo/beta": 0.0033293329179286957,
      "fcm_dpo/delta": -0.014567896723747253,
      "fcm_dpo/margin": 183.69131469726562,
      "fcm_dpo/q_t": 0.3665677309036255,
      "grad_norm": 27.15936851501465,
      "learning_rate": 2.9088213361849126e-07,
      "logits/chosen": -0.38978269696235657,
      "logits/rejected": -0.3856002688407898,
      "logps/chosen": -273.5009460449219,
      "logps/ref_chosen": -53.96466827392578,
      "logps/ref_rejected": -90.62336730957031,
      "logps/rejected": -493.8509521484375,
      "loss": 0.9563,
      "margin_dpo/margin_mean": 183.69131469726562,
      "margin_dpo/margin_std": 185.71316528320312,
      "step": 344
    },
    {
      "epoch": 0.5066079295154186,
      "fcm_dpo/beta": 0.0032876995392143726,
      "fcm_dpo/delta": -0.037146538496017456,
      "fcm_dpo/margin": 193.00808715820312,
      "fcm_dpo/q_t": 0.36083394289016724,
      "grad_norm": 27.978347778320312,
      "learning_rate": 2.896155456223163e-07,
      "logits/chosen": -0.39028286933898926,
      "logits/rejected": -0.3786432147026062,
      "logps/chosen": -334.9024963378906,
      "logps/ref_chosen": -61.685699462890625,
      "logps/ref_rejected": -99.49041748046875,
      "logps/rejected": -565.7152709960938,
      "loss": 0.952,
      "margin_dpo/margin_mean": 193.00807189941406,
      "margin_dpo/margin_std": 207.6051025390625,
      "step": 345
    },
    {
      "epoch": 0.5080763582966226,
      "fcm_dpo/beta": 0.003262849524617195,
      "fcm_dpo/delta": -0.004547563381493092,
      "fcm_dpo/margin": 185.1959686279297,
      "fcm_dpo/q_t": 0.3652818500995636,
      "grad_norm": 34.94870376586914,
      "learning_rate": 2.883479137196714e-07,
      "logits/chosen": -0.3476218581199646,
      "logits/rejected": -0.3306209444999695,
      "logps/chosen": -337.10003662109375,
      "logps/ref_chosen": -55.256263732910156,
      "logps/ref_rejected": -77.41532135009766,
      "logps/rejected": -544.455078125,
      "loss": 0.9697,
      "margin_dpo/margin_mean": 185.1959686279297,
      "margin_dpo/margin_std": 203.53085327148438,
      "step": 346
    },
    {
      "epoch": 0.5095447870778267,
      "fcm_dpo/beta": 0.0032533821649849415,
      "fcm_dpo/delta": -0.015425489284098148,
      "fcm_dpo/margin": 188.865478515625,
      "fcm_dpo/q_t": 0.3658721446990967,
      "grad_norm": 38.90462112426758,
      "learning_rate": 2.8707927131383614e-07,
      "logits/chosen": -0.3579314947128296,
      "logits/rejected": -0.3465426564216614,
      "logps/chosen": -343.6216125488281,
      "logps/ref_chosen": -57.56623840332031,
      "logps/ref_rejected": -92.35509490966797,
      "logps/rejected": -567.2759399414062,
      "loss": 0.9762,
      "margin_dpo/margin_mean": 188.865478515625,
      "margin_dpo/margin_std": 218.76934814453125,
      "step": 347
    },
    {
      "epoch": 0.5110132158590308,
      "fcm_dpo/beta": 0.0032581479754298925,
      "fcm_dpo/delta": 0.0629473477602005,
      "fcm_dpo/margin": 165.63992309570312,
      "fcm_dpo/q_t": 0.381051242351532,
      "grad_norm": 30.090953826904297,
      "learning_rate": 2.858096518347179e-07,
      "logits/chosen": -0.4058574438095093,
      "logits/rejected": -0.4066629707813263,
      "logps/chosen": -306.64599609375,
      "logps/ref_chosen": -56.31770324707031,
      "logps/ref_rejected": -89.13836669921875,
      "logps/rejected": -505.1065673828125,
      "loss": 1.0098,
      "margin_dpo/margin_mean": 165.63992309570312,
      "margin_dpo/margin_std": 191.34750366210938,
      "step": 348
    },
    {
      "epoch": 0.5124816446402349,
      "fcm_dpo/beta": 0.003329505678266287,
      "fcm_dpo/delta": 0.02792828157544136,
      "fcm_dpo/margin": 172.01544189453125,
      "fcm_dpo/q_t": 0.3782421946525574,
      "grad_norm": 26.204023361206055,
      "learning_rate": 2.845390887379706e-07,
      "logits/chosen": -0.3858356475830078,
      "logits/rejected": -0.3851817548274994,
      "logps/chosen": -302.1793212890625,
      "logps/ref_chosen": -58.025516510009766,
      "logps/ref_rejected": -97.50515747070312,
      "logps/rejected": -513.6744384765625,
      "loss": 1.0217,
      "margin_dpo/margin_mean": 172.01544189453125,
      "margin_dpo/margin_std": 228.81277465820312,
      "step": 349
    },
    {
      "epoch": 0.5139500734214391,
      "fcm_dpo/beta": 0.003294072113931179,
      "fcm_dpo/delta": -0.006476100534200668,
      "fcm_dpo/margin": 183.84515380859375,
      "fcm_dpo/q_t": 0.36599087715148926,
      "grad_norm": 39.012115478515625,
      "learning_rate": 2.8326761550411346e-07,
      "logits/chosen": -0.4358096122741699,
      "logits/rejected": -0.43361157178878784,
      "logps/chosen": -314.4185485839844,
      "logps/ref_chosen": -64.33049011230469,
      "logps/ref_rejected": -89.87164306640625,
      "logps/rejected": -523.8048706054688,
      "loss": 0.9903,
      "margin_dpo/margin_mean": 183.84518432617188,
      "margin_dpo/margin_std": 221.12252807617188,
      "step": 350
    },
    {
      "epoch": 0.5154185022026432,
      "fcm_dpo/beta": 0.00327136623673141,
      "fcm_dpo/delta": -0.07544407248497009,
      "fcm_dpo/margin": 204.92010498046875,
      "fcm_dpo/q_t": 0.355930358171463,
      "grad_norm": 33.651607513427734,
      "learning_rate": 2.819952656376487e-07,
      "logits/chosen": -0.40549007058143616,
      "logits/rejected": -0.3955671489238739,
      "logps/chosen": -296.75994873046875,
      "logps/ref_chosen": -60.6721305847168,
      "logps/ref_rejected": -101.5654296875,
      "logps/rejected": -542.5733642578125,
      "loss": 0.9437,
      "margin_dpo/margin_mean": 204.92010498046875,
      "margin_dpo/margin_std": 227.87014770507812,
      "step": 351
    },
    {
      "epoch": 0.5168869309838473,
      "fcm_dpo/beta": 0.003319011302664876,
      "fcm_dpo/delta": 0.15692806243896484,
      "fcm_dpo/margin": 135.69808959960938,
      "fcm_dpo/q_t": 0.40290194749832153,
      "grad_norm": 41.35541915893555,
      "learning_rate": 2.8072207266617854e-07,
      "logits/chosen": -0.39126765727996826,
      "logits/rejected": -0.35691601037979126,
      "logps/chosen": -348.217041015625,
      "logps/ref_chosen": -70.9434585571289,
      "logps/ref_rejected": -76.6419677734375,
      "logps/rejected": -489.6136169433594,
      "loss": 1.0962,
      "margin_dpo/margin_mean": 135.69808959960938,
      "margin_dpo/margin_std": 207.16409301757812,
      "step": 352
    },
    {
      "epoch": 0.5183553597650514,
      "fcm_dpo/beta": 0.003364278469234705,
      "fcm_dpo/delta": 0.043343670666217804,
      "fcm_dpo/margin": 166.03475952148438,
      "fcm_dpo/q_t": 0.3799568712711334,
      "grad_norm": 41.904701232910156,
      "learning_rate": 2.794480701395219e-07,
      "logits/chosen": -0.43321138620376587,
      "logits/rejected": -0.41663485765457153,
      "logps/chosen": -319.3279113769531,
      "logps/ref_chosen": -58.39533996582031,
      "logps/ref_rejected": -80.33553314208984,
      "logps/rejected": -507.3028259277344,
      "loss": 1.0207,
      "margin_dpo/margin_mean": 166.03475952148438,
      "margin_dpo/margin_std": 212.48361206054688,
      "step": 353
    },
    {
      "epoch": 0.5198237885462555,
      "fcm_dpo/beta": 0.0033673213329166174,
      "fcm_dpo/delta": -0.036777447909116745,
      "fcm_dpo/margin": 188.36825561523438,
      "fcm_dpo/q_t": 0.3579559922218323,
      "grad_norm": 39.02664566040039,
      "learning_rate": 2.781732916288303e-07,
      "logits/chosen": -0.4032512307167053,
      "logits/rejected": -0.383810818195343,
      "logps/chosen": -275.3306579589844,
      "logps/ref_chosen": -59.80299377441406,
      "logps/ref_rejected": -88.75750732421875,
      "logps/rejected": -492.6534423828125,
      "loss": 0.9294,
      "margin_dpo/margin_mean": 188.36827087402344,
      "margin_dpo/margin_std": 177.9414520263672,
      "step": 354
    },
    {
      "epoch": 0.5212922173274597,
      "fcm_dpo/beta": 0.003320076037198305,
      "fcm_dpo/delta": -0.06938491016626358,
      "fcm_dpo/margin": 200.06072998046875,
      "fcm_dpo/q_t": 0.3513967990875244,
      "grad_norm": 39.97414779663086,
      "learning_rate": 2.7689777072570284e-07,
      "logits/chosen": -0.4452924132347107,
      "logits/rejected": -0.42499247193336487,
      "logps/chosen": -272.80706787109375,
      "logps/ref_chosen": -54.12849807739258,
      "logps/ref_rejected": -82.40606689453125,
      "logps/rejected": -501.1453552246094,
      "loss": 0.9087,
      "margin_dpo/margin_mean": 200.06072998046875,
      "margin_dpo/margin_std": 179.74571228027344,
      "step": 355
    },
    {
      "epoch": 0.5227606461086637,
      "fcm_dpo/beta": 0.0034437617287039757,
      "fcm_dpo/delta": 0.1814279407262802,
      "fcm_dpo/margin": 122.82986450195312,
      "fcm_dpo/q_t": 0.41210052371025085,
      "grad_norm": 50.92268753051758,
      "learning_rate": 2.7562154104130176e-07,
      "logits/chosen": -0.4147285223007202,
      "logits/rejected": -0.39140522480010986,
      "logps/chosen": -330.02154541015625,
      "logps/ref_chosen": -64.6738052368164,
      "logps/ref_rejected": -75.89926147460938,
      "logps/rejected": -464.07684326171875,
      "loss": 1.1595,
      "margin_dpo/margin_mean": 122.82986450195312,
      "margin_dpo/margin_std": 233.60147094726562,
      "step": 356
    },
    {
      "epoch": 0.5242290748898678,
      "fcm_dpo/beta": 0.0034146038815379143,
      "fcm_dpo/delta": -0.019891854375600815,
      "fcm_dpo/margin": 180.96995544433594,
      "fcm_dpo/q_t": 0.3629704713821411,
      "grad_norm": 36.10367202758789,
      "learning_rate": 2.7434463620546594e-07,
      "logits/chosen": -0.4508202075958252,
      "logits/rejected": -0.43361175060272217,
      "logps/chosen": -294.203857421875,
      "logps/ref_chosen": -52.725799560546875,
      "logps/ref_rejected": -86.84115600585938,
      "logps/rejected": -509.2891845703125,
      "loss": 0.9528,
      "margin_dpo/margin_mean": 180.96995544433594,
      "margin_dpo/margin_std": 186.50103759765625,
      "step": 357
    },
    {
      "epoch": 0.5256975036710719,
      "fcm_dpo/beta": 0.003465921152383089,
      "fcm_dpo/delta": 0.06651946902275085,
      "fcm_dpo/margin": 154.9437255859375,
      "fcm_dpo/q_t": 0.3825710117816925,
      "grad_norm": 31.915424346923828,
      "learning_rate": 2.730670898658255e-07,
      "logits/chosen": -0.4602763056755066,
      "logits/rejected": -0.44098126888275146,
      "logps/chosen": -284.6385192871094,
      "logps/ref_chosen": -63.20543670654297,
      "logps/ref_rejected": -88.373291015625,
      "logps/rejected": -464.7501220703125,
      "loss": 1.0108,
      "margin_dpo/margin_mean": 154.9437255859375,
      "margin_dpo/margin_std": 186.69895935058594,
      "step": 358
    },
    {
      "epoch": 0.527165932452276,
      "fcm_dpo/beta": 0.003439284861087799,
      "fcm_dpo/delta": -0.05646362528204918,
      "fcm_dpo/margin": 189.73446655273438,
      "fcm_dpo/q_t": 0.36093398928642273,
      "grad_norm": 38.223670959472656,
      "learning_rate": 2.717889356869146e-07,
      "logits/chosen": -0.39621424674987793,
      "logits/rejected": -0.3747130036354065,
      "logps/chosen": -310.32684326171875,
      "logps/ref_chosen": -56.370216369628906,
      "logps/ref_rejected": -82.17375183105469,
      "logps/rejected": -525.8648681640625,
      "loss": 0.9534,
      "margin_dpo/margin_mean": 189.73448181152344,
      "margin_dpo/margin_std": 211.576416015625,
      "step": 359
    },
    {
      "epoch": 0.5286343612334802,
      "fcm_dpo/beta": 0.0034922873601317406,
      "fcm_dpo/delta": 0.09579764306545258,
      "fcm_dpo/margin": 145.714599609375,
      "fcm_dpo/q_t": 0.3853462338447571,
      "grad_norm": 46.432003021240234,
      "learning_rate": 2.7051020734928443e-07,
      "logits/chosen": -0.37489083409309387,
      "logits/rejected": -0.3580286502838135,
      "logps/chosen": -298.4937744140625,
      "logps/ref_chosen": -51.460384368896484,
      "logps/ref_rejected": -69.83892059326172,
      "logps/rejected": -462.5869445800781,
      "loss": 1.0106,
      "margin_dpo/margin_mean": 145.714599609375,
      "margin_dpo/margin_std": 157.42027282714844,
      "step": 360
    },
    {
      "epoch": 0.5301027900146843,
      "fcm_dpo/beta": 0.0035505923442542553,
      "fcm_dpo/delta": 0.06202498450875282,
      "fcm_dpo/margin": 152.30416870117188,
      "fcm_dpo/q_t": 0.3825526237487793,
      "grad_norm": 37.47382736206055,
      "learning_rate": 2.6923093854861593e-07,
      "logits/chosen": -0.3717784285545349,
      "logits/rejected": -0.36584901809692383,
      "logps/chosen": -310.5164794921875,
      "logps/ref_chosen": -53.86951446533203,
      "logps/ref_rejected": -90.7692642211914,
      "logps/rejected": -499.7203674316406,
      "loss": 1.0306,
      "margin_dpo/margin_mean": 152.30416870117188,
      "margin_dpo/margin_std": 200.70108032226562,
      "step": 361
    },
    {
      "epoch": 0.5315712187958884,
      "fcm_dpo/beta": 0.0034302864223718643,
      "fcm_dpo/delta": -0.19361282885074615,
      "fcm_dpo/margin": 226.57215881347656,
      "fcm_dpo/q_t": 0.3321119248867035,
      "grad_norm": 34.013301849365234,
      "learning_rate": 2.679511629948319e-07,
      "logits/chosen": -0.4100894331932068,
      "logits/rejected": -0.4160076677799225,
      "logps/chosen": -279.9183654785156,
      "logps/ref_chosen": -58.639060974121094,
      "logps/ref_rejected": -105.58195495605469,
      "logps/rejected": -553.4334106445312,
      "loss": 0.8572,
      "margin_dpo/margin_mean": 226.57215881347656,
      "margin_dpo/margin_std": 208.89010620117188,
      "step": 362
    },
    {
      "epoch": 0.5330396475770925,
      "fcm_dpo/beta": 0.003321536350995302,
      "fcm_dpo/delta": -0.20300859212875366,
      "fcm_dpo/margin": 237.05946350097656,
      "fcm_dpo/q_t": 0.32788795232772827,
      "grad_norm": 25.153287887573242,
      "learning_rate": 2.6667091441120816e-07,
      "logits/chosen": -0.3687742054462433,
      "logits/rejected": -0.3484545350074768,
      "logps/chosen": -246.83908081054688,
      "logps/ref_chosen": -44.558380126953125,
      "logps/ref_rejected": -74.69496154785156,
      "logps/rejected": -514.03515625,
      "loss": 0.8467,
      "margin_dpo/margin_mean": 237.0594482421875,
      "margin_dpo/margin_std": 207.8511962890625,
      "step": 363
    },
    {
      "epoch": 0.5345080763582967,
      "fcm_dpo/beta": 0.003306190250441432,
      "fcm_dpo/delta": 0.02047722041606903,
      "fcm_dpo/margin": 175.40988159179688,
      "fcm_dpo/q_t": 0.37260884046554565,
      "grad_norm": 35.79054641723633,
      "learning_rate": 2.6539022653348575e-07,
      "logits/chosen": -0.40862590074539185,
      "logits/rejected": -0.41578084230422974,
      "logps/chosen": -270.4530944824219,
      "logps/ref_chosen": -48.894622802734375,
      "logps/ref_rejected": -91.395751953125,
      "logps/rejected": -488.36407470703125,
      "loss": 0.9919,
      "margin_dpo/margin_mean": 175.40988159179688,
      "margin_dpo/margin_std": 205.0936737060547,
      "step": 364
    },
    {
      "epoch": 0.5359765051395007,
      "fcm_dpo/beta": 0.0033004698343575,
      "fcm_dpo/delta": 0.016225282102823257,
      "fcm_dpo/margin": 177.1582794189453,
      "fcm_dpo/q_t": 0.3734648823738098,
      "grad_norm": 36.35362243652344,
      "learning_rate": 2.641091331089811e-07,
      "logits/chosen": -0.38673701882362366,
      "logits/rejected": -0.3943500518798828,
      "logps/chosen": -276.78192138671875,
      "logps/ref_chosen": -51.49274444580078,
      "logps/ref_rejected": -92.70166778564453,
      "logps/rejected": -495.1491394042969,
      "loss": 0.9778,
      "margin_dpo/margin_mean": 177.1582794189453,
      "margin_dpo/margin_std": 200.8518524169922,
      "step": 365
    },
    {
      "epoch": 0.5374449339207048,
      "fcm_dpo/beta": 0.0032841768115758896,
      "fcm_dpo/delta": 0.0016945749521255493,
      "fcm_dpo/margin": 182.0388946533203,
      "fcm_dpo/q_t": 0.3690670430660248,
      "grad_norm": 28.885610580444336,
      "learning_rate": 2.6282766789569736e-07,
      "logits/chosen": -0.36415767669677734,
      "logits/rejected": -0.37583303451538086,
      "logps/chosen": -248.95938110351562,
      "logps/ref_chosen": -44.7205696105957,
      "logps/ref_rejected": -83.31040954589844,
      "logps/rejected": -469.588134765625,
      "loss": 0.9792,
      "margin_dpo/margin_mean": 182.0388946533203,
      "margin_dpo/margin_std": 206.19607543945312,
      "step": 366
    },
    {
      "epoch": 0.5389133627019089,
      "fcm_dpo/beta": 0.0033485644962638617,
      "fcm_dpo/delta": 0.08677735924720764,
      "fcm_dpo/margin": 154.576416015625,
      "fcm_dpo/q_t": 0.3839992582798004,
      "grad_norm": 24.783180236816406,
      "learning_rate": 2.615458646614349e-07,
      "logits/chosen": -0.39714837074279785,
      "logits/rejected": -0.37421467900276184,
      "logps/chosen": -266.13519287109375,
      "logps/ref_chosen": -58.405418395996094,
      "logps/ref_rejected": -76.75132751464844,
      "logps/rejected": -439.0574951171875,
      "loss": 1.016,
      "margin_dpo/margin_mean": 154.576416015625,
      "margin_dpo/margin_std": 178.5806884765625,
      "step": 367
    },
    {
      "epoch": 0.540381791483113,
      "fcm_dpo/beta": 0.003304690821096301,
      "fcm_dpo/delta": -0.12389971315860748,
      "fcm_dpo/margin": 216.4133758544922,
      "fcm_dpo/q_t": 0.33748477697372437,
      "grad_norm": 53.478485107421875,
      "learning_rate": 2.6026375718290083e-07,
      "logits/chosen": -0.4093177318572998,
      "logits/rejected": -0.4137042760848999,
      "logps/chosen": -243.46835327148438,
      "logps/ref_chosen": -44.452518463134766,
      "logps/ref_rejected": -98.55526733398438,
      "logps/rejected": -513.9844970703125,
      "loss": 0.8591,
      "margin_dpo/margin_mean": 216.4133758544922,
      "margin_dpo/margin_std": 163.95318603515625,
      "step": 368
    },
    {
      "epoch": 0.5418502202643172,
      "fcm_dpo/beta": 0.0033651897683739662,
      "fcm_dpo/delta": 0.14462685585021973,
      "fcm_dpo/margin": 136.97042846679688,
      "fcm_dpo/q_t": 0.39847463369369507,
      "grad_norm": 42.55653762817383,
      "learning_rate": 2.589813792448196e-07,
      "logits/chosen": -0.3915749788284302,
      "logits/rejected": -0.3701251447200775,
      "logps/chosen": -312.446533203125,
      "logps/ref_chosen": -71.38150024414062,
      "logps/ref_rejected": -91.29582214355469,
      "logps/rejected": -469.33123779296875,
      "loss": 1.083,
      "margin_dpo/margin_mean": 136.9704132080078,
      "margin_dpo/margin_std": 200.96856689453125,
      "step": 369
    },
    {
      "epoch": 0.5433186490455213,
      "fcm_dpo/beta": 0.003478851169347763,
      "fcm_dpo/delta": 0.18450786173343658,
      "fcm_dpo/margin": 121.46247863769531,
      "fcm_dpo/q_t": 0.40680354833602905,
      "grad_norm": 36.18484878540039,
      "learning_rate": 2.5769876463904263e-07,
      "logits/chosen": -0.4049254357814789,
      "logits/rejected": -0.39879292249679565,
      "logps/chosen": -330.7537841796875,
      "logps/ref_chosen": -71.60749816894531,
      "logps/ref_rejected": -97.25978088378906,
      "logps/rejected": -477.86859130859375,
      "loss": 1.1047,
      "margin_dpo/margin_mean": 121.46247863769531,
      "margin_dpo/margin_std": 185.816162109375,
      "step": 370
    },
    {
      "epoch": 0.5447870778267254,
      "fcm_dpo/beta": 0.003517968812957406,
      "fcm_dpo/delta": 0.012149915099143982,
      "fcm_dpo/margin": 167.24020385742188,
      "fcm_dpo/q_t": 0.37212252616882324,
      "grad_norm": 32.21125030517578,
      "learning_rate": 2.5641594716365744e-07,
      "logits/chosen": -0.41469210386276245,
      "logits/rejected": -0.39996105432510376,
      "logps/chosen": -320.701171875,
      "logps/ref_chosen": -69.41448974609375,
      "logps/ref_rejected": -99.17217254638672,
      "logps/rejected": -517.6990966796875,
      "loss": 1.0022,
      "margin_dpo/margin_mean": 167.24020385742188,
      "margin_dpo/margin_std": 208.5806121826172,
      "step": 371
    },
    {
      "epoch": 0.5462555066079295,
      "fcm_dpo/beta": 0.00341993011534214,
      "fcm_dpo/delta": -0.14982974529266357,
      "fcm_dpo/margin": 215.7982635498047,
      "fcm_dpo/q_t": 0.3453604578971863,
      "grad_norm": 25.910696029663086,
      "learning_rate": 2.551329606220976e-07,
      "logits/chosen": -0.4080229103565216,
      "logits/rejected": -0.38077276945114136,
      "logps/chosen": -302.6246643066406,
      "logps/ref_chosen": -61.8179931640625,
      "logps/ref_rejected": -78.53948974609375,
      "logps/rejected": -535.1444091796875,
      "loss": 0.9163,
      "margin_dpo/margin_mean": 215.79824829101562,
      "margin_dpo/margin_std": 238.89315795898438,
      "step": 372
    },
    {
      "epoch": 0.5477239353891337,
      "fcm_dpo/beta": 0.0033825889695435762,
      "fcm_dpo/delta": -0.09976015239953995,
      "fcm_dpo/margin": 204.6912841796875,
      "fcm_dpo/q_t": 0.3472912907600403,
      "grad_norm": 46.24806213378906,
      "learning_rate": 2.538498388222517e-07,
      "logits/chosen": -0.3898068368434906,
      "logits/rejected": -0.36087289452552795,
      "logps/chosen": -328.5521240234375,
      "logps/ref_chosen": -64.21713256835938,
      "logps/ref_rejected": -85.95960998535156,
      "logps/rejected": -554.9859619140625,
      "loss": 0.9146,
      "margin_dpo/margin_mean": 204.6912841796875,
      "margin_dpo/margin_std": 197.73797607421875,
      "step": 373
    },
    {
      "epoch": 0.5491923641703378,
      "fcm_dpo/beta": 0.0033088005147874355,
      "fcm_dpo/delta": -0.009856566786766052,
      "fcm_dpo/margin": 183.782958984375,
      "fcm_dpo/q_t": 0.3736165761947632,
      "grad_norm": 32.12522888183594,
      "learning_rate": 2.525666155755725e-07,
      "logits/chosen": -0.46630632877349854,
      "logits/rejected": -0.44315189123153687,
      "logps/chosen": -309.981201171875,
      "logps/ref_chosen": -70.65018463134766,
      "logps/ref_rejected": -93.64016723632812,
      "logps/rejected": -516.754150390625,
      "loss": 1.0042,
      "margin_dpo/margin_mean": 183.782958984375,
      "margin_dpo/margin_std": 240.5712890625,
      "step": 374
    },
    {
      "epoch": 0.5506607929515418,
      "fcm_dpo/beta": 0.003304019570350647,
      "fcm_dpo/delta": 0.007909119129180908,
      "fcm_dpo/margin": 178.82217407226562,
      "fcm_dpo/q_t": 0.3718445301055908,
      "grad_norm": 32.83172607421875,
      "learning_rate": 2.512833246961859e-07,
      "logits/chosen": -0.4039704203605652,
      "logits/rejected": -0.40034106373786926,
      "logps/chosen": -325.0075378417969,
      "logps/ref_chosen": -60.080223083496094,
      "logps/ref_rejected": -88.93830871582031,
      "logps/rejected": -532.6878051757812,
      "loss": 1.0158,
      "margin_dpo/margin_mean": 178.8221893310547,
      "margin_dpo/margin_std": 229.82867431640625,
      "step": 375
    },
    {
      "epoch": 0.5521292217327459,
      "fcm_dpo/beta": 0.003269658889621496,
      "fcm_dpo/delta": -0.13697855174541473,
      "fcm_dpo/margin": 222.34677124023438,
      "fcm_dpo/q_t": 0.3445650041103363,
      "grad_norm": 41.24235153198242,
      "learning_rate": 2.5e-07,
      "logits/chosen": -0.42362266778945923,
      "logits/rejected": -0.41283369064331055,
      "logps/chosen": -312.84063720703125,
      "logps/ref_chosen": -62.660308837890625,
      "logps/ref_rejected": -105.52660369873047,
      "logps/rejected": -578.0537109375,
      "loss": 0.9182,
      "margin_dpo/margin_mean": 222.34677124023438,
      "margin_dpo/margin_std": 240.65061950683594,
      "step": 376
    },
    {
      "epoch": 0.55359765051395,
      "fcm_dpo/beta": 0.0032267069909721613,
      "fcm_dpo/delta": -0.05978459492325783,
      "fcm_dpo/margin": 203.26560974121094,
      "fcm_dpo/q_t": 0.3565753102302551,
      "grad_norm": 30.29025650024414,
      "learning_rate": 2.487166753038141e-07,
      "logits/chosen": -0.37672150135040283,
      "logits/rejected": -0.3738042712211609,
      "logps/chosen": -322.83746337890625,
      "logps/ref_chosen": -54.478736877441406,
      "logps/ref_rejected": -98.70335388183594,
      "logps/rejected": -570.3276977539062,
      "loss": 0.9434,
      "margin_dpo/margin_mean": 203.265625,
      "margin_dpo/margin_std": 216.9762420654297,
      "step": 377
    },
    {
      "epoch": 0.5550660792951542,
      "fcm_dpo/beta": 0.0031417347490787506,
      "fcm_dpo/delta": -0.11842157691717148,
      "fcm_dpo/margin": 225.80239868164062,
      "fcm_dpo/q_t": 0.3420916795730591,
      "grad_norm": 36.97312545776367,
      "learning_rate": 2.4743338442442754e-07,
      "logits/chosen": -0.37459754943847656,
      "logits/rejected": -0.3812897205352783,
      "logps/chosen": -291.92535400390625,
      "logps/ref_chosen": -45.02053451538086,
      "logps/ref_rejected": -88.0469741821289,
      "logps/rejected": -560.754150390625,
      "loss": 0.8968,
      "margin_dpo/margin_mean": 225.80239868164062,
      "margin_dpo/margin_std": 211.446533203125,
      "step": 378
    },
    {
      "epoch": 0.5565345080763583,
      "fcm_dpo/beta": 0.0030931192450225353,
      "fcm_dpo/delta": -0.05514880642294884,
      "fcm_dpo/margin": 210.42945861816406,
      "fcm_dpo/q_t": 0.35938382148742676,
      "grad_norm": 45.417083740234375,
      "learning_rate": 2.461501611777483e-07,
      "logits/chosen": -0.34937772154808044,
      "logits/rejected": -0.36817657947540283,
      "logps/chosen": -330.56414794921875,
      "logps/ref_chosen": -53.182098388671875,
      "logps/ref_rejected": -114.3001708984375,
      "logps/rejected": -602.1116333007812,
      "loss": 0.9546,
      "margin_dpo/margin_mean": 210.429443359375,
      "margin_dpo/margin_std": 237.06201171875,
      "step": 379
    },
    {
      "epoch": 0.5580029368575624,
      "fcm_dpo/beta": 0.0030490856152027845,
      "fcm_dpo/delta": -0.1288592517375946,
      "fcm_dpo/margin": 236.06048583984375,
      "fcm_dpo/q_t": 0.34213754534721375,
      "grad_norm": 34.14671325683594,
      "learning_rate": 2.4486703937790243e-07,
      "logits/chosen": -0.38159555196762085,
      "logits/rejected": -0.4034666419029236,
      "logps/chosen": -325.86065673828125,
      "logps/ref_chosen": -51.3530387878418,
      "logps/ref_rejected": -104.19169616699219,
      "logps/rejected": -614.759765625,
      "loss": 0.9184,
      "margin_dpo/margin_mean": 236.06048583984375,
      "margin_dpo/margin_std": 250.18896484375,
      "step": 380
    },
    {
      "epoch": 0.5594713656387665,
      "fcm_dpo/beta": 0.0030402536503970623,
      "fcm_dpo/delta": 0.06344518810510635,
      "fcm_dpo/margin": 177.61572265625,
      "fcm_dpo/q_t": 0.38519516587257385,
      "grad_norm": 35.91572189331055,
      "learning_rate": 2.435840528363426e-07,
      "logits/chosen": -0.4085123538970947,
      "logits/rejected": -0.386846125125885,
      "logps/chosen": -332.9157409667969,
      "logps/ref_chosen": -57.80306625366211,
      "logps/ref_rejected": -79.21940612792969,
      "logps/rejected": -531.9478149414062,
      "loss": 1.0584,
      "margin_dpo/margin_mean": 177.61572265625,
      "margin_dpo/margin_std": 262.201904296875,
      "step": 381
    },
    {
      "epoch": 0.5609397944199707,
      "fcm_dpo/beta": 0.0030455852393060923,
      "fcm_dpo/delta": -0.024747373536229134,
      "fcm_dpo/margin": 204.61878967285156,
      "fcm_dpo/q_t": 0.3605649173259735,
      "grad_norm": 28.64333724975586,
      "learning_rate": 2.4230123536095745e-07,
      "logits/chosen": -0.43623343110084534,
      "logits/rejected": -0.43505585193634033,
      "logps/chosen": -338.3182067871094,
      "logps/ref_chosen": -66.02030181884766,
      "logps/ref_rejected": -110.71016693115234,
      "logps/rejected": -587.6268310546875,
      "loss": 0.9376,
      "margin_dpo/margin_mean": 204.6187744140625,
      "margin_dpo/margin_std": 196.5201416015625,
      "step": 382
    },
    {
      "epoch": 0.5624082232011748,
      "fcm_dpo/beta": 0.003025210928171873,
      "fcm_dpo/delta": -0.03023645095527172,
      "fcm_dpo/margin": 207.67013549804688,
      "fcm_dpo/q_t": 0.36507272720336914,
      "grad_norm": 36.634037017822266,
      "learning_rate": 2.4101862075518037e-07,
      "logits/chosen": -0.38792383670806885,
      "logits/rejected": -0.39373254776000977,
      "logps/chosen": -328.1868896484375,
      "logps/ref_chosen": -50.39148712158203,
      "logps/ref_rejected": -93.71589660644531,
      "logps/rejected": -579.181396484375,
      "loss": 0.9913,
      "margin_dpo/margin_mean": 207.67013549804688,
      "margin_dpo/margin_std": 263.74371337890625,
      "step": 383
    },
    {
      "epoch": 0.5638766519823789,
      "fcm_dpo/beta": 0.003078867681324482,
      "fcm_dpo/delta": 0.08402273058891296,
      "fcm_dpo/margin": 168.70809936523438,
      "fcm_dpo/q_t": 0.38155868649482727,
      "grad_norm": 27.592374801635742,
      "learning_rate": 2.397362428170992e-07,
      "logits/chosen": -0.425597608089447,
      "logits/rejected": -0.4135643243789673,
      "logps/chosen": -330.5281677246094,
      "logps/ref_chosen": -52.046104431152344,
      "logps/ref_rejected": -85.76089477539062,
      "logps/rejected": -532.9510498046875,
      "loss": 0.9975,
      "margin_dpo/margin_mean": 168.70809936523438,
      "margin_dpo/margin_std": 170.51783752441406,
      "step": 384
    },
    {
      "epoch": 0.5653450807635829,
      "fcm_dpo/beta": 0.0030736164189875126,
      "fcm_dpo/delta": -0.00494825653731823,
      "fcm_dpo/margin": 196.7218475341797,
      "fcm_dpo/q_t": 0.3634437322616577,
      "grad_norm": 29.260663986206055,
      "learning_rate": 2.3845413533856514e-07,
      "logits/chosen": -0.4318080544471741,
      "logits/rejected": -0.39295560121536255,
      "logps/chosen": -317.27215576171875,
      "logps/ref_chosen": -65.55215454101562,
      "logps/ref_rejected": -77.82792663574219,
      "logps/rejected": -526.269775390625,
      "loss": 0.9452,
      "margin_dpo/margin_mean": 196.7218475341797,
      "margin_dpo/margin_std": 186.86444091796875,
      "step": 385
    },
    {
      "epoch": 0.566813509544787,
      "fcm_dpo/beta": 0.003056345507502556,
      "fcm_dpo/delta": -0.04953234642744064,
      "fcm_dpo/margin": 211.47879028320312,
      "fcm_dpo/q_t": 0.3594558835029602,
      "grad_norm": 32.3004264831543,
      "learning_rate": 2.3717233210430254e-07,
      "logits/chosen": -0.4355229139328003,
      "logits/rejected": -0.4245448112487793,
      "logps/chosen": -326.9559326171875,
      "logps/ref_chosen": -58.22185516357422,
      "logps/ref_rejected": -92.32742309570312,
      "logps/rejected": -572.5403442382812,
      "loss": 0.9513,
      "margin_dpo/margin_mean": 211.4788055419922,
      "margin_dpo/margin_std": 234.755859375,
      "step": 386
    },
    {
      "epoch": 0.5682819383259912,
      "fcm_dpo/beta": 0.003059773938730359,
      "fcm_dpo/delta": 0.06919734925031662,
      "fcm_dpo/margin": 174.63983154296875,
      "fcm_dpo/q_t": 0.37939631938934326,
      "grad_norm": 37.251277923583984,
      "learning_rate": 2.3589086689101889e-07,
      "logits/chosen": -0.5007586479187012,
      "logits/rejected": -0.4742280840873718,
      "logps/chosen": -350.56463623046875,
      "logps/ref_chosen": -66.41944885253906,
      "logps/ref_rejected": -92.16915893554688,
      "logps/rejected": -550.9541625976562,
      "loss": 0.9959,
      "margin_dpo/margin_mean": 174.63983154296875,
      "margin_dpo/margin_std": 184.31121826171875,
      "step": 387
    },
    {
      "epoch": 0.5697503671071953,
      "fcm_dpo/beta": 0.0030251401476562023,
      "fcm_dpo/delta": -0.11198361963033676,
      "fcm_dpo/margin": 232.57754516601562,
      "fcm_dpo/q_t": 0.34893184900283813,
      "grad_norm": 34.62775421142578,
      "learning_rate": 2.3460977346651428e-07,
      "logits/chosen": -0.37753403186798096,
      "logits/rejected": -0.384876012802124,
      "logps/chosen": -325.88348388671875,
      "logps/ref_chosen": -50.129459381103516,
      "logps/ref_rejected": -104.43305969238281,
      "logps/rejected": -612.7646484375,
      "loss": 0.9198,
      "margin_dpo/margin_mean": 232.57754516601562,
      "margin_dpo/margin_std": 249.87030029296875,
      "step": 388
    },
    {
      "epoch": 0.5712187958883994,
      "fcm_dpo/beta": 0.0030121058225631714,
      "fcm_dpo/delta": -0.0019872207194566727,
      "fcm_dpo/margin": 199.80288696289062,
      "fcm_dpo/q_t": 0.36984607577323914,
      "grad_norm": 36.19541549682617,
      "learning_rate": 2.3332908558879177e-07,
      "logits/chosen": -0.45936498045921326,
      "logits/rejected": -0.44065529108047485,
      "logps/chosen": -367.11614990234375,
      "logps/ref_chosen": -57.906593322753906,
      "logps/ref_rejected": -77.91454315185547,
      "logps/rejected": -586.927001953125,
      "loss": 0.9889,
      "margin_dpo/margin_mean": 199.80288696289062,
      "margin_dpo/margin_std": 242.24429321289062,
      "step": 389
    },
    {
      "epoch": 0.5726872246696035,
      "fcm_dpo/beta": 0.0030045639723539352,
      "fcm_dpo/delta": 0.008383665233850479,
      "fcm_dpo/margin": 196.9191131591797,
      "fcm_dpo/q_t": 0.37548089027404785,
      "grad_norm": 35.78683853149414,
      "learning_rate": 2.320488370051681e-07,
      "logits/chosen": -0.4140510559082031,
      "logits/rejected": -0.40045636892318726,
      "logps/chosen": -360.5129699707031,
      "logps/ref_chosen": -49.22591781616211,
      "logps/ref_rejected": -85.5281982421875,
      "logps/rejected": -593.734375,
      "loss": 1.0231,
      "margin_dpo/margin_mean": 196.9191131591797,
      "margin_dpo/margin_std": 272.89666748046875,
      "step": 390
    },
    {
      "epoch": 0.5741556534508077,
      "fcm_dpo/beta": 0.0031170903239399195,
      "fcm_dpo/delta": 0.19101738929748535,
      "fcm_dpo/margin": 133.54165649414062,
      "fcm_dpo/q_t": 0.41037672758102417,
      "grad_norm": 45.97058868408203,
      "learning_rate": 2.3076906145138405e-07,
      "logits/chosen": -0.41732552647590637,
      "logits/rejected": -0.40580475330352783,
      "logps/chosen": -364.70263671875,
      "logps/ref_chosen": -64.32965087890625,
      "logps/ref_rejected": -86.73820495605469,
      "logps/rejected": -520.65283203125,
      "loss": 1.1084,
      "margin_dpo/margin_mean": 133.54165649414062,
      "margin_dpo/margin_std": 206.95831298828125,
      "step": 391
    },
    {
      "epoch": 0.5756240822320118,
      "fcm_dpo/beta": 0.003100549802184105,
      "fcm_dpo/delta": -0.10810496658086777,
      "fcm_dpo/margin": 225.9717559814453,
      "fcm_dpo/q_t": 0.3456001877784729,
      "grad_norm": 33.09029769897461,
      "learning_rate": 2.294897926507156e-07,
      "logits/chosen": -0.4238108992576599,
      "logits/rejected": -0.41437631845474243,
      "logps/chosen": -324.51971435546875,
      "logps/ref_chosen": -53.50397872924805,
      "logps/ref_rejected": -102.34584045410156,
      "logps/rejected": -599.3333129882812,
      "loss": 0.8946,
      "margin_dpo/margin_mean": 225.9717559814453,
      "margin_dpo/margin_std": 207.6659393310547,
      "step": 392
    },
    {
      "epoch": 0.5770925110132159,
      "fcm_dpo/beta": 0.003088606521487236,
      "fcm_dpo/delta": 0.04707473888993263,
      "fcm_dpo/margin": 179.875244140625,
      "fcm_dpo/q_t": 0.3865922689437866,
      "grad_norm": 37.300777435302734,
      "learning_rate": 2.2821106431308543e-07,
      "logits/chosen": -0.4085625410079956,
      "logits/rejected": -0.40135449171066284,
      "logps/chosen": -313.4761962890625,
      "logps/ref_chosen": -46.473915100097656,
      "logps/ref_rejected": -71.96885681152344,
      "logps/rejected": -518.846435546875,
      "loss": 1.0513,
      "margin_dpo/margin_mean": 179.87525939941406,
      "margin_dpo/margin_std": 273.9698791503906,
      "step": 393
    },
    {
      "epoch": 0.57856093979442,
      "fcm_dpo/beta": 0.003088343655690551,
      "fcm_dpo/delta": -0.005594061687588692,
      "fcm_dpo/margin": 195.913330078125,
      "fcm_dpo/q_t": 0.371346116065979,
      "grad_norm": 37.6540641784668,
      "learning_rate": 2.2693291013417452e-07,
      "logits/chosen": -0.4252457022666931,
      "logits/rejected": -0.4192967116832733,
      "logps/chosen": -353.8863525390625,
      "logps/ref_chosen": -52.91154861450195,
      "logps/ref_rejected": -90.8226318359375,
      "logps/rejected": -587.7107543945312,
      "loss": 0.9933,
      "margin_dpo/margin_mean": 195.913330078125,
      "margin_dpo/margin_std": 244.0496826171875,
      "step": 394
    },
    {
      "epoch": 0.580029368575624,
      "fcm_dpo/beta": 0.0030713113956153393,
      "fcm_dpo/delta": -0.03419704735279083,
      "fcm_dpo/margin": 205.683837890625,
      "fcm_dpo/q_t": 0.36777931451797485,
      "grad_norm": 43.52488708496094,
      "learning_rate": 2.2565536379453404e-07,
      "logits/chosen": -0.43619823455810547,
      "logits/rejected": -0.4260198175907135,
      "logps/chosen": -357.0216369628906,
      "logps/ref_chosen": -62.546112060546875,
      "logps/ref_rejected": -83.78262329101562,
      "logps/rejected": -583.9420166015625,
      "loss": 0.9959,
      "margin_dpo/margin_mean": 205.683837890625,
      "margin_dpo/margin_std": 268.2707214355469,
      "step": 395
    },
    {
      "epoch": 0.5814977973568282,
      "fcm_dpo/beta": 0.0030712145380675793,
      "fcm_dpo/delta": 0.00153273344039917,
      "fcm_dpo/margin": 194.87359619140625,
      "fcm_dpo/q_t": 0.36835265159606934,
      "grad_norm": 34.06664276123047,
      "learning_rate": 2.2437845895869825e-07,
      "logits/chosen": -0.4365978240966797,
      "logits/rejected": -0.40913087129592896,
      "logps/chosen": -365.53350830078125,
      "logps/ref_chosen": -68.99594116210938,
      "logps/ref_rejected": -88.64665985107422,
      "logps/rejected": -580.0578002929688,
      "loss": 0.9636,
      "margin_dpo/margin_mean": 194.87359619140625,
      "margin_dpo/margin_std": 210.96783447265625,
      "step": 396
    },
    {
      "epoch": 0.5829662261380323,
      "fcm_dpo/beta": 0.0029885326512157917,
      "fcm_dpo/delta": -0.15819747745990753,
      "fcm_dpo/margin": 249.2646942138672,
      "fcm_dpo/q_t": 0.3350715637207031,
      "grad_norm": 35.845821380615234,
      "learning_rate": 2.2310222927429716e-07,
      "logits/chosen": -0.40235501527786255,
      "logits/rejected": -0.3899458944797516,
      "logps/chosen": -331.36700439453125,
      "logps/ref_chosen": -61.27716827392578,
      "logps/ref_rejected": -103.11612701416016,
      "logps/rejected": -622.4706420898438,
      "loss": 0.8687,
      "margin_dpo/margin_mean": 249.26467895507812,
      "margin_dpo/margin_std": 219.42138671875,
      "step": 397
    },
    {
      "epoch": 0.5844346549192364,
      "fcm_dpo/beta": 0.0029659317806363106,
      "fcm_dpo/delta": -0.029825370758771896,
      "fcm_dpo/margin": 211.71212768554688,
      "fcm_dpo/q_t": 0.3643757104873657,
      "grad_norm": 33.558929443359375,
      "learning_rate": 2.2182670837116972e-07,
      "logits/chosen": -0.46954572200775146,
      "logits/rejected": -0.458831787109375,
      "logps/chosen": -372.77703857421875,
      "logps/ref_chosen": -68.15155029296875,
      "logps/ref_rejected": -108.52360534667969,
      "logps/rejected": -624.8612060546875,
      "loss": 0.9628,
      "margin_dpo/margin_mean": 211.71212768554688,
      "margin_dpo/margin_std": 241.5102081298828,
      "step": 398
    },
    {
      "epoch": 0.5859030837004405,
      "fcm_dpo/beta": 0.0029565119184553623,
      "fcm_dpo/delta": 0.0046921707689762115,
      "fcm_dpo/margin": 201.39321899414062,
      "fcm_dpo/q_t": 0.3706039786338806,
      "grad_norm": 38.02647399902344,
      "learning_rate": 2.2055192986047804e-07,
      "logits/chosen": -0.4315683841705322,
      "logits/rejected": -0.38041120767593384,
      "logps/chosen": -330.4461975097656,
      "logps/ref_chosen": -60.889801025390625,
      "logps/ref_rejected": -77.965576171875,
      "logps/rejected": -548.9151611328125,
      "loss": 0.9914,
      "margin_dpo/margin_mean": 201.39321899414062,
      "margin_dpo/margin_std": 242.23544311523438,
      "step": 399
    },
    {
      "epoch": 0.5873715124816447,
      "fcm_dpo/beta": 0.002852116245776415,
      "fcm_dpo/delta": -0.23298129439353943,
      "fcm_dpo/margin": 284.9051818847656,
      "fcm_dpo/q_t": 0.31991565227508545,
      "grad_norm": 30.26348876953125,
      "learning_rate": 2.192779273338215e-07,
      "logits/chosen": -0.40956538915634155,
      "logits/rejected": -0.3953632712364197,
      "logps/chosen": -309.8514404296875,
      "logps/ref_chosen": -63.64359664916992,
      "logps/ref_rejected": -105.252685546875,
      "logps/rejected": -636.36572265625,
      "loss": 0.8283,
      "margin_dpo/margin_mean": 284.9051818847656,
      "margin_dpo/margin_std": 236.2659912109375,
      "step": 400
    },
    {
      "epoch": 0.5873715124816447,
      "eval_fcm_dpo/beta": 0.0028311661444604397,
      "eval_logits/chosen": -0.44635966420173645,
      "eval_logits/rejected": -0.42632707953453064,
      "eval_logps/chosen": -410.6361999511719,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -548.2955932617188,
      "eval_loss": 0.579971969127655,
      "eval_margin_dpo/margin_mean": 129.91244506835938,
      "eval_margin_dpo/margin_std": 237.20855712890625,
      "eval_runtime": 39.3402,
      "eval_samples_per_second": 59.456,
      "eval_steps_per_second": 1.881,
      "step": 400
    },
    {
      "epoch": 0.5888399412628488,
      "fcm_dpo/beta": 0.0028878701850771904,
      "fcm_dpo/delta": 0.15109677612781525,
      "fcm_dpo/margin": 157.91604614257812,
      "fcm_dpo/q_t": 0.40476858615875244,
      "grad_norm": 35.52607727050781,
      "learning_rate": 2.1800473436235136e-07,
      "logits/chosen": -0.39845049381256104,
      "logits/rejected": -0.3905218243598938,
      "logps/chosen": -345.0669250488281,
      "logps/ref_chosen": -57.16303253173828,
      "logps/ref_rejected": -83.79249572753906,
      "logps/rejected": -529.6124267578125,
      "loss": 1.1349,
      "margin_dpo/margin_mean": 157.91604614257812,
      "margin_dpo/margin_std": 286.06787109375,
      "step": 401
    },
    {
      "epoch": 0.5903083700440529,
      "fcm_dpo/beta": 0.002814686391502619,
      "fcm_dpo/delta": -0.2406725436449051,
      "fcm_dpo/margin": 291.61102294921875,
      "fcm_dpo/q_t": 0.3189072012901306,
      "grad_norm": 26.38422203063965,
      "learning_rate": 2.1673238449588665e-07,
      "logits/chosen": -0.423443078994751,
      "logits/rejected": -0.3997713625431061,
      "logps/chosen": -261.9085998535156,
      "logps/ref_chosen": -50.74037170410156,
      "logps/ref_rejected": -81.0460433959961,
      "logps/rejected": -583.8253173828125,
      "loss": 0.814,
      "margin_dpo/margin_mean": 291.61102294921875,
      "margin_dpo/margin_std": 232.8087158203125,
      "step": 402
    },
    {
      "epoch": 0.591776798825257,
      "fcm_dpo/beta": 0.002770167775452137,
      "fcm_dpo/delta": -0.014987404458224773,
      "fcm_dpo/margin": 221.64427185058594,
      "fcm_dpo/q_t": 0.36400288343429565,
      "grad_norm": 28.025609970092773,
      "learning_rate": 2.154609112620295e-07,
      "logits/chosen": -0.3887700140476227,
      "logits/rejected": -0.3797057271003723,
      "logps/chosen": -287.8169860839844,
      "logps/ref_chosen": -47.14731216430664,
      "logps/ref_rejected": -77.2666015625,
      "logps/rejected": -539.58056640625,
      "loss": 0.9562,
      "margin_dpo/margin_mean": 221.64425659179688,
      "margin_dpo/margin_std": 230.0044708251953,
      "step": 403
    },
    {
      "epoch": 0.593245227606461,
      "fcm_dpo/beta": 0.0027550768572837114,
      "fcm_dpo/delta": -0.033548858016729355,
      "fcm_dpo/margin": 229.15975952148438,
      "fcm_dpo/q_t": 0.36247456073760986,
      "grad_norm": 29.232345581054688,
      "learning_rate": 2.1419034816528218e-07,
      "logits/chosen": -0.35594630241394043,
      "logits/rejected": -0.3386983871459961,
      "logps/chosen": -307.4197998046875,
      "logps/ref_chosen": -47.875274658203125,
      "logps/ref_rejected": -77.15499877929688,
      "logps/rejected": -565.8592529296875,
      "loss": 0.9693,
      "margin_dpo/margin_mean": 229.15975952148438,
      "margin_dpo/margin_std": 263.5882568359375,
      "step": 404
    },
    {
      "epoch": 0.5947136563876652,
      "fcm_dpo/beta": 0.0028160212095826864,
      "fcm_dpo/delta": 0.0803837925195694,
      "fcm_dpo/margin": 184.99761962890625,
      "fcm_dpo/q_t": 0.38957273960113525,
      "grad_norm": 46.26492691040039,
      "learning_rate": 2.129207286861638e-07,
      "logits/chosen": -0.3900597095489502,
      "logits/rejected": -0.37792617082595825,
      "logps/chosen": -363.26409912109375,
      "logps/ref_chosen": -65.16290283203125,
      "logps/ref_rejected": -87.18678283691406,
      "logps/rejected": -570.2855834960938,
      "loss": 1.0657,
      "margin_dpo/margin_mean": 184.99761962890625,
      "margin_dpo/margin_std": 266.91748046875,
      "step": 405
    },
    {
      "epoch": 0.5961820851688693,
      "fcm_dpo/beta": 0.002798373345285654,
      "fcm_dpo/delta": -0.047089073807001114,
      "fcm_dpo/margin": 229.93914794921875,
      "fcm_dpo/q_t": 0.3619244396686554,
      "grad_norm": 38.67859649658203,
      "learning_rate": 2.1165208628032861e-07,
      "logits/chosen": -0.4114735722541809,
      "logits/rejected": -0.4083394408226013,
      "logps/chosen": -326.46710205078125,
      "logps/ref_chosen": -49.740814208984375,
      "logps/ref_rejected": -92.07862854003906,
      "logps/rejected": -598.744140625,
      "loss": 0.9584,
      "margin_dpo/margin_mean": 229.93914794921875,
      "margin_dpo/margin_std": 259.556640625,
      "step": 406
    },
    {
      "epoch": 0.5976505139500734,
      "fcm_dpo/beta": 0.002859012223780155,
      "fcm_dpo/delta": 0.14943772554397583,
      "fcm_dpo/margin": 159.05093383789062,
      "fcm_dpo/q_t": 0.3990488648414612,
      "grad_norm": 51.8092041015625,
      "learning_rate": 2.1038445437768375e-07,
      "logits/chosen": -0.41653358936309814,
      "logits/rejected": -0.38666877150535583,
      "logps/chosen": -360.2216796875,
      "logps/ref_chosen": -56.33069610595703,
      "logps/ref_rejected": -77.51209259033203,
      "logps/rejected": -540.4539794921875,
      "loss": 1.1054,
      "margin_dpo/margin_mean": 159.05093383789062,
      "margin_dpo/margin_std": 249.28366088867188,
      "step": 407
    },
    {
      "epoch": 0.5991189427312775,
      "fcm_dpo/beta": 0.0029165763407945633,
      "fcm_dpo/delta": 0.1471884399652481,
      "fcm_dpo/margin": 157.62078857421875,
      "fcm_dpo/q_t": 0.39709725975990295,
      "grad_norm": 37.17081832885742,
      "learning_rate": 2.0911786638150872e-07,
      "logits/chosen": -0.43049564957618713,
      "logits/rejected": -0.4026661217212677,
      "logps/chosen": -369.4244384765625,
      "logps/ref_chosen": -69.789306640625,
      "logps/ref_rejected": -90.09693908691406,
      "logps/rejected": -547.3529052734375,
      "loss": 1.0566,
      "margin_dpo/margin_mean": 157.62078857421875,
      "margin_dpo/margin_std": 198.9236602783203,
      "step": 408
    },
    {
      "epoch": 0.6005873715124816,
      "fcm_dpo/beta": 0.0029886472038924694,
      "fcm_dpo/delta": 0.08729657530784607,
      "fcm_dpo/margin": 172.93191528320312,
      "fcm_dpo/q_t": 0.38554543256759644,
      "grad_norm": 46.94636154174805,
      "learning_rate": 2.0785235566757517e-07,
      "logits/chosen": -0.3939806818962097,
      "logits/rejected": -0.3780689239501953,
      "logps/chosen": -353.55657958984375,
      "logps/ref_chosen": -67.31744384765625,
      "logps/ref_rejected": -84.904296875,
      "logps/rejected": -544.0753173828125,
      "loss": 1.0233,
      "margin_dpo/margin_mean": 172.93191528320312,
      "margin_dpo/margin_std": 209.38848876953125,
      "step": 409
    },
    {
      "epoch": 0.6020558002936858,
      "fcm_dpo/beta": 0.0030058538541197777,
      "fcm_dpo/delta": 0.03576916456222534,
      "fcm_dpo/margin": 188.38836669921875,
      "fcm_dpo/q_t": 0.3729361891746521,
      "grad_norm": 36.37736129760742,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -0.3966106176376343,
      "logits/rejected": -0.39567676186561584,
      "logps/chosen": -317.6330261230469,
      "logps/ref_chosen": -51.465354919433594,
      "logps/ref_rejected": -83.198974609375,
      "logps/rejected": -537.7550048828125,
      "loss": 0.9902,
      "margin_dpo/margin_mean": 188.38836669921875,
      "margin_dpo/margin_std": 211.78468322753906,
      "step": 410
    },
    {
      "epoch": 0.6035242290748899,
      "fcm_dpo/beta": 0.0030007443856447935,
      "fcm_dpo/delta": 0.03217388689517975,
      "fcm_dpo/margin": 189.33251953125,
      "fcm_dpo/q_t": 0.37786054611206055,
      "grad_norm": 50.11013412475586,
      "learning_rate": 2.0532469944670343e-07,
      "logits/chosen": -0.3935418725013733,
      "logits/rejected": -0.3961487412452698,
      "logps/chosen": -325.54656982421875,
      "logps/ref_chosen": -52.30727005004883,
      "logps/ref_rejected": -80.69495391845703,
      "logps/rejected": -543.2667846679688,
      "loss": 1.0067,
      "margin_dpo/margin_mean": 189.33250427246094,
      "margin_dpo/margin_std": 229.78692626953125,
      "step": 411
    },
    {
      "epoch": 0.604992657856094,
      "fcm_dpo/beta": 0.0030541394371539354,
      "fcm_dpo/delta": 0.038746751844882965,
      "fcm_dpo/margin": 184.47250366210938,
      "fcm_dpo/q_t": 0.37744957208633423,
      "grad_norm": 60.06779861450195,
      "learning_rate": 2.0406262054585738e-07,
      "logits/chosen": -0.4395294785499573,
      "logits/rejected": -0.45951640605926514,
      "logps/chosen": -333.88372802734375,
      "logps/ref_chosen": -53.144126892089844,
      "logps/ref_rejected": -100.0608139038086,
      "logps/rejected": -565.27294921875,
      "loss": 1.0095,
      "margin_dpo/margin_mean": 184.47250366210938,
      "margin_dpo/margin_std": 230.4259490966797,
      "step": 412
    },
    {
      "epoch": 0.6064610866372981,
      "fcm_dpo/beta": 0.003098301589488983,
      "fcm_dpo/delta": 0.049318261444568634,
      "fcm_dpo/margin": 178.3287353515625,
      "fcm_dpo/q_t": 0.37706446647644043,
      "grad_norm": 51.303245544433594,
      "learning_rate": 2.0280175213768205e-07,
      "logits/chosen": -0.4190981388092041,
      "logits/rejected": -0.414195716381073,
      "logps/chosen": -356.578369140625,
      "logps/ref_chosen": -61.58196258544922,
      "logps/ref_rejected": -99.47340393066406,
      "logps/rejected": -572.7985229492188,
      "loss": 1.0196,
      "margin_dpo/margin_mean": 178.3287353515625,
      "margin_dpo/margin_std": 226.3348388671875,
      "step": 413
    },
    {
      "epoch": 0.6079295154185022,
      "fcm_dpo/beta": 0.0030625397339463234,
      "fcm_dpo/delta": -0.10935955494642258,
      "fcm_dpo/margin": 229.00564575195312,
      "fcm_dpo/q_t": 0.3444908857345581,
      "grad_norm": 45.593589782714844,
      "learning_rate": 2.0154212744723247e-07,
      "logits/chosen": -0.377156138420105,
      "logits/rejected": -0.3661056160926819,
      "logps/chosen": -312.2870178222656,
      "logps/ref_chosen": -46.63148498535156,
      "logps/ref_rejected": -87.64653015136719,
      "logps/rejected": -582.3077392578125,
      "loss": 0.9087,
      "margin_dpo/margin_mean": 229.00564575195312,
      "margin_dpo/margin_std": 222.5440216064453,
      "step": 414
    },
    {
      "epoch": 0.6093979441997063,
      "fcm_dpo/beta": 0.0030559361912310123,
      "fcm_dpo/delta": 0.09887358546257019,
      "fcm_dpo/margin": 165.65187072753906,
      "fcm_dpo/q_t": 0.3890509009361267,
      "grad_norm": 40.08522033691406,
      "learning_rate": 2.002837796667909e-07,
      "logits/chosen": -0.43923091888427734,
      "logits/rejected": -0.4275917410850525,
      "logps/chosen": -386.06353759765625,
      "logps/ref_chosen": -78.6182861328125,
      "logps/ref_rejected": -100.47752380371094,
      "logps/rejected": -573.5745849609375,
      "loss": 1.0445,
      "margin_dpo/margin_mean": 165.65187072753906,
      "margin_dpo/margin_std": 219.1352081298828,
      "step": 415
    },
    {
      "epoch": 0.6108663729809104,
      "fcm_dpo/beta": 0.003010387998074293,
      "fcm_dpo/delta": -0.19164395332336426,
      "fcm_dpo/margin": 258.158447265625,
      "fcm_dpo/q_t": 0.3281788229942322,
      "grad_norm": 34.98270797729492,
      "learning_rate": 1.990267419549914e-07,
      "logits/chosen": -0.40908828377723694,
      "logits/rejected": -0.39938801527023315,
      "logps/chosen": -331.32080078125,
      "logps/ref_chosen": -58.27912521362305,
      "logps/ref_rejected": -90.56871795654297,
      "logps/rejected": -621.7688598632812,
      "loss": 0.8446,
      "margin_dpo/margin_mean": 258.158447265625,
      "margin_dpo/margin_std": 212.94345092773438,
      "step": 416
    },
    {
      "epoch": 0.6123348017621145,
      "fcm_dpo/beta": 0.002966091735288501,
      "fcm_dpo/delta": -0.016102882102131844,
      "fcm_dpo/margin": 207.3574981689453,
      "fcm_dpo/q_t": 0.360975980758667,
      "grad_norm": 30.207260131835938,
      "learning_rate": 1.9777104743594686e-07,
      "logits/chosen": -0.3575609624385834,
      "logits/rejected": -0.3195306360721588,
      "logps/chosen": -326.48291015625,
      "logps/ref_chosen": -50.1987190246582,
      "logps/ref_rejected": -68.15184020996094,
      "logps/rejected": -551.7935791015625,
      "loss": 0.935,
      "margin_dpo/margin_mean": 207.35751342773438,
      "margin_dpo/margin_std": 190.68011474609375,
      "step": 417
    },
    {
      "epoch": 0.6138032305433186,
      "fcm_dpo/beta": 0.002962034661322832,
      "fcm_dpo/delta": -0.04637201130390167,
      "fcm_dpo/margin": 216.85317993164062,
      "fcm_dpo/q_t": 0.3637494742870331,
      "grad_norm": 25.070138931274414,
      "learning_rate": 1.965167291983757e-07,
      "logits/chosen": -0.46340426802635193,
      "logits/rejected": -0.4286314845085144,
      "logps/chosen": -367.762451171875,
      "logps/ref_chosen": -81.97846984863281,
      "logps/ref_rejected": -104.69148254394531,
      "logps/rejected": -607.32861328125,
      "loss": 0.9707,
      "margin_dpo/margin_mean": 216.85317993164062,
      "margin_dpo/margin_std": 255.80337524414062,
      "step": 418
    },
    {
      "epoch": 0.6152716593245228,
      "fcm_dpo/beta": 0.0028951792046427727,
      "fcm_dpo/delta": -0.08015348017215729,
      "fcm_dpo/margin": 233.0614013671875,
      "fcm_dpo/q_t": 0.3508976697921753,
      "grad_norm": 24.830020904541016,
      "learning_rate": 1.9526382029472988e-07,
      "logits/chosen": -0.39471104741096497,
      "logits/rejected": -0.3786032199859619,
      "logps/chosen": -315.84515380859375,
      "logps/ref_chosen": -52.948646545410156,
      "logps/ref_rejected": -91.58309936523438,
      "logps/rejected": -587.541015625,
      "loss": 0.9221,
      "margin_dpo/margin_mean": 233.0614013671875,
      "margin_dpo/margin_std": 232.522705078125,
      "step": 419
    },
    {
      "epoch": 0.6167400881057269,
      "fcm_dpo/beta": 0.0029447104316204786,
      "fcm_dpo/delta": 0.1310030221939087,
      "fcm_dpo/margin": 161.28619384765625,
      "fcm_dpo/q_t": 0.40055570006370544,
      "grad_norm": 52.39372253417969,
      "learning_rate": 1.9401235374032425e-07,
      "logits/chosen": -0.44154638051986694,
      "logits/rejected": -0.3922814130783081,
      "logps/chosen": -384.56658935546875,
      "logps/ref_chosen": -77.7699203491211,
      "logps/ref_rejected": -69.31985473632812,
      "logps/rejected": -537.4027099609375,
      "loss": 1.0989,
      "margin_dpo/margin_mean": 161.28619384765625,
      "margin_dpo/margin_std": 265.3115234375,
      "step": 420
    },
    {
      "epoch": 0.618208516886931,
      "fcm_dpo/beta": 0.0030234779696911573,
      "fcm_dpo/delta": 0.11057442426681519,
      "fcm_dpo/margin": 163.34596252441406,
      "fcm_dpo/q_t": 0.3888086974620819,
      "grad_norm": 33.57830810546875,
      "learning_rate": 1.9276236251246653e-07,
      "logits/chosen": -0.45666855573654175,
      "logits/rejected": -0.43738096952438354,
      "logps/chosen": -311.1793212890625,
      "logps/ref_chosen": -53.765865325927734,
      "logps/ref_rejected": -89.28144836425781,
      "logps/rejected": -510.0408630371094,
      "loss": 1.0469,
      "margin_dpo/margin_mean": 163.34596252441406,
      "margin_dpo/margin_std": 208.17727661132812,
      "step": 421
    },
    {
      "epoch": 0.6196769456681351,
      "fcm_dpo/beta": 0.0030494979582726955,
      "fcm_dpo/delta": 0.04655279591679573,
      "fcm_dpo/margin": 182.31634521484375,
      "fcm_dpo/q_t": 0.3765103816986084,
      "grad_norm": 35.29560852050781,
      "learning_rate": 1.9151387954958792e-07,
      "logits/chosen": -0.4439750909805298,
      "logits/rejected": -0.42819076776504517,
      "logps/chosen": -358.15643310546875,
      "logps/ref_chosen": -68.6337661743164,
      "logps/ref_rejected": -87.86351013183594,
      "logps/rejected": -559.7025146484375,
      "loss": 1.0112,
      "margin_dpo/margin_mean": 182.31634521484375,
      "margin_dpo/margin_std": 225.18934631347656,
      "step": 422
    },
    {
      "epoch": 0.6211453744493393,
      "fcm_dpo/beta": 0.003037120681256056,
      "fcm_dpo/delta": -0.029378529638051987,
      "fcm_dpo/margin": 206.6036376953125,
      "fcm_dpo/q_t": 0.3606680631637573,
      "grad_norm": 30.540699005126953,
      "learning_rate": 1.902669377503756e-07,
      "logits/chosen": -0.42354506254196167,
      "logits/rejected": -0.42130446434020996,
      "logps/chosen": -310.9537353515625,
      "logps/ref_chosen": -54.99030303955078,
      "logps/ref_rejected": -86.30654907226562,
      "logps/rejected": -548.8735961914062,
      "loss": 0.9463,
      "margin_dpo/margin_mean": 206.6036376953125,
      "margin_dpo/margin_std": 212.5541229248047,
      "step": 423
    },
    {
      "epoch": 0.6226138032305433,
      "fcm_dpo/beta": 0.0030368948355317116,
      "fcm_dpo/delta": 0.032666608691215515,
      "fcm_dpo/margin": 187.3394775390625,
      "fcm_dpo/q_t": 0.37737834453582764,
      "grad_norm": 32.117645263671875,
      "learning_rate": 1.890215699729057e-07,
      "logits/chosen": -0.4487920105457306,
      "logits/rejected": -0.41544824838638306,
      "logps/chosen": -282.232177734375,
      "logps/ref_chosen": -56.01192092895508,
      "logps/ref_rejected": -66.47896575927734,
      "logps/rejected": -480.0386962890625,
      "loss": 1.0066,
      "margin_dpo/margin_mean": 187.3394775390625,
      "margin_dpo/margin_std": 233.7342529296875,
      "step": 424
    },
    {
      "epoch": 0.6240822320117474,
      "fcm_dpo/beta": 0.0031079701147973537,
      "fcm_dpo/delta": 0.09193390607833862,
      "fcm_dpo/margin": 164.8294677734375,
      "fcm_dpo/q_t": 0.3847385048866272,
      "grad_norm": 31.431589126586914,
      "learning_rate": 1.8777780903377732e-07,
      "logits/chosen": -0.38863712549209595,
      "logits/rejected": -0.3869234323501587,
      "logps/chosen": -284.51678466796875,
      "logps/ref_chosen": -46.86899948120117,
      "logps/ref_rejected": -95.92545318603516,
      "logps/rejected": -498.4027099609375,
      "loss": 1.0322,
      "margin_dpo/margin_mean": 164.8294677734375,
      "margin_dpo/margin_std": 204.4449920654297,
      "step": 425
    },
    {
      "epoch": 0.6255506607929515,
      "fcm_dpo/beta": 0.0031212703324854374,
      "fcm_dpo/delta": -0.033512186259031296,
      "fcm_dpo/margin": 201.99557495117188,
      "fcm_dpo/q_t": 0.3606271743774414,
      "grad_norm": 38.98627853393555,
      "learning_rate": 1.8653568770724803e-07,
      "logits/chosen": -0.4529041051864624,
      "logits/rejected": -0.4022233486175537,
      "logps/chosen": -299.26812744140625,
      "logps/ref_chosen": -76.58354187011719,
      "logps/ref_rejected": -81.26658630371094,
      "logps/rejected": -505.94671630859375,
      "loss": 0.9505,
      "margin_dpo/margin_mean": 201.99557495117188,
      "margin_dpo/margin_std": 206.97454833984375,
      "step": 426
    },
    {
      "epoch": 0.6270190895741556,
      "fcm_dpo/beta": 0.003162715584039688,
      "fcm_dpo/delta": 0.15372638404369354,
      "fcm_dpo/margin": 143.31716918945312,
      "fcm_dpo/q_t": 0.3991009294986725,
      "grad_norm": 34.75946044921875,
      "learning_rate": 1.8529523872436977e-07,
      "logits/chosen": -0.45945197343826294,
      "logits/rejected": -0.4228222370147705,
      "logps/chosen": -276.22698974609375,
      "logps/ref_chosen": -64.8538818359375,
      "logps/ref_rejected": -78.5660171508789,
      "logps/rejected": -433.2562561035156,
      "loss": 1.0598,
      "margin_dpo/margin_mean": 143.31715393066406,
      "margin_dpo/margin_std": 183.68850708007812,
      "step": 427
    },
    {
      "epoch": 0.6284875183553598,
      "fcm_dpo/beta": 0.003189402399584651,
      "fcm_dpo/delta": 0.009054990485310555,
      "fcm_dpo/margin": 185.44857788085938,
      "fcm_dpo/q_t": 0.37188664078712463,
      "grad_norm": 29.212984085083008,
      "learning_rate": 1.8405649477212697e-07,
      "logits/chosen": -0.4812716245651245,
      "logits/rejected": -0.47259581089019775,
      "logps/chosen": -344.0758972167969,
      "logps/ref_chosen": -62.63666534423828,
      "logps/ref_rejected": -103.28181457519531,
      "logps/rejected": -570.169677734375,
      "loss": 1.0084,
      "margin_dpo/margin_mean": 185.44857788085938,
      "margin_dpo/margin_std": 239.06033325195312,
      "step": 428
    },
    {
      "epoch": 0.6299559471365639,
      "fcm_dpo/beta": 0.0032644562888890505,
      "fcm_dpo/delta": 0.08702676743268967,
      "fcm_dpo/margin": 157.81967163085938,
      "fcm_dpo/q_t": 0.3883768320083618,
      "grad_norm": 37.52164077758789,
      "learning_rate": 1.828194884925749e-07,
      "logits/chosen": -0.46438342332839966,
      "logits/rejected": -0.4186452627182007,
      "logps/chosen": -361.667236328125,
      "logps/ref_chosen": -81.23401641845703,
      "logps/ref_rejected": -91.79493713378906,
      "logps/rejected": -530.0478515625,
      "loss": 1.0732,
      "margin_dpo/margin_mean": 157.81967163085938,
      "margin_dpo/margin_std": 228.81576538085938,
      "step": 429
    },
    {
      "epoch": 0.631424375917768,
      "fcm_dpo/beta": 0.0032723741605877876,
      "fcm_dpo/delta": 0.02481374703347683,
      "fcm_dpo/margin": 176.12640380859375,
      "fcm_dpo/q_t": 0.37325915694236755,
      "grad_norm": 30.306285858154297,
      "learning_rate": 1.8158425248197928e-07,
      "logits/chosen": -0.4715721011161804,
      "logits/rejected": -0.46700191497802734,
      "logps/chosen": -303.973388671875,
      "logps/ref_chosen": -60.920326232910156,
      "logps/ref_rejected": -104.42280578613281,
      "logps/rejected": -523.602294921875,
      "loss": 0.9759,
      "margin_dpo/margin_mean": 176.12640380859375,
      "margin_dpo/margin_std": 191.27938842773438,
      "step": 430
    },
    {
      "epoch": 0.6328928046989721,
      "fcm_dpo/beta": 0.0031659076921641827,
      "fcm_dpo/delta": -0.19049368798732758,
      "fcm_dpo/margin": 244.8009033203125,
      "fcm_dpo/q_t": 0.3305261731147766,
      "grad_norm": 26.351268768310547,
      "learning_rate": 1.8035081928995788e-07,
      "logits/chosen": -0.3872508406639099,
      "logits/rejected": -0.3752667307853699,
      "logps/chosen": -293.5878601074219,
      "logps/ref_chosen": -57.34874725341797,
      "logps/ref_rejected": -92.84022521972656,
      "logps/rejected": -573.8802490234375,
      "loss": 0.8614,
      "margin_dpo/margin_mean": 244.8009033203125,
      "margin_dpo/margin_std": 219.5264892578125,
      "step": 431
    },
    {
      "epoch": 0.6343612334801763,
      "fcm_dpo/beta": 0.0031368029303848743,
      "fcm_dpo/delta": -0.083712637424469,
      "fcm_dpo/margin": 215.79534912109375,
      "fcm_dpo/q_t": 0.35200947523117065,
      "grad_norm": 28.590843200683594,
      "learning_rate": 1.791192214186223e-07,
      "logits/chosen": -0.4346217215061188,
      "logits/rejected": -0.4086345434188843,
      "logps/chosen": -320.74261474609375,
      "logps/ref_chosen": -71.07479095458984,
      "logps/ref_rejected": -98.57952880859375,
      "logps/rejected": -564.042724609375,
      "loss": 0.9165,
      "margin_dpo/margin_mean": 215.79534912109375,
      "margin_dpo/margin_std": 203.4432373046875,
      "step": 432
    },
    {
      "epoch": 0.6358296622613803,
      "fcm_dpo/beta": 0.0031476817093789577,
      "fcm_dpo/delta": 0.1059950441122055,
      "fcm_dpo/margin": 158.52908325195312,
      "fcm_dpo/q_t": 0.3886904716491699,
      "grad_norm": 33.54300308227539,
      "learning_rate": 1.7788949132172193e-07,
      "logits/chosen": -0.3967306911945343,
      "logits/rejected": -0.37517887353897095,
      "logps/chosen": -373.89337158203125,
      "logps/ref_chosen": -58.273193359375,
      "logps/ref_rejected": -95.95089721679688,
      "logps/rejected": -570.1001586914062,
      "loss": 1.0645,
      "margin_dpo/margin_mean": 158.52908325195312,
      "margin_dpo/margin_std": 224.95269775390625,
      "step": 433
    },
    {
      "epoch": 0.6372980910425844,
      "fcm_dpo/beta": 0.0031643174588680267,
      "fcm_dpo/delta": 0.033846087753772736,
      "fcm_dpo/margin": 179.49761962890625,
      "fcm_dpo/q_t": 0.38102903962135315,
      "grad_norm": 28.94676399230957,
      "learning_rate": 1.7666166140378853e-07,
      "logits/chosen": -0.42005491256713867,
      "logits/rejected": -0.40054017305374146,
      "logps/chosen": -327.9237060546875,
      "logps/ref_chosen": -61.97370147705078,
      "logps/ref_rejected": -78.49861145019531,
      "logps/rejected": -523.9462280273438,
      "loss": 1.0064,
      "margin_dpo/margin_mean": 179.49761962890625,
      "margin_dpo/margin_std": 235.10498046875,
      "step": 434
    },
    {
      "epoch": 0.6387665198237885,
      "fcm_dpo/beta": 0.0031498861499130726,
      "fcm_dpo/delta": -0.05088559538125992,
      "fcm_dpo/margin": 205.5423583984375,
      "fcm_dpo/q_t": 0.3588791489601135,
      "grad_norm": 28.799325942993164,
      "learning_rate": 1.7543576401928218e-07,
      "logits/chosen": -0.4303112328052521,
      "logits/rejected": -0.40882986783981323,
      "logps/chosen": -297.64056396484375,
      "logps/ref_chosen": -51.502052307128906,
      "logps/ref_rejected": -87.56689453125,
      "logps/rejected": -539.247802734375,
      "loss": 0.9575,
      "margin_dpo/margin_mean": 205.54234313964844,
      "margin_dpo/margin_std": 227.2718048095703,
      "step": 435
    },
    {
      "epoch": 0.6402349486049926,
      "fcm_dpo/beta": 0.003141575027257204,
      "fcm_dpo/delta": -0.0027038920670747757,
      "fcm_dpo/margin": 191.78115844726562,
      "fcm_dpo/q_t": 0.36727362871170044,
      "grad_norm": 53.2458610534668,
      "learning_rate": 1.742118314717391e-07,
      "logits/chosen": -0.4502115249633789,
      "logits/rejected": -0.4046837091445923,
      "logps/chosen": -322.1340026855469,
      "logps/ref_chosen": -71.40371704101562,
      "logps/ref_rejected": -82.72775268554688,
      "logps/rejected": -525.2391967773438,
      "loss": 0.9633,
      "margin_dpo/margin_mean": 191.78115844726562,
      "margin_dpo/margin_std": 206.23292541503906,
      "step": 436
    },
    {
      "epoch": 0.6417033773861968,
      "fcm_dpo/beta": 0.0031595812179148197,
      "fcm_dpo/delta": 0.026493970304727554,
      "fcm_dpo/margin": 181.99075317382812,
      "fcm_dpo/q_t": 0.372455894947052,
      "grad_norm": 30.233713150024414,
      "learning_rate": 1.7298989601292036e-07,
      "logits/chosen": -0.43642017245292664,
      "logits/rejected": -0.4024639129638672,
      "logps/chosen": -321.0672912597656,
      "logps/ref_chosen": -64.7442626953125,
      "logps/ref_rejected": -82.04356384277344,
      "logps/rejected": -520.3573608398438,
      "loss": 0.9863,
      "margin_dpo/margin_mean": 181.99075317382812,
      "margin_dpo/margin_std": 205.34609985351562,
      "step": 437
    },
    {
      "epoch": 0.6431718061674009,
      "fcm_dpo/beta": 0.00310856057330966,
      "fcm_dpo/delta": -0.08261559903621674,
      "fcm_dpo/margin": 217.52000427246094,
      "fcm_dpo/q_t": 0.34879451990127563,
      "grad_norm": 41.74407196044922,
      "learning_rate": 1.7176998984196144e-07,
      "logits/chosen": -0.3906601667404175,
      "logits/rejected": -0.35248345136642456,
      "logps/chosen": -311.93084716796875,
      "logps/ref_chosen": -59.0186653137207,
      "logps/ref_rejected": -83.07682800292969,
      "logps/rejected": -553.5089721679688,
      "loss": 0.9029,
      "margin_dpo/margin_mean": 217.51998901367188,
      "margin_dpo/margin_std": 194.8534393310547,
      "step": 438
    },
    {
      "epoch": 0.644640234948605,
      "fcm_dpo/beta": 0.0031026997603476048,
      "fcm_dpo/delta": 0.05992882698774338,
      "fcm_dpo/margin": 174.4813232421875,
      "fcm_dpo/q_t": 0.38289695978164673,
      "grad_norm": 30.436159133911133,
      "learning_rate": 1.7055214510452458e-07,
      "logits/chosen": -0.38404345512390137,
      "logits/rejected": -0.37690842151641846,
      "logps/chosen": -328.8792724609375,
      "logps/ref_chosen": -53.78407669067383,
      "logps/ref_rejected": -83.98545837402344,
      "logps/rejected": -533.56201171875,
      "loss": 1.0235,
      "margin_dpo/margin_mean": 174.4813232421875,
      "margin_dpo/margin_std": 211.0643768310547,
      "step": 439
    },
    {
      "epoch": 0.6461086637298091,
      "fcm_dpo/beta": 0.0031752916984260082,
      "fcm_dpo/delta": 0.037204399704933167,
      "fcm_dpo/margin": 177.81304931640625,
      "fcm_dpo/q_t": 0.3790428638458252,
      "grad_norm": 36.150543212890625,
      "learning_rate": 1.6933639389195134e-07,
      "logits/chosen": -0.45105117559432983,
      "logits/rejected": -0.4272584915161133,
      "logps/chosen": -372.9900207519531,
      "logps/ref_chosen": -78.56671905517578,
      "logps/ref_rejected": -96.49775695800781,
      "logps/rejected": -568.734130859375,
      "loss": 1.0121,
      "margin_dpo/margin_mean": 177.81304931640625,
      "margin_dpo/margin_std": 225.9781494140625,
      "step": 440
    },
    {
      "epoch": 0.6475770925110133,
      "fcm_dpo/beta": 0.0031558992341160774,
      "fcm_dpo/delta": -0.05471445247530937,
      "fcm_dpo/margin": 206.31964111328125,
      "fcm_dpo/q_t": 0.36530476808547974,
      "grad_norm": 41.47607421875,
      "learning_rate": 1.681227682404166e-07,
      "logits/chosen": -0.42267531156539917,
      "logits/rejected": -0.4021593928337097,
      "logps/chosen": -371.01458740234375,
      "logps/ref_chosen": -60.824440002441406,
      "logps/ref_rejected": -96.47080993652344,
      "logps/rejected": -612.9805908203125,
      "loss": 0.9977,
      "margin_dpo/margin_mean": 206.31964111328125,
      "margin_dpo/margin_std": 277.037841796875,
      "step": 441
    },
    {
      "epoch": 0.6490455212922174,
      "fcm_dpo/beta": 0.0031145289540290833,
      "fcm_dpo/delta": -0.10305148363113403,
      "fcm_dpo/margin": 223.28561401367188,
      "fcm_dpo/q_t": 0.3511419892311096,
      "grad_norm": 28.58584213256836,
      "learning_rate": 1.669113001300851e-07,
      "logits/chosen": -0.4091102182865143,
      "logits/rejected": -0.3904363512992859,
      "logps/chosen": -326.81689453125,
      "logps/ref_chosen": -47.01121520996094,
      "logps/ref_rejected": -76.53926086425781,
      "logps/rejected": -579.6304931640625,
      "loss": 0.9206,
      "margin_dpo/margin_mean": 223.28561401367188,
      "margin_dpo/margin_std": 229.4536590576172,
      "step": 442
    },
    {
      "epoch": 0.6505139500734214,
      "fcm_dpo/beta": 0.003117314074188471,
      "fcm_dpo/delta": 0.14508318901062012,
      "fcm_dpo/margin": 148.06362915039062,
      "fcm_dpo/q_t": 0.40172627568244934,
      "grad_norm": 37.50111389160156,
      "learning_rate": 1.6570202148426815e-07,
      "logits/chosen": -0.408527135848999,
      "logits/rejected": -0.38471612334251404,
      "logps/chosen": -397.83453369140625,
      "logps/ref_chosen": -71.27301788330078,
      "logps/ref_rejected": -86.679931640625,
      "logps/rejected": -561.3050537109375,
      "loss": 1.1228,
      "margin_dpo/margin_mean": 148.06362915039062,
      "margin_dpo/margin_std": 257.1429138183594,
      "step": 443
    },
    {
      "epoch": 0.6519823788546255,
      "fcm_dpo/beta": 0.00309735513292253,
      "fcm_dpo/delta": -0.12234368920326233,
      "fcm_dpo/margin": 230.2260284423828,
      "fcm_dpo/q_t": 0.34877198934555054,
      "grad_norm": 35.36920928955078,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": -0.36250776052474976,
      "logits/rejected": -0.3549155592918396,
      "logps/chosen": -372.5677185058594,
      "logps/ref_chosen": -57.213706970214844,
      "logps/ref_rejected": -97.25489807128906,
      "logps/rejected": -642.8349609375,
      "loss": 0.93,
      "margin_dpo/margin_mean": 230.22604370117188,
      "margin_dpo/margin_std": 260.1424865722656,
      "step": 444
    },
    {
      "epoch": 0.6534508076358296,
      "fcm_dpo/beta": 0.003053246531635523,
      "fcm_dpo/delta": -0.07174232602119446,
      "fcm_dpo/margin": 218.43179321289062,
      "fcm_dpo/q_t": 0.3583186864852905,
      "grad_norm": 34.992069244384766,
      "learning_rate": 1.6329015999011182e-07,
      "logits/chosen": -0.39381998777389526,
      "logits/rejected": -0.37214112281799316,
      "logps/chosen": -351.34881591796875,
      "logps/ref_chosen": -67.29979705810547,
      "logps/ref_rejected": -92.68267059326172,
      "logps/rejected": -595.1634521484375,
      "loss": 0.9516,
      "margin_dpo/margin_mean": 218.43177795410156,
      "margin_dpo/margin_std": 246.44183349609375,
      "step": 445
    },
    {
      "epoch": 0.6549192364170338,
      "fcm_dpo/beta": 0.0030171829275786877,
      "fcm_dpo/delta": -0.10393056273460388,
      "fcm_dpo/margin": 230.83352661132812,
      "fcm_dpo/q_t": 0.34617435932159424,
      "grad_norm": 26.35415267944336,
      "learning_rate": 1.6208764069656578e-07,
      "logits/chosen": -0.43713754415512085,
      "logits/rejected": -0.43820661306381226,
      "logps/chosen": -321.90740966796875,
      "logps/ref_chosen": -59.098487854003906,
      "logps/ref_rejected": -101.26419067382812,
      "logps/rejected": -594.9066162109375,
      "loss": 0.8993,
      "margin_dpo/margin_mean": 230.83352661132812,
      "margin_dpo/margin_std": 212.1956024169922,
      "step": 446
    },
    {
      "epoch": 0.6563876651982379,
      "fcm_dpo/beta": 0.0029287966899573803,
      "fcm_dpo/delta": -0.08693499863147736,
      "fcm_dpo/margin": 232.25527954101562,
      "fcm_dpo/q_t": 0.35467979311943054,
      "grad_norm": 32.44998550415039,
      "learning_rate": 1.608874379754465e-07,
      "logits/chosen": -0.421570360660553,
      "logits/rejected": -0.42356860637664795,
      "logps/chosen": -343.432861328125,
      "logps/ref_chosen": -56.07533264160156,
      "logps/ref_rejected": -98.69475555419922,
      "logps/rejected": -618.3075561523438,
      "loss": 0.9272,
      "margin_dpo/margin_mean": 232.2552947998047,
      "margin_dpo/margin_std": 253.9686737060547,
      "step": 447
    },
    {
      "epoch": 0.657856093979442,
      "fcm_dpo/beta": 0.002891901880502701,
      "fcm_dpo/delta": -0.11446341872215271,
      "fcm_dpo/margin": 244.29922485351562,
      "fcm_dpo/q_t": 0.34647947549819946,
      "grad_norm": 37.00398254394531,
      "learning_rate": 1.5968958345321177e-07,
      "logits/chosen": -0.3976057767868042,
      "logits/rejected": -0.39330196380615234,
      "logps/chosen": -364.0402526855469,
      "logps/ref_chosen": -60.00384521484375,
      "logps/ref_rejected": -102.26465606689453,
      "logps/rejected": -650.6002807617188,
      "loss": 0.9,
      "margin_dpo/margin_mean": 244.29922485351562,
      "margin_dpo/margin_std": 234.91090393066406,
      "step": 448
    },
    {
      "epoch": 0.6593245227606461,
      "fcm_dpo/beta": 0.002847596537321806,
      "fcm_dpo/delta": -0.026260854676365852,
      "fcm_dpo/margin": 219.3227081298828,
      "fcm_dpo/q_t": 0.3709370791912079,
      "grad_norm": 35.01579284667969,
      "learning_rate": 1.584941086944423e-07,
      "logits/chosen": -0.40996721386909485,
      "logits/rejected": -0.3869406580924988,
      "logps/chosen": -385.76275634765625,
      "logps/ref_chosen": -67.52661895751953,
      "logps/ref_rejected": -88.59690856933594,
      "logps/rejected": -626.15576171875,
      "loss": 1.0099,
      "margin_dpo/margin_mean": 219.3227081298828,
      "margin_dpo/margin_std": 304.9261474609375,
      "step": 449
    },
    {
      "epoch": 0.6607929515418502,
      "fcm_dpo/beta": 0.0028021461330354214,
      "fcm_dpo/delta": -0.12278854101896286,
      "fcm_dpo/margin": 254.86483764648438,
      "fcm_dpo/q_t": 0.33995717763900757,
      "grad_norm": 35.38984680175781,
      "learning_rate": 1.573010452010098e-07,
      "logits/chosen": -0.4261053204536438,
      "logits/rejected": -0.42196720838546753,
      "logps/chosen": -306.01397705078125,
      "logps/ref_chosen": -57.10811996459961,
      "logps/ref_rejected": -102.75494384765625,
      "logps/rejected": -606.525634765625,
      "loss": 0.8839,
      "margin_dpo/margin_mean": 254.86483764648438,
      "margin_dpo/margin_std": 228.66526794433594,
      "step": 450
    },
    {
      "epoch": 0.6622613803230544,
      "fcm_dpo/beta": 0.002811973448842764,
      "fcm_dpo/delta": 0.04851696640253067,
      "fcm_dpo/margin": 196.84996032714844,
      "fcm_dpo/q_t": 0.3752724528312683,
      "grad_norm": 42.07421112060547,
      "learning_rate": 1.5611042441124687e-07,
      "logits/chosen": -0.415117084980011,
      "logits/rejected": -0.38188982009887695,
      "logps/chosen": -367.1541748046875,
      "logps/ref_chosen": -58.46883010864258,
      "logps/ref_rejected": -72.92941284179688,
      "logps/rejected": -578.4647216796875,
      "loss": 1.0271,
      "margin_dpo/margin_mean": 196.8499755859375,
      "margin_dpo/margin_std": 257.9520263671875,
      "step": 451
    },
    {
      "epoch": 0.6637298091042585,
      "fcm_dpo/beta": 0.002792142331600189,
      "fcm_dpo/delta": -0.020108355209231377,
      "fcm_dpo/margin": 221.60552978515625,
      "fcm_dpo/q_t": 0.36079761385917664,
      "grad_norm": 25.38531494140625,
      "learning_rate": 1.549222776991186e-07,
      "logits/chosen": -0.3913271129131317,
      "logits/rejected": -0.3999473750591278,
      "logps/chosen": -272.1903076171875,
      "logps/ref_chosen": -50.39055252075195,
      "logps/ref_rejected": -97.77142333984375,
      "logps/rejected": -541.1766967773438,
      "loss": 0.9344,
      "margin_dpo/margin_mean": 221.60552978515625,
      "margin_dpo/margin_std": 206.4978790283203,
      "step": 452
    },
    {
      "epoch": 0.6651982378854625,
      "fcm_dpo/beta": 0.0028024991042912006,
      "fcm_dpo/delta": 0.04751697555184364,
      "fcm_dpo/margin": 198.021484375,
      "fcm_dpo/q_t": 0.37886279821395874,
      "grad_norm": 26.131412506103516,
      "learning_rate": 1.5373663637339584e-07,
      "logits/chosen": -0.4181209206581116,
      "logits/rejected": -0.38588768243789673,
      "logps/chosen": -332.87738037109375,
      "logps/ref_chosen": -57.71485137939453,
      "logps/ref_rejected": -82.20741271972656,
      "logps/rejected": -555.3914184570312,
      "loss": 0.9948,
      "margin_dpo/margin_mean": 198.021484375,
      "margin_dpo/margin_std": 228.34547424316406,
      "step": 453
    },
    {
      "epoch": 0.6666666666666666,
      "fcm_dpo/beta": 0.0027867662720382214,
      "fcm_dpo/delta": -0.059162285178899765,
      "fcm_dpo/margin": 234.98402404785156,
      "fcm_dpo/q_t": 0.35987627506256104,
      "grad_norm": 29.379961013793945,
      "learning_rate": 1.5255353167683017e-07,
      "logits/chosen": -0.40170955657958984,
      "logits/rejected": -0.3783451318740845,
      "logps/chosen": -366.53466796875,
      "logps/ref_chosen": -60.945648193359375,
      "logps/ref_rejected": -84.95079040527344,
      "logps/rejected": -625.5238037109375,
      "loss": 0.9475,
      "margin_dpo/margin_mean": 234.98403930664062,
      "margin_dpo/margin_std": 266.738037109375,
      "step": 454
    },
    {
      "epoch": 0.6681350954478708,
      "fcm_dpo/beta": 0.002752738306298852,
      "fcm_dpo/delta": -0.13085488975048065,
      "fcm_dpo/margin": 262.0626220703125,
      "fcm_dpo/q_t": 0.34647172689437866,
      "grad_norm": 29.516231536865234,
      "learning_rate": 1.5137299478533064e-07,
      "logits/chosen": -0.4226793646812439,
      "logits/rejected": -0.4276910424232483,
      "logps/chosen": -327.73028564453125,
      "logps/ref_chosen": -44.88671112060547,
      "logps/ref_rejected": -115.30147552490234,
      "logps/rejected": -660.2076416015625,
      "loss": 0.9117,
      "margin_dpo/margin_mean": 262.0626220703125,
      "margin_dpo/margin_std": 271.49981689453125,
      "step": 455
    },
    {
      "epoch": 0.6696035242290749,
      "fcm_dpo/beta": 0.0026842241641134024,
      "fcm_dpo/delta": -0.08822280168533325,
      "fcm_dpo/margin": 254.17987060546875,
      "fcm_dpo/q_t": 0.3486155569553375,
      "grad_norm": 30.855884552001953,
      "learning_rate": 1.5019505680714232e-07,
      "logits/chosen": -0.42566657066345215,
      "logits/rejected": -0.4341641068458557,
      "logps/chosen": -339.21624755859375,
      "logps/ref_chosen": -57.036781311035156,
      "logps/ref_rejected": -105.21784210205078,
      "logps/rejected": -641.5771484375,
      "loss": 0.8963,
      "margin_dpo/margin_mean": 254.17987060546875,
      "margin_dpo/margin_std": 227.32037353515625,
      "step": 456
    },
    {
      "epoch": 0.671071953010279,
      "fcm_dpo/beta": 0.002638116478919983,
      "fcm_dpo/delta": -0.051346272230148315,
      "fcm_dpo/margin": 245.4560546875,
      "fcm_dpo/q_t": 0.3536643385887146,
      "grad_norm": 43.32398223876953,
      "learning_rate": 1.4901974878202627e-07,
      "logits/chosen": -0.4410599172115326,
      "logits/rejected": -0.4189218282699585,
      "logps/chosen": -335.3031311035156,
      "logps/ref_chosen": -54.24253845214844,
      "logps/ref_rejected": -85.10956573486328,
      "logps/rejected": -611.626220703125,
      "loss": 0.9209,
      "margin_dpo/margin_mean": 245.4560546875,
      "margin_dpo/margin_std": 225.08139038085938,
      "step": 457
    },
    {
      "epoch": 0.6725403817914831,
      "fcm_dpo/beta": 0.0026078918017446995,
      "fcm_dpo/delta": -0.07165485620498657,
      "fcm_dpo/margin": 255.62677001953125,
      "fcm_dpo/q_t": 0.35423409938812256,
      "grad_norm": 27.470394134521484,
      "learning_rate": 1.4784710168044212e-07,
      "logits/chosen": -0.42076390981674194,
      "logits/rejected": -0.40564727783203125,
      "logps/chosen": -329.96343994140625,
      "logps/ref_chosen": -55.40888214111328,
      "logps/ref_rejected": -97.68325805664062,
      "logps/rejected": -627.8646240234375,
      "loss": 0.9208,
      "margin_dpo/margin_mean": 255.6267852783203,
      "margin_dpo/margin_std": 259.33697509765625,
      "step": 458
    },
    {
      "epoch": 0.6740088105726872,
      "fcm_dpo/beta": 0.0025895205326378345,
      "fcm_dpo/delta": -0.016186371445655823,
      "fcm_dpo/margin": 237.4794158935547,
      "fcm_dpo/q_t": 0.36387407779693604,
      "grad_norm": 28.839862823486328,
      "learning_rate": 1.466771464027316e-07,
      "logits/chosen": -0.4365195035934448,
      "logits/rejected": -0.43345052003860474,
      "logps/chosen": -352.8772277832031,
      "logps/ref_chosen": -46.55748748779297,
      "logps/ref_rejected": -86.16854095458984,
      "logps/rejected": -629.9677124023438,
      "loss": 0.9595,
      "margin_dpo/margin_mean": 237.4794158935547,
      "margin_dpo/margin_std": 249.88954162597656,
      "step": 459
    },
    {
      "epoch": 0.6754772393538914,
      "fcm_dpo/beta": 0.0025813106913119555,
      "fcm_dpo/delta": -0.04352150857448578,
      "fcm_dpo/margin": 248.228271484375,
      "fcm_dpo/q_t": 0.35783928632736206,
      "grad_norm": 29.873188018798828,
      "learning_rate": 1.4550991377830423e-07,
      "logits/chosen": -0.45997023582458496,
      "logits/rejected": -0.47314104437828064,
      "logps/chosen": -400.25897216796875,
      "logps/ref_chosen": -51.63489532470703,
      "logps/ref_rejected": -104.11935424804688,
      "logps/rejected": -700.9716796875,
      "loss": 0.9376,
      "margin_dpo/margin_mean": 248.22825622558594,
      "margin_dpo/margin_std": 249.94369506835938,
      "step": 460
    },
    {
      "epoch": 0.6769456681350955,
      "fcm_dpo/beta": 0.002606459427624941,
      "fcm_dpo/delta": 0.06772121787071228,
      "fcm_dpo/margin": 205.47274780273438,
      "fcm_dpo/q_t": 0.3833613097667694,
      "grad_norm": 33.35320281982422,
      "learning_rate": 1.4434543456482518e-07,
      "logits/chosen": -0.49127471446990967,
      "logits/rejected": -0.4890371561050415,
      "logps/chosen": -408.4715576171875,
      "logps/ref_chosen": -55.18195724487305,
      "logps/ref_rejected": -86.47689819335938,
      "logps/rejected": -645.2392578125,
      "loss": 1.0183,
      "margin_dpo/margin_mean": 205.47274780273438,
      "margin_dpo/margin_std": 255.91586303710938,
      "step": 461
    },
    {
      "epoch": 0.6784140969162996,
      "fcm_dpo/beta": 0.002664759289473295,
      "fcm_dpo/delta": 0.15602310001850128,
      "fcm_dpo/margin": 169.3271026611328,
      "fcm_dpo/q_t": 0.40088993310928345,
      "grad_norm": 56.87949752807617,
      "learning_rate": 1.4318373944740484e-07,
      "logits/chosen": -0.5266599655151367,
      "logits/rejected": -0.5125139951705933,
      "logps/chosen": -429.41357421875,
      "logps/ref_chosen": -69.92803192138672,
      "logps/ref_rejected": -78.84111022949219,
      "logps/rejected": -607.6537475585938,
      "loss": 1.0795,
      "margin_dpo/margin_mean": 169.3271026611328,
      "margin_dpo/margin_std": 241.06423950195312,
      "step": 462
    },
    {
      "epoch": 0.6798825256975036,
      "fcm_dpo/beta": 0.0027028373442590237,
      "fcm_dpo/delta": -0.00691063329577446,
      "fcm_dpo/margin": 224.18450927734375,
      "fcm_dpo/q_t": 0.37208259105682373,
      "grad_norm": 59.45559310913086,
      "learning_rate": 1.4202485903778976e-07,
      "logits/chosen": -0.48024556040763855,
      "logits/rejected": -0.4725441634654999,
      "logps/chosen": -393.7677001953125,
      "logps/ref_chosen": -55.27437210083008,
      "logps/ref_rejected": -89.02497863769531,
      "logps/rejected": -651.7028198242188,
      "loss": 1.0023,
      "margin_dpo/margin_mean": 224.18450927734375,
      "margin_dpo/margin_std": 288.2208557128906,
      "step": 463
    },
    {
      "epoch": 0.6813509544787077,
      "fcm_dpo/beta": 0.002575058490037918,
      "fcm_dpo/delta": -0.24060693383216858,
      "fcm_dpo/margin": 317.8370361328125,
      "fcm_dpo/q_t": 0.32363659143447876,
      "grad_norm": 35.653228759765625,
      "learning_rate": 1.4086882387355658e-07,
      "logits/chosen": -0.4769009053707123,
      "logits/rejected": -0.5046235918998718,
      "logps/chosen": -376.1098327636719,
      "logps/ref_chosen": -50.91230010986328,
      "logps/ref_rejected": -102.4893798828125,
      "logps/rejected": -745.52392578125,
      "loss": 0.8444,
      "margin_dpo/margin_mean": 317.8370361328125,
      "margin_dpo/margin_std": 292.25225830078125,
      "step": 464
    },
    {
      "epoch": 0.6828193832599119,
      "fcm_dpo/beta": 0.0025294728111475706,
      "fcm_dpo/delta": -0.11184176802635193,
      "fcm_dpo/margin": 278.3428039550781,
      "fcm_dpo/q_t": 0.34381166100502014,
      "grad_norm": 32.800621032714844,
      "learning_rate": 1.3971566441730714e-07,
      "logits/chosen": -0.4514947533607483,
      "logits/rejected": -0.4547080993652344,
      "logps/chosen": -367.2335205078125,
      "logps/ref_chosen": -60.116851806640625,
      "logps/ref_rejected": -113.94602966308594,
      "logps/rejected": -699.405517578125,
      "loss": 0.8945,
      "margin_dpo/margin_mean": 278.3428039550781,
      "margin_dpo/margin_std": 257.3295593261719,
      "step": 465
    },
    {
      "epoch": 0.684287812041116,
      "fcm_dpo/beta": 0.0025112878065556288,
      "fcm_dpo/delta": 0.04471685737371445,
      "fcm_dpo/margin": 222.06112670898438,
      "fcm_dpo/q_t": 0.37593716382980347,
      "grad_norm": 33.81809616088867,
      "learning_rate": 1.3856541105586545e-07,
      "logits/chosen": -0.46073633432388306,
      "logits/rejected": -0.45000267028808594,
      "logps/chosen": -391.29534912109375,
      "logps/ref_chosen": -52.920921325683594,
      "logps/ref_rejected": -90.3154296875,
      "logps/rejected": -650.7509765625,
      "loss": 1.0039,
      "margin_dpo/margin_mean": 222.06109619140625,
      "margin_dpo/margin_std": 262.95599365234375,
      "step": 466
    },
    {
      "epoch": 0.6857562408223201,
      "fcm_dpo/beta": 0.00250377063639462,
      "fcm_dpo/delta": 0.003041490912437439,
      "fcm_dpo/margin": 237.8001251220703,
      "fcm_dpo/q_t": 0.37523841857910156,
      "grad_norm": 46.20204544067383,
      "learning_rate": 1.3741809409947729e-07,
      "logits/chosen": -0.5411770343780518,
      "logits/rejected": -0.5251420736312866,
      "logps/chosen": -482.17987060546875,
      "logps/ref_chosen": -78.7158203125,
      "logps/ref_rejected": -102.86019897460938,
      "logps/rejected": -744.1243896484375,
      "loss": 1.0294,
      "margin_dpo/margin_mean": 237.80014038085938,
      "margin_dpo/margin_std": 325.20404052734375,
      "step": 467
    },
    {
      "epoch": 0.6872246696035242,
      "fcm_dpo/beta": 0.002495876979082823,
      "fcm_dpo/delta": -0.055481918156147,
      "fcm_dpo/margin": 260.9195251464844,
      "fcm_dpo/q_t": 0.35720133781433105,
      "grad_norm": 33.269073486328125,
      "learning_rate": 1.362737437810114e-07,
      "logits/chosen": -0.5177669525146484,
      "logits/rejected": -0.49751150608062744,
      "logps/chosen": -376.0140075683594,
      "logps/ref_chosen": -69.93536376953125,
      "logps/ref_rejected": -101.02880859375,
      "logps/rejected": -668.0269775390625,
      "loss": 0.9373,
      "margin_dpo/margin_mean": 260.9195556640625,
      "margin_dpo/margin_std": 270.485595703125,
      "step": 468
    },
    {
      "epoch": 0.6886930983847284,
      "fcm_dpo/beta": 0.002456499030813575,
      "fcm_dpo/delta": -0.07530368864536285,
      "fcm_dpo/margin": 272.3406982421875,
      "fcm_dpo/q_t": 0.35292261838912964,
      "grad_norm": 29.80097770690918,
      "learning_rate": 1.351323902551631e-07,
      "logits/chosen": -0.5581041574478149,
      "logits/rejected": -0.5368698835372925,
      "logps/chosen": -412.0981140136719,
      "logps/ref_chosen": -68.12469482421875,
      "logps/ref_rejected": -104.78640747070312,
      "logps/rejected": -721.1005249023438,
      "loss": 0.9203,
      "margin_dpo/margin_mean": 272.3406982421875,
      "margin_dpo/margin_std": 266.16925048828125,
      "step": 469
    },
    {
      "epoch": 0.6901615271659325,
      "fcm_dpo/beta": 0.0024513239040970802,
      "fcm_dpo/delta": -0.03624919056892395,
      "fcm_dpo/margin": 258.62164306640625,
      "fcm_dpo/q_t": 0.35686296224594116,
      "grad_norm": 25.734210968017578,
      "learning_rate": 1.339940635976592e-07,
      "logits/chosen": -0.5032815933227539,
      "logits/rejected": -0.49432438611984253,
      "logps/chosen": -312.69384765625,
      "logps/ref_chosen": -43.791927337646484,
      "logps/ref_rejected": -82.70285034179688,
      "logps/rejected": -610.2263793945312,
      "loss": 0.9388,
      "margin_dpo/margin_mean": 258.62164306640625,
      "margin_dpo/margin_std": 256.03778076171875,
      "step": 470
    },
    {
      "epoch": 0.6916299559471366,
      "fcm_dpo/beta": 0.0024561528116464615,
      "fcm_dpo/delta": 0.06438558548688889,
      "fcm_dpo/margin": 219.4058074951172,
      "fcm_dpo/q_t": 0.38015252351760864,
      "grad_norm": 27.724517822265625,
      "learning_rate": 1.3285879380446563e-07,
      "logits/chosen": -0.5248172879219055,
      "logits/rejected": -0.5118055939674377,
      "logps/chosen": -382.17425537109375,
      "logps/ref_chosen": -63.33952331542969,
      "logps/ref_rejected": -83.61048126220703,
      "logps/rejected": -621.85107421875,
      "loss": 1.001,
      "margin_dpo/margin_mean": 219.40579223632812,
      "margin_dpo/margin_std": 245.6639404296875,
      "step": 471
    },
    {
      "epoch": 0.6930983847283406,
      "fcm_dpo/beta": 0.0024520205333828926,
      "fcm_dpo/delta": -0.06850513815879822,
      "fcm_dpo/margin": 270.7828063964844,
      "fcm_dpo/q_t": 0.3614341616630554,
      "grad_norm": 26.621925354003906,
      "learning_rate": 1.317266107909975e-07,
      "logits/chosen": -0.5987716317176819,
      "logits/rejected": -0.5613675117492676,
      "logps/chosen": -406.96942138671875,
      "logps/ref_chosen": -83.66610717773438,
      "logps/ref_rejected": -117.20919799804688,
      "logps/rejected": -711.2952880859375,
      "loss": 0.9546,
      "margin_dpo/margin_mean": 270.78277587890625,
      "margin_dpo/margin_std": 321.0069885253906,
      "step": 472
    },
    {
      "epoch": 0.6945668135095447,
      "fcm_dpo/beta": 0.00249496940523386,
      "fcm_dpo/delta": 0.14937041699886322,
      "fcm_dpo/margin": 183.41294860839844,
      "fcm_dpo/q_t": 0.40312713384628296,
      "grad_norm": 36.99402618408203,
      "learning_rate": 1.3059754439133002e-07,
      "logits/chosen": -0.5006664395332336,
      "logits/rejected": -0.4722933769226074,
      "logps/chosen": -418.47216796875,
      "logps/ref_chosen": -63.49696731567383,
      "logps/ref_rejected": -81.14657592773438,
      "logps/rejected": -619.5347900390625,
      "loss": 1.1136,
      "margin_dpo/margin_mean": 183.4129638671875,
      "margin_dpo/margin_std": 317.38916015625,
      "step": 473
    },
    {
      "epoch": 0.6960352422907489,
      "fcm_dpo/beta": 0.002567752031609416,
      "fcm_dpo/delta": 0.09946488589048386,
      "fcm_dpo/margin": 196.4961700439453,
      "fcm_dpo/q_t": 0.38934361934661865,
      "grad_norm": 33.70315933227539,
      "learning_rate": 1.2947162435741277e-07,
      "logits/chosen": -0.5230387449264526,
      "logits/rejected": -0.5138710737228394,
      "logps/chosen": -361.4368591308594,
      "logps/ref_chosen": -52.6119384765625,
      "logps/ref_rejected": -90.08041381835938,
      "logps/rejected": -595.4014892578125,
      "loss": 1.0559,
      "margin_dpo/margin_mean": 196.49615478515625,
      "margin_dpo/margin_std": 272.44683837890625,
      "step": 474
    },
    {
      "epoch": 0.697503671071953,
      "fcm_dpo/beta": 0.002530063269659877,
      "fcm_dpo/delta": -0.09672711789608002,
      "fcm_dpo/margin": 272.7248840332031,
      "fcm_dpo/q_t": 0.3485596776008606,
      "grad_norm": 23.911319732666016,
      "learning_rate": 1.2834888035828596e-07,
      "logits/chosen": -0.48518672585487366,
      "logits/rejected": -0.49074888229370117,
      "logps/chosen": -290.80499267578125,
      "logps/ref_chosen": -42.49519348144531,
      "logps/ref_rejected": -90.06294250488281,
      "logps/rejected": -611.09765625,
      "loss": 0.9061,
      "margin_dpo/margin_mean": 272.7248840332031,
      "margin_dpo/margin_std": 265.2176208496094,
      "step": 475
    },
    {
      "epoch": 0.6989720998531571,
      "fcm_dpo/beta": 0.002525830641388893,
      "fcm_dpo/delta": 0.016785459592938423,
      "fcm_dpo/margin": 231.28424072265625,
      "fcm_dpo/q_t": 0.3694440722465515,
      "grad_norm": 32.7138786315918,
      "learning_rate": 1.2722934197929802e-07,
      "logits/chosen": -0.5376179814338684,
      "logits/rejected": -0.5307664275169373,
      "logps/chosen": -305.538818359375,
      "logps/ref_chosen": -42.94938278198242,
      "logps/ref_rejected": -73.71023559570312,
      "logps/rejected": -567.5839233398438,
      "loss": 0.9606,
      "margin_dpo/margin_mean": 231.28424072265625,
      "margin_dpo/margin_std": 231.99273681640625,
      "step": 476
    },
    {
      "epoch": 0.7004405286343612,
      "fcm_dpo/beta": 0.0025543358642607927,
      "fcm_dpo/delta": 0.06146734952926636,
      "fcm_dpo/margin": 212.0760498046875,
      "fcm_dpo/q_t": 0.3796027898788452,
      "grad_norm": 30.43485450744629,
      "learning_rate": 1.2611303872132631e-07,
      "logits/chosen": -0.5261760354042053,
      "logits/rejected": -0.4716786742210388,
      "logps/chosen": -365.0345458984375,
      "logps/ref_chosen": -70.77261352539062,
      "logps/ref_rejected": -76.13737487792969,
      "logps/rejected": -582.475341796875,
      "loss": 1.02,
      "margin_dpo/margin_mean": 212.0760498046875,
      "margin_dpo/margin_std": 266.8735656738281,
      "step": 477
    },
    {
      "epoch": 0.7019089574155654,
      "fcm_dpo/beta": 0.0025457185693085194,
      "fcm_dpo/delta": -0.05643880367279053,
      "fcm_dpo/margin": 256.39349365234375,
      "fcm_dpo/q_t": 0.3551124632358551,
      "grad_norm": 30.269575119018555,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -0.4699116349220276,
      "logits/rejected": -0.4670166075229645,
      "logps/chosen": -282.0042724609375,
      "logps/ref_chosen": -41.440513610839844,
      "logps/ref_rejected": -85.36196899414062,
      "logps/rejected": -582.3192138671875,
      "loss": 0.9133,
      "margin_dpo/margin_mean": 256.39349365234375,
      "margin_dpo/margin_std": 231.91416931152344,
      "step": 478
    },
    {
      "epoch": 0.7033773861967695,
      "fcm_dpo/beta": 0.002581070177257061,
      "fcm_dpo/delta": 0.0580781027674675,
      "fcm_dpo/margin": 209.93270874023438,
      "fcm_dpo/q_t": 0.3800834119319916,
      "grad_norm": 34.0640754699707,
      "learning_rate": 1.2389025514492456e-07,
      "logits/chosen": -0.49057966470718384,
      "logits/rejected": -0.49215784668922424,
      "logps/chosen": -384.4983215332031,
      "logps/ref_chosen": -53.907920837402344,
      "logps/ref_rejected": -95.1163330078125,
      "logps/rejected": -635.639404296875,
      "loss": 1.0201,
      "margin_dpo/margin_mean": 209.93270874023438,
      "margin_dpo/margin_std": 248.3424530029297,
      "step": 479
    },
    {
      "epoch": 0.7048458149779736,
      "fcm_dpo/beta": 0.002632645657286048,
      "fcm_dpo/delta": 0.1320512592792511,
      "fcm_dpo/margin": 179.1344757080078,
      "fcm_dpo/q_t": 0.3967488408088684,
      "grad_norm": 46.27069091796875,
      "learning_rate": 1.227838333989088e-07,
      "logits/chosen": -0.4810282588005066,
      "logits/rejected": -0.4517630636692047,
      "logps/chosen": -410.599853515625,
      "logps/ref_chosen": -58.682701110839844,
      "logps/ref_rejected": -82.93248748779297,
      "logps/rejected": -613.984130859375,
      "loss": 1.083,
      "margin_dpo/margin_mean": 179.1344757080078,
      "margin_dpo/margin_std": 258.1438293457031,
      "step": 480
    },
    {
      "epoch": 0.7063142437591777,
      "fcm_dpo/beta": 0.002601025393232703,
      "fcm_dpo/delta": -0.051007017493247986,
      "fcm_dpo/margin": 248.92288208007812,
      "fcm_dpo/q_t": 0.3576533794403076,
      "grad_norm": 35.464439392089844,
      "learning_rate": 1.2168076391719489e-07,
      "logits/chosen": -0.5199482440948486,
      "logits/rejected": -0.5128802061080933,
      "logps/chosen": -346.0648498535156,
      "logps/ref_chosen": -54.964271545410156,
      "logps/ref_rejected": -92.42044067382812,
      "logps/rejected": -632.44384765625,
      "loss": 0.94,
      "margin_dpo/margin_mean": 248.92288208007812,
      "margin_dpo/margin_std": 260.282958984375,
      "step": 481
    },
    {
      "epoch": 0.7077826725403817,
      "fcm_dpo/beta": 0.002659988822415471,
      "fcm_dpo/delta": 0.1587131917476654,
      "fcm_dpo/margin": 168.63143920898438,
      "fcm_dpo/q_t": 0.4008188843727112,
      "grad_norm": 38.83755111694336,
      "learning_rate": 1.2058107576668938e-07,
      "logits/chosen": -0.47180038690567017,
      "logits/rejected": -0.45411762595176697,
      "logps/chosen": -398.88873291015625,
      "logps/ref_chosen": -67.553466796875,
      "logps/ref_rejected": -87.58953857421875,
      "logps/rejected": -587.5562133789062,
      "loss": 1.0902,
      "margin_dpo/margin_mean": 168.63143920898438,
      "margin_dpo/margin_std": 254.7269287109375,
      "step": 482
    },
    {
      "epoch": 0.7092511013215859,
      "fcm_dpo/beta": 0.002619517268612981,
      "fcm_dpo/delta": -0.1474650502204895,
      "fcm_dpo/margin": 281.0007019042969,
      "fcm_dpo/q_t": 0.34216371178627014,
      "grad_norm": 40.65428161621094,
      "learning_rate": 1.194847979251979e-07,
      "logits/chosen": -0.5000494718551636,
      "logits/rejected": -0.48498857021331787,
      "logps/chosen": -357.7904052734375,
      "logps/ref_chosen": -63.32981872558594,
      "logps/ref_rejected": -95.78697204589844,
      "logps/rejected": -671.248291015625,
      "loss": 0.8912,
      "margin_dpo/margin_mean": 281.0007019042969,
      "margin_dpo/margin_std": 279.4383544921875,
      "step": 483
    },
    {
      "epoch": 0.71071953010279,
      "fcm_dpo/beta": 0.0026109693571925163,
      "fcm_dpo/delta": -0.03564995154738426,
      "fcm_dpo/margin": 242.32601928710938,
      "fcm_dpo/q_t": 0.3604073226451874,
      "grad_norm": 27.125112533569336,
      "learning_rate": 1.1839195928066101e-07,
      "logits/chosen": -0.48026251792907715,
      "logits/rejected": -0.46726343035697937,
      "logps/chosen": -332.4265441894531,
      "logps/ref_chosen": -59.13812255859375,
      "logps/ref_rejected": -84.37144470214844,
      "logps/rejected": -599.9859008789062,
      "loss": 0.9377,
      "margin_dpo/margin_mean": 242.32603454589844,
      "margin_dpo/margin_std": 237.0671844482422,
      "step": 484
    },
    {
      "epoch": 0.7121879588839941,
      "fcm_dpo/beta": 0.0025863349437713623,
      "fcm_dpo/delta": -0.01730707660317421,
      "fcm_dpo/margin": 238.25384521484375,
      "fcm_dpo/q_t": 0.36559098958969116,
      "grad_norm": 36.989036560058594,
      "learning_rate": 1.1730258863039347e-07,
      "logits/chosen": -0.468616247177124,
      "logits/rejected": -0.466255784034729,
      "logps/chosen": -326.2464904785156,
      "logps/ref_chosen": -58.849571228027344,
      "logps/ref_rejected": -103.36408233642578,
      "logps/rejected": -609.0148315429688,
      "loss": 0.967,
      "margin_dpo/margin_mean": 238.25384521484375,
      "margin_dpo/margin_std": 268.4663391113281,
      "step": 485
    },
    {
      "epoch": 0.7136563876651982,
      "fcm_dpo/beta": 0.0025734733790159225,
      "fcm_dpo/delta": -0.033280279487371445,
      "fcm_dpo/margin": 245.24191284179688,
      "fcm_dpo/q_t": 0.36465585231781006,
      "grad_norm": 40.85445022583008,
      "learning_rate": 1.1621671468032493e-07,
      "logits/chosen": -0.50041663646698,
      "logits/rejected": -0.48742035031318665,
      "logps/chosen": -367.93310546875,
      "logps/ref_chosen": -55.25966262817383,
      "logps/ref_rejected": -92.13936614990234,
      "logps/rejected": -650.0546875,
      "loss": 0.9882,
      "margin_dpo/margin_mean": 245.24192810058594,
      "margin_dpo/margin_std": 306.00677490234375,
      "step": 486
    },
    {
      "epoch": 0.7151248164464024,
      "fcm_dpo/beta": 0.002564340131357312,
      "fcm_dpo/delta": 0.024611998349428177,
      "fcm_dpo/margin": 224.8999786376953,
      "fcm_dpo/q_t": 0.3688669800758362,
      "grad_norm": 35.55867004394531,
      "learning_rate": 1.1513436604424378e-07,
      "logits/chosen": -0.48910582065582275,
      "logits/rejected": -0.4775072932243347,
      "logps/chosen": -351.23388671875,
      "logps/ref_chosen": -53.06330871582031,
      "logps/ref_rejected": -92.41883087158203,
      "logps/rejected": -615.4893798828125,
      "loss": 0.9727,
      "margin_dpo/margin_mean": 224.8999786376953,
      "margin_dpo/margin_std": 231.9244842529297,
      "step": 487
    },
    {
      "epoch": 0.7165932452276065,
      "fcm_dpo/beta": 0.0025914846919476986,
      "fcm_dpo/delta": 0.017736222594976425,
      "fcm_dpo/margin": 224.94769287109375,
      "fcm_dpo/q_t": 0.3664684593677521,
      "grad_norm": 39.95408248901367,
      "learning_rate": 1.1405557124304335e-07,
      "logits/chosen": -0.4504004716873169,
      "logits/rejected": -0.43368983268737793,
      "logps/chosen": -295.21221923828125,
      "logps/ref_chosen": -52.22815704345703,
      "logps/ref_rejected": -84.00656127929688,
      "logps/rejected": -551.9383544921875,
      "loss": 0.9446,
      "margin_dpo/margin_mean": 224.94769287109375,
      "margin_dpo/margin_std": 193.76791381835938,
      "step": 488
    },
    {
      "epoch": 0.7180616740088106,
      "fcm_dpo/beta": 0.00260374927893281,
      "fcm_dpo/delta": 0.053067468106746674,
      "fcm_dpo/margin": 211.18069458007812,
      "fcm_dpo/q_t": 0.378859281539917,
      "grad_norm": 33.997283935546875,
      "learning_rate": 1.1298035870396985e-07,
      "logits/chosen": -0.49653559923171997,
      "logits/rejected": -0.47625601291656494,
      "logps/chosen": -304.00372314453125,
      "logps/ref_chosen": -55.989627838134766,
      "logps/ref_rejected": -79.39812469482422,
      "logps/rejected": -538.5928955078125,
      "loss": 0.9855,
      "margin_dpo/margin_mean": 211.18069458007812,
      "margin_dpo/margin_std": 227.5222930908203,
      "step": 489
    },
    {
      "epoch": 0.7195301027900147,
      "fcm_dpo/beta": 0.002616376616060734,
      "fcm_dpo/delta": -0.01173383742570877,
      "fcm_dpo/margin": 233.45465087890625,
      "fcm_dpo/q_t": 0.36978480219841003,
      "grad_norm": 51.60263442993164,
      "learning_rate": 1.1190875675987355e-07,
      "logits/chosen": -0.5432174801826477,
      "logits/rejected": -0.5576708316802979,
      "logps/chosen": -326.5618896484375,
      "logps/ref_chosen": -52.36639404296875,
      "logps/ref_rejected": -110.4090576171875,
      "logps/rejected": -618.0592041015625,
      "loss": 0.995,
      "margin_dpo/margin_mean": 233.45465087890625,
      "margin_dpo/margin_std": 293.9273376464844,
      "step": 490
    },
    {
      "epoch": 0.7209985315712188,
      "fcm_dpo/beta": 0.002662881277501583,
      "fcm_dpo/delta": 0.1412055492401123,
      "fcm_dpo/margin": 174.71432495117188,
      "fcm_dpo/q_t": 0.39850908517837524,
      "grad_norm": 46.16297912597656,
      "learning_rate": 1.1084079364846241e-07,
      "logits/chosen": -0.4562872052192688,
      "logits/rejected": -0.4420876204967499,
      "logps/chosen": -348.5367431640625,
      "logps/ref_chosen": -60.11626434326172,
      "logps/ref_rejected": -73.27278900146484,
      "logps/rejected": -536.4075927734375,
      "loss": 1.072,
      "margin_dpo/margin_mean": 174.71432495117188,
      "margin_dpo/margin_std": 248.17523193359375,
      "step": 491
    },
    {
      "epoch": 0.7224669603524229,
      "fcm_dpo/beta": 0.002773004351183772,
      "fcm_dpo/delta": 0.22383955121040344,
      "fcm_dpo/margin": 138.69735717773438,
      "fcm_dpo/q_t": 0.4172734022140503,
      "grad_norm": 42.61263656616211,
      "learning_rate": 1.097764975115576e-07,
      "logits/chosen": -0.44189101457595825,
      "logits/rejected": -0.4200980067253113,
      "logps/chosen": -361.65570068359375,
      "logps/ref_chosen": -53.994178771972656,
      "logps/ref_rejected": -72.65962219238281,
      "logps/rejected": -519.0184936523438,
      "loss": 1.1625,
      "margin_dpo/margin_mean": 138.69735717773438,
      "margin_dpo/margin_std": 266.28662109375,
      "step": 492
    },
    {
      "epoch": 0.723935389133627,
      "fcm_dpo/beta": 0.0028385447803884745,
      "fcm_dpo/delta": 0.1169082373380661,
      "fcm_dpo/margin": 172.2346954345703,
      "fcm_dpo/q_t": 0.39189258217811584,
      "grad_norm": 43.42572021484375,
      "learning_rate": 1.0871589639435203e-07,
      "logits/chosen": -0.5289523601531982,
      "logits/rejected": -0.5010632276535034,
      "logps/chosen": -380.6925048828125,
      "logps/ref_chosen": -75.49723815917969,
      "logps/ref_rejected": -87.32301330566406,
      "logps/rejected": -564.7529296875,
      "loss": 1.0419,
      "margin_dpo/margin_mean": 172.2346954345703,
      "margin_dpo/margin_std": 214.199951171875,
      "step": 493
    },
    {
      "epoch": 0.7254038179148311,
      "fcm_dpo/beta": 0.00280939694494009,
      "fcm_dpo/delta": -0.12965027987957,
      "fcm_dpo/margin": 256.29547119140625,
      "fcm_dpo/q_t": 0.3384782075881958,
      "grad_norm": 34.547096252441406,
      "learning_rate": 1.0765901824467166e-07,
      "logits/chosen": -0.4465922713279724,
      "logits/rejected": -0.45816731452941895,
      "logps/chosen": -296.73284912109375,
      "logps/ref_chosen": -41.35926818847656,
      "logps/ref_rejected": -86.09136962890625,
      "logps/rejected": -597.7604370117188,
      "loss": 0.8787,
      "margin_dpo/margin_mean": 256.2955017089844,
      "margin_dpo/margin_std": 221.27516174316406,
      "step": 494
    },
    {
      "epoch": 0.7268722466960352,
      "fcm_dpo/beta": 0.002777573186904192,
      "fcm_dpo/delta": -0.08846628665924072,
      "fcm_dpo/margin": 245.58688354492188,
      "fcm_dpo/q_t": 0.35333937406539917,
      "grad_norm": 34.23904037475586,
      "learning_rate": 1.0660589091223854e-07,
      "logits/chosen": -0.493415504693985,
      "logits/rejected": -0.4882367253303528,
      "logps/chosen": -356.96539306640625,
      "logps/ref_chosen": -63.53507995605469,
      "logps/ref_rejected": -91.42443084716797,
      "logps/rejected": -630.441650390625,
      "loss": 0.929,
      "margin_dpo/margin_mean": 245.58688354492188,
      "margin_dpo/margin_std": 258.7152099609375,
      "step": 495
    },
    {
      "epoch": 0.7283406754772394,
      "fcm_dpo/beta": 0.002794483909383416,
      "fcm_dpo/delta": 0.13067297637462616,
      "fcm_dpo/margin": 170.2359619140625,
      "fcm_dpo/q_t": 0.39404138922691345,
      "grad_norm": 46.63887405395508,
      "learning_rate": 1.0555654214793722e-07,
      "logits/chosen": -0.4721938669681549,
      "logits/rejected": -0.4511212706565857,
      "logps/chosen": -414.6033630371094,
      "logps/ref_chosen": -72.5919189453125,
      "logps/ref_rejected": -84.32933807373047,
      "logps/rejected": -596.5767822265625,
      "loss": 1.0508,
      "margin_dpo/margin_mean": 170.2359619140625,
      "margin_dpo/margin_std": 214.48468017578125,
      "step": 496
    },
    {
      "epoch": 0.7298091042584435,
      "fcm_dpo/beta": 0.0028514941222965717,
      "fcm_dpo/delta": 0.13596858084201813,
      "fcm_dpo/margin": 164.8284912109375,
      "fcm_dpo/q_t": 0.3949023485183716,
      "grad_norm": 35.32782745361328,
      "learning_rate": 1.0451099960308374e-07,
      "logits/chosen": -0.4963914752006531,
      "logits/rejected": -0.479754239320755,
      "logps/chosen": -384.9673156738281,
      "logps/ref_chosen": -58.59397506713867,
      "logps/ref_rejected": -76.28836822509766,
      "logps/rejected": -567.490234375,
      "loss": 1.0561,
      "margin_dpo/margin_mean": 164.8284912109375,
      "margin_dpo/margin_std": 201.14633178710938,
      "step": 497
    },
    {
      "epoch": 0.7312775330396476,
      "fcm_dpo/beta": 0.0028984027449041605,
      "fcm_dpo/delta": 0.004962385166436434,
      "fcm_dpo/margin": 205.40037536621094,
      "fcm_dpo/q_t": 0.37139469385147095,
      "grad_norm": 38.93318557739258,
      "learning_rate": 1.0346929082869641e-07,
      "logits/chosen": -0.4890047311782837,
      "logits/rejected": -0.4707816243171692,
      "logps/chosen": -396.84356689453125,
      "logps/ref_chosen": -71.20565795898438,
      "logps/ref_rejected": -83.95803833007812,
      "logps/rejected": -614.996337890625,
      "loss": 1.0052,
      "margin_dpo/margin_mean": 205.40037536621094,
      "margin_dpo/margin_std": 263.6328125,
      "step": 498
    },
    {
      "epoch": 0.7327459618208517,
      "fcm_dpo/beta": 0.0028373058885335922,
      "fcm_dpo/delta": -0.13358387351036072,
      "fcm_dpo/margin": 255.02593994140625,
      "fcm_dpo/q_t": 0.3414595425128937,
      "grad_norm": 47.647823333740234,
      "learning_rate": 1.0243144327477013e-07,
      "logits/chosen": -0.4490722417831421,
      "logits/rejected": -0.45660898089408875,
      "logps/chosen": -363.71142578125,
      "logps/ref_chosen": -51.25519561767578,
      "logps/ref_rejected": -101.07870483398438,
      "logps/rejected": -668.5609130859375,
      "loss": 0.9152,
      "margin_dpo/margin_mean": 255.02593994140625,
      "margin_dpo/margin_std": 267.2959899902344,
      "step": 499
    },
    {
      "epoch": 0.7342143906020558,
      "fcm_dpo/beta": 0.0028089380357414484,
      "fcm_dpo/delta": -0.024006161838769913,
      "fcm_dpo/margin": 221.58847045898438,
      "fcm_dpo/q_t": 0.3665502965450287,
      "grad_norm": 52.78643798828125,
      "learning_rate": 1.0139748428955333e-07,
      "logits/chosen": -0.4591681957244873,
      "logits/rejected": -0.4673272371292114,
      "logps/chosen": -429.0184326171875,
      "logps/ref_chosen": -57.027442932128906,
      "logps/ref_rejected": -93.93421173095703,
      "logps/rejected": -687.513671875,
      "loss": 0.9865,
      "margin_dpo/margin_mean": 221.58847045898438,
      "margin_dpo/margin_std": 274.5337829589844,
      "step": 500
    },
    {
      "epoch": 0.73568281938326,
      "fcm_dpo/beta": 0.002789230551570654,
      "fcm_dpo/delta": -0.06634011119604111,
      "fcm_dpo/margin": 237.33590698242188,
      "fcm_dpo/q_t": 0.3573167026042938,
      "grad_norm": 34.69628143310547,
      "learning_rate": 1.0036744111882672e-07,
      "logits/chosen": -0.45637276768684387,
      "logits/rejected": -0.4298150837421417,
      "logps/chosen": -364.97088623046875,
      "logps/ref_chosen": -54.359527587890625,
      "logps/ref_rejected": -80.15670013427734,
      "logps/rejected": -628.10400390625,
      "loss": 0.9556,
      "margin_dpo/margin_mean": 237.33590698242188,
      "margin_dpo/margin_std": 273.72589111328125,
      "step": 501
    },
    {
      "epoch": 0.737151248164464,
      "fcm_dpo/beta": 0.002761396113783121,
      "fcm_dpo/delta": -0.06494729220867157,
      "fcm_dpo/margin": 239.1594696044922,
      "fcm_dpo/q_t": 0.3486095666885376,
      "grad_norm": 28.551965713500977,
      "learning_rate": 9.934134090518592e-08,
      "logits/chosen": -0.4536617398262024,
      "logits/rejected": -0.4335412085056305,
      "logps/chosen": -356.140380859375,
      "logps/ref_chosen": -67.60050964355469,
      "logps/ref_rejected": -82.94876098632812,
      "logps/rejected": -610.6480712890625,
      "loss": 0.8913,
      "margin_dpo/margin_mean": 239.1594696044922,
      "margin_dpo/margin_std": 181.16015625,
      "step": 502
    },
    {
      "epoch": 0.7386196769456681,
      "fcm_dpo/beta": 0.002728442195802927,
      "fcm_dpo/delta": 0.006640031933784485,
      "fcm_dpo/margin": 217.5033721923828,
      "fcm_dpo/q_t": 0.36779502034187317,
      "grad_norm": 33.08012008666992,
      "learning_rate": 9.831921068732571e-08,
      "logits/chosen": -0.43443652987480164,
      "logits/rejected": -0.4157460927963257,
      "logps/chosen": -348.41259765625,
      "logps/ref_chosen": -55.078407287597656,
      "logps/ref_rejected": -82.50544738769531,
      "logps/rejected": -593.343017578125,
      "loss": 0.9555,
      "margin_dpo/margin_mean": 217.50338745117188,
      "margin_dpo/margin_std": 215.96286010742188,
      "step": 503
    },
    {
      "epoch": 0.7400881057268722,
      "fcm_dpo/beta": 0.0026825086679309607,
      "fcm_dpo/delta": -0.13238787651062012,
      "fcm_dpo/margin": 269.34735107421875,
      "fcm_dpo/q_t": 0.34110093116760254,
      "grad_norm": 35.52935791015625,
      "learning_rate": 9.730107739932805e-08,
      "logits/chosen": -0.45639944076538086,
      "logits/rejected": -0.4605451822280884,
      "logps/chosen": -375.00860595703125,
      "logps/ref_chosen": -59.96575164794922,
      "logps/ref_rejected": -103.76212310791016,
      "logps/rejected": -688.15234375,
      "loss": 0.8908,
      "margin_dpo/margin_mean": 269.34735107421875,
      "margin_dpo/margin_std": 249.264892578125,
      "step": 504
    },
    {
      "epoch": 0.7415565345080763,
      "fcm_dpo/beta": 0.0027532954700291157,
      "fcm_dpo/delta": 0.1878781020641327,
      "fcm_dpo/margin": 152.27743530273438,
      "fcm_dpo/q_t": 0.4108385741710663,
      "grad_norm": 54.627689361572266,
      "learning_rate": 9.628696786995188e-08,
      "logits/chosen": -0.5202739834785461,
      "logits/rejected": -0.49124228954315186,
      "logps/chosen": -429.90838623046875,
      "logps/ref_chosen": -76.1549072265625,
      "logps/ref_rejected": -88.58537292480469,
      "logps/rejected": -594.6163330078125,
      "loss": 1.1332,
      "margin_dpo/margin_mean": 152.27743530273438,
      "margin_dpo/margin_std": 267.0574951171875,
      "step": 505
    },
    {
      "epoch": 0.7430249632892805,
      "fcm_dpo/beta": 0.002741964068263769,
      "fcm_dpo/delta": -0.04416107013821602,
      "fcm_dpo/margin": 233.71510314941406,
      "fcm_dpo/q_t": 0.358504056930542,
      "grad_norm": 38.65158462524414,
      "learning_rate": 9.527690882192635e-08,
      "logits/chosen": -0.47684890031814575,
      "logits/rejected": -0.4706017076969147,
      "logps/chosen": -328.3465576171875,
      "logps/ref_chosen": -48.96050262451172,
      "logps/ref_rejected": -78.41505432128906,
      "logps/rejected": -591.5161743164062,
      "loss": 0.9438,
      "margin_dpo/margin_mean": 233.71510314941406,
      "margin_dpo/margin_std": 243.26914978027344,
      "step": 506
    },
    {
      "epoch": 0.7444933920704846,
      "fcm_dpo/beta": 0.002767894882708788,
      "fcm_dpo/delta": 0.04769434034824371,
      "fcm_dpo/margin": 200.48585510253906,
      "fcm_dpo/q_t": 0.38139885663986206,
      "grad_norm": 31.428619384765625,
      "learning_rate": 9.427092687124691e-08,
      "logits/chosen": -0.4897286891937256,
      "logits/rejected": -0.4771530032157898,
      "logps/chosen": -389.1927185058594,
      "logps/ref_chosen": -66.80149841308594,
      "logps/ref_rejected": -95.37289428710938,
      "logps/rejected": -618.25,
      "loss": 1.0113,
      "margin_dpo/margin_mean": 200.48583984375,
      "margin_dpo/margin_std": 256.5914001464844,
      "step": 507
    },
    {
      "epoch": 0.7459618208516887,
      "fcm_dpo/beta": 0.002809921745210886,
      "fcm_dpo/delta": 0.07570519298315048,
      "fcm_dpo/margin": 187.93719482421875,
      "fcm_dpo/q_t": 0.38831907510757446,
      "grad_norm": 37.678199768066406,
      "learning_rate": 9.326904852647344e-08,
      "logits/chosen": -0.4942491054534912,
      "logits/rejected": -0.48562514781951904,
      "logps/chosen": -381.1417236328125,
      "logps/ref_chosen": -71.303466796875,
      "logps/ref_rejected": -95.6275405883789,
      "logps/rejected": -593.4030151367188,
      "loss": 1.0468,
      "margin_dpo/margin_mean": 187.93719482421875,
      "margin_dpo/margin_std": 266.2950744628906,
      "step": 508
    },
    {
      "epoch": 0.7474302496328928,
      "fcm_dpo/beta": 0.0028523635119199753,
      "fcm_dpo/delta": 0.05893818289041519,
      "fcm_dpo/margin": 190.60777282714844,
      "fcm_dpo/q_t": 0.38169676065444946,
      "grad_norm": 39.39438247680664,
      "learning_rate": 9.227130018803195e-08,
      "logits/chosen": -0.5167714357376099,
      "logits/rejected": -0.5027569532394409,
      "logps/chosen": -326.60260009765625,
      "logps/ref_chosen": -63.81895065307617,
      "logps/ref_rejected": -83.25643920898438,
      "logps/rejected": -536.6478881835938,
      "loss": 1.0086,
      "margin_dpo/margin_mean": 190.60775756835938,
      "margin_dpo/margin_std": 227.0200653076172,
      "step": 509
    },
    {
      "epoch": 0.748898678414097,
      "fcm_dpo/beta": 0.002829122357070446,
      "fcm_dpo/delta": -0.050209492444992065,
      "fcm_dpo/margin": 228.68527221679688,
      "fcm_dpo/q_t": 0.35418280959129333,
      "grad_norm": 33.37362289428711,
      "learning_rate": 9.127770814751932e-08,
      "logits/chosen": -0.47170090675354004,
      "logits/rejected": -0.4742484390735626,
      "logps/chosen": -364.83221435546875,
      "logps/ref_chosen": -51.878448486328125,
      "logps/ref_rejected": -102.7651596069336,
      "logps/rejected": -644.4041748046875,
      "loss": 0.9209,
      "margin_dpo/margin_mean": 228.68527221679688,
      "margin_dpo/margin_std": 210.78187561035156,
      "step": 510
    },
    {
      "epoch": 0.750367107195301,
      "fcm_dpo/beta": 0.0028122500516474247,
      "fcm_dpo/delta": -0.04460298642516136,
      "fcm_dpo/margin": 228.15643310546875,
      "fcm_dpo/q_t": 0.3623583912849426,
      "grad_norm": 28.944910049438477,
      "learning_rate": 9.028829858700973e-08,
      "logits/chosen": -0.4922507405281067,
      "logits/rejected": -0.4849390387535095,
      "logps/chosen": -339.91571044921875,
      "logps/ref_chosen": -60.23811721801758,
      "logps/ref_rejected": -92.85676574707031,
      "logps/rejected": -600.6907958984375,
      "loss": 0.9745,
      "margin_dpo/margin_mean": 228.15643310546875,
      "margin_dpo/margin_std": 276.27374267578125,
      "step": 511
    },
    {
      "epoch": 0.7518355359765051,
      "fcm_dpo/beta": 0.002760728355497122,
      "fcm_dpo/delta": -0.08452171087265015,
      "fcm_dpo/margin": 245.87234497070312,
      "fcm_dpo/q_t": 0.3484812378883362,
      "grad_norm": 27.64395523071289,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": -0.4628169536590576,
      "logits/rejected": -0.44934043288230896,
      "logps/chosen": -307.468994140625,
      "logps/ref_chosen": -54.905494689941406,
      "logps/ref_rejected": -81.87586975097656,
      "logps/rejected": -580.3117065429688,
      "loss": 0.8986,
      "margin_dpo/margin_mean": 245.87234497070312,
      "margin_dpo/margin_std": 219.15728759765625,
      "step": 512
    },
    {
      "epoch": 0.7533039647577092,
      "fcm_dpo/beta": 0.002783479169011116,
      "fcm_dpo/delta": 0.1268652230501175,
      "fcm_dpo/margin": 172.15696716308594,
      "fcm_dpo/q_t": 0.39400118589401245,
      "grad_norm": 46.16716766357422,
      "learning_rate": 8.832213108254863e-08,
      "logits/chosen": -0.4615207314491272,
      "logits/rejected": -0.42703330516815186,
      "logps/chosen": -352.5402526855469,
      "logps/ref_chosen": -64.91644287109375,
      "logps/ref_rejected": -76.06245422363281,
      "logps/rejected": -535.84326171875,
      "loss": 1.0662,
      "margin_dpo/margin_mean": 172.15696716308594,
      "margin_dpo/margin_std": 235.25601196289062,
      "step": 513
    },
    {
      "epoch": 0.7547723935389133,
      "fcm_dpo/beta": 0.0028532061260193586,
      "fcm_dpo/delta": 0.07234474271535873,
      "fcm_dpo/margin": 186.24154663085938,
      "fcm_dpo/q_t": 0.38589781522750854,
      "grad_norm": 40.779319763183594,
      "learning_rate": 8.734542494893954e-08,
      "logits/chosen": -0.47675463557243347,
      "logits/rejected": -0.44973114132881165,
      "logps/chosen": -357.24749755859375,
      "logps/ref_chosen": -74.22957611083984,
      "logps/ref_rejected": -78.945556640625,
      "logps/rejected": -548.2050170898438,
      "loss": 1.0352,
      "margin_dpo/margin_mean": 186.24154663085938,
      "margin_dpo/margin_std": 250.20767211914062,
      "step": 514
    },
    {
      "epoch": 0.7562408223201175,
      "fcm_dpo/beta": 0.0028960055205971003,
      "fcm_dpo/delta": 0.09402909129858017,
      "fcm_dpo/margin": 176.40408325195312,
      "fcm_dpo/q_t": 0.3851974606513977,
      "grad_norm": 31.811119079589844,
      "learning_rate": 8.637300491465272e-08,
      "logits/chosen": -0.47848212718963623,
      "logits/rejected": -0.47844403982162476,
      "logps/chosen": -294.98284912109375,
      "logps/ref_chosen": -50.40156555175781,
      "logps/ref_rejected": -87.09774780273438,
      "logps/rejected": -508.0831604003906,
      "loss": 1.0331,
      "margin_dpo/margin_mean": 176.40408325195312,
      "margin_dpo/margin_std": 221.04446411132812,
      "step": 515
    },
    {
      "epoch": 0.7577092511013216,
      "fcm_dpo/beta": 0.00288777775131166,
      "fcm_dpo/delta": -0.08516449481248856,
      "fcm_dpo/margin": 235.27685546875,
      "fcm_dpo/q_t": 0.3478805422782898,
      "grad_norm": 32.073699951171875,
      "learning_rate": 8.540489660386064e-08,
      "logits/chosen": -0.50160813331604,
      "logits/rejected": -0.5122185945510864,
      "logps/chosen": -312.9991455078125,
      "logps/ref_chosen": -64.64956665039062,
      "logps/ref_rejected": -111.72237396240234,
      "logps/rejected": -595.3488159179688,
      "loss": 0.8995,
      "margin_dpo/margin_mean": 235.27685546875,
      "margin_dpo/margin_std": 209.3588104248047,
      "step": 516
    },
    {
      "epoch": 0.7591776798825257,
      "fcm_dpo/beta": 0.002824044320732355,
      "fcm_dpo/delta": -0.06978069990873337,
      "fcm_dpo/margin": 235.31272888183594,
      "fcm_dpo/q_t": 0.35607290267944336,
      "grad_norm": 34.7307014465332,
      "learning_rate": 8.444112552711752e-08,
      "logits/chosen": -0.498099148273468,
      "logits/rejected": -0.47780922055244446,
      "logps/chosen": -338.8596496582031,
      "logps/ref_chosen": -60.913551330566406,
      "logps/ref_rejected": -89.08308410644531,
      "logps/rejected": -602.3419189453125,
      "loss": 0.9334,
      "margin_dpo/margin_mean": 235.31271362304688,
      "margin_dpo/margin_std": 250.22042846679688,
      "step": 517
    },
    {
      "epoch": 0.7606461086637298,
      "fcm_dpo/beta": 0.002825916511937976,
      "fcm_dpo/delta": 0.01667827181518078,
      "fcm_dpo/margin": 206.73744201660156,
      "fcm_dpo/q_t": 0.37291136384010315,
      "grad_norm": 30.245616912841797,
      "learning_rate": 8.348171708068747e-08,
      "logits/chosen": -0.48336124420166016,
      "logits/rejected": -0.4803961515426636,
      "logps/chosen": -331.5633850097656,
      "logps/ref_chosen": -57.45589065551758,
      "logps/ref_rejected": -85.31269836425781,
      "logps/rejected": -566.1575927734375,
      "loss": 1.0028,
      "margin_dpo/margin_mean": 206.73744201660156,
      "margin_dpo/margin_std": 258.0599365234375,
      "step": 518
    },
    {
      "epoch": 0.762114537444934,
      "fcm_dpo/beta": 0.0028772333171218634,
      "fcm_dpo/delta": 0.08668302744626999,
      "fcm_dpo/margin": 179.88906860351562,
      "fcm_dpo/q_t": 0.3874738812446594,
      "grad_norm": 30.495445251464844,
      "learning_rate": 8.25266965458755e-08,
      "logits/chosen": -0.49129921197891235,
      "logits/rejected": -0.470879465341568,
      "logps/chosen": -341.64581298828125,
      "logps/ref_chosen": -74.06331634521484,
      "logps/ref_rejected": -104.44416809082031,
      "logps/rejected": -551.9157104492188,
      "loss": 1.035,
      "margin_dpo/margin_mean": 179.88906860351562,
      "margin_dpo/margin_std": 234.95318603515625,
      "step": 519
    },
    {
      "epoch": 0.7635829662261381,
      "fcm_dpo/beta": 0.002877956721931696,
      "fcm_dpo/delta": 0.02885432541370392,
      "fcm_dpo/margin": 198.79312133789062,
      "fcm_dpo/q_t": 0.3750660717487335,
      "grad_norm": 30.533952713012695,
      "learning_rate": 8.15760890883607e-08,
      "logits/chosen": -0.4489268660545349,
      "logits/rejected": -0.4413859248161316,
      "logps/chosen": -339.8787841796875,
      "logps/ref_chosen": -70.2998275756836,
      "logps/ref_rejected": -99.98133850097656,
      "logps/rejected": -568.3533935546875,
      "loss": 0.9855,
      "margin_dpo/margin_mean": 198.79312133789062,
      "margin_dpo/margin_std": 215.6772918701172,
      "step": 520
    },
    {
      "epoch": 0.7650513950073421,
      "fcm_dpo/beta": 0.0029383916407823563,
      "fcm_dpo/delta": 0.026043057441711426,
      "fcm_dpo/margin": 195.25540161132812,
      "fcm_dpo/q_t": 0.3743501305580139,
      "grad_norm": 37.878440856933594,
      "learning_rate": 8.062991975753378e-08,
      "logits/chosen": -0.46159619092941284,
      "logits/rejected": -0.44562065601348877,
      "logps/chosen": -301.5888366699219,
      "logps/ref_chosen": -58.14292526245117,
      "logps/ref_rejected": -83.28060913085938,
      "logps/rejected": -521.98193359375,
      "loss": 0.9898,
      "margin_dpo/margin_mean": 195.2554168701172,
      "margin_dpo/margin_std": 218.83047485351562,
      "step": 521
    },
    {
      "epoch": 0.7665198237885462,
      "fcm_dpo/beta": 0.002963225357234478,
      "fcm_dpo/delta": 0.09250913560390472,
      "fcm_dpo/margin": 172.7684326171875,
      "fcm_dpo/q_t": 0.3848268687725067,
      "grad_norm": 41.32555389404297,
      "learning_rate": 7.968821348583643e-08,
      "logits/chosen": -0.45410579442977905,
      "logits/rejected": -0.44396814703941345,
      "logps/chosen": -321.1343994140625,
      "logps/ref_chosen": -46.54766845703125,
      "logps/ref_rejected": -66.01388549804688,
      "logps/rejected": -513.3690185546875,
      "loss": 1.0304,
      "margin_dpo/margin_mean": 172.7684326171875,
      "margin_dpo/margin_std": 213.2279052734375,
      "step": 522
    },
    {
      "epoch": 0.7679882525697503,
      "fcm_dpo/beta": 0.002987142652273178,
      "fcm_dpo/delta": 0.034964967519044876,
      "fcm_dpo/margin": 189.8165283203125,
      "fcm_dpo/q_t": 0.3737178146839142,
      "grad_norm": 38.12350082397461,
      "learning_rate": 7.875099508810484e-08,
      "logits/chosen": -0.5127274394035339,
      "logits/rejected": -0.48612135648727417,
      "logps/chosen": -336.416015625,
      "logps/ref_chosen": -61.76960372924805,
      "logps/ref_rejected": -83.76141357421875,
      "logps/rejected": -548.2243041992188,
      "loss": 1.0043,
      "margin_dpo/margin_mean": 189.8165283203125,
      "margin_dpo/margin_std": 229.8765869140625,
      "step": 523
    },
    {
      "epoch": 0.7694566813509545,
      "fcm_dpo/beta": 0.002963773440569639,
      "fcm_dpo/delta": -0.03581948205828667,
      "fcm_dpo/margin": 213.60824584960938,
      "fcm_dpo/q_t": 0.3576499819755554,
      "grad_norm": 43.33149719238281,
      "learning_rate": 7.781828926091535e-08,
      "logits/chosen": -0.5263937711715698,
      "logits/rejected": -0.49376797676086426,
      "logps/chosen": -350.075439453125,
      "logps/ref_chosen": -78.0720443725586,
      "logps/ref_rejected": -81.30198669433594,
      "logps/rejected": -566.9136352539062,
      "loss": 0.9439,
      "margin_dpo/margin_mean": 213.60824584960938,
      "margin_dpo/margin_std": 214.0634765625,
      "step": 524
    },
    {
      "epoch": 0.7709251101321586,
      "fcm_dpo/beta": 0.0028939242474734783,
      "fcm_dpo/delta": -0.140090212225914,
      "fcm_dpo/margin": 251.78009033203125,
      "fcm_dpo/q_t": 0.3388723134994507,
      "grad_norm": 34.86455154418945,
      "learning_rate": 7.689012058193384e-08,
      "logits/chosen": -0.4875791668891907,
      "logits/rejected": -0.4962403476238251,
      "logps/chosen": -318.8263244628906,
      "logps/ref_chosen": -50.827857971191406,
      "logps/ref_rejected": -100.05294036865234,
      "logps/rejected": -619.83154296875,
      "loss": 0.8763,
      "margin_dpo/margin_mean": 251.7801055908203,
      "margin_dpo/margin_std": 222.5386505126953,
      "step": 525
    },
    {
      "epoch": 0.7723935389133627,
      "fcm_dpo/beta": 0.0028644688427448273,
      "fcm_dpo/delta": -0.102485790848732,
      "fcm_dpo/margin": 242.66949462890625,
      "fcm_dpo/q_t": 0.34745296835899353,
      "grad_norm": 30.96253776550293,
      "learning_rate": 7.596651350926836e-08,
      "logits/chosen": -0.5117580890655518,
      "logits/rejected": -0.48653119802474976,
      "logps/chosen": -344.20849609375,
      "logps/ref_chosen": -63.167236328125,
      "logps/ref_rejected": -86.30934143066406,
      "logps/rejected": -610.0201416015625,
      "loss": 0.9091,
      "margin_dpo/margin_mean": 242.6695098876953,
      "margin_dpo/margin_std": 234.477294921875,
      "step": 526
    },
    {
      "epoch": 0.7738619676945668,
      "fcm_dpo/beta": 0.002855823142454028,
      "fcm_dpo/delta": 0.0754547268152237,
      "fcm_dpo/margin": 185.0948486328125,
      "fcm_dpo/q_t": 0.38190531730651855,
      "grad_norm": 37.24220275878906,
      "learning_rate": 7.504749238082414e-08,
      "logits/chosen": -0.551037073135376,
      "logits/rejected": -0.5188230276107788,
      "logps/chosen": -381.26397705078125,
      "logps/ref_chosen": -71.12867736816406,
      "logps/ref_rejected": -78.3425521850586,
      "logps/rejected": -573.57275390625,
      "loss": 1.004,
      "margin_dpo/margin_mean": 185.0948486328125,
      "margin_dpo/margin_std": 205.111572265625,
      "step": 527
    },
    {
      "epoch": 0.775330396475771,
      "fcm_dpo/beta": 0.0028600182849913836,
      "fcm_dpo/delta": -0.007810359820723534,
      "fcm_dpo/margin": 212.27090454101562,
      "fcm_dpo/q_t": 0.37373942136764526,
      "grad_norm": 35.34698486328125,
      "learning_rate": 7.413308141366254e-08,
      "logits/chosen": -0.5303002595901489,
      "logits/rejected": -0.509202241897583,
      "logps/chosen": -384.4117736816406,
      "logps/ref_chosen": -68.0894546508789,
      "logps/ref_rejected": -93.91006469726562,
      "logps/rejected": -622.5032958984375,
      "loss": 1.0079,
      "margin_dpo/margin_mean": 212.27090454101562,
      "margin_dpo/margin_std": 285.313720703125,
      "step": 528
    },
    {
      "epoch": 0.7767988252569751,
      "fcm_dpo/beta": 0.002924061380326748,
      "fcm_dpo/delta": 0.13557088375091553,
      "fcm_dpo/margin": 161.04995727539062,
      "fcm_dpo/q_t": 0.3966936767101288,
      "grad_norm": 46.321571350097656,
      "learning_rate": 7.322330470336313e-08,
      "logits/chosen": -0.49917709827423096,
      "logits/rejected": -0.49793556332588196,
      "logps/chosen": -410.2076416015625,
      "logps/ref_chosen": -55.57495880126953,
      "logps/ref_rejected": -89.20909118652344,
      "logps/rejected": -604.8917236328125,
      "loss": 1.0838,
      "margin_dpo/margin_mean": 161.04995727539062,
      "margin_dpo/margin_std": 237.48895263671875,
      "step": 529
    },
    {
      "epoch": 0.7782672540381792,
      "fcm_dpo/beta": 0.0028766831383109093,
      "fcm_dpo/delta": -0.14135703444480896,
      "fcm_dpo/margin": 253.9014892578125,
      "fcm_dpo/q_t": 0.3434082269668579,
      "grad_norm": 33.014739990234375,
      "learning_rate": 7.231818622338822e-08,
      "logits/chosen": -0.47442853450775146,
      "logits/rejected": -0.45949307084083557,
      "logps/chosen": -333.4599609375,
      "logps/ref_chosen": -47.601417541503906,
      "logps/ref_rejected": -87.2845230102539,
      "logps/rejected": -627.0445556640625,
      "loss": 0.9293,
      "margin_dpo/margin_mean": 253.90150451660156,
      "margin_dpo/margin_std": 285.0568542480469,
      "step": 530
    },
    {
      "epoch": 0.7797356828193832,
      "fcm_dpo/beta": 0.0028520459309220314,
      "fcm_dpo/delta": -0.02389775589108467,
      "fcm_dpo/margin": 218.2183837890625,
      "fcm_dpo/q_t": 0.36491817235946655,
      "grad_norm": 39.8703498840332,
      "learning_rate": 7.141774982445147e-08,
      "logits/chosen": -0.5686095952987671,
      "logits/rejected": -0.5440300107002258,
      "logps/chosen": -381.2401123046875,
      "logps/ref_chosen": -55.246063232421875,
      "logps/ref_rejected": -70.60598754882812,
      "logps/rejected": -614.8184204101562,
      "loss": 0.9667,
      "margin_dpo/margin_mean": 218.2183837890625,
      "margin_dpo/margin_std": 248.57521057128906,
      "step": 531
    },
    {
      "epoch": 0.7812041116005873,
      "fcm_dpo/beta": 0.0027955423574894667,
      "fcm_dpo/delta": -0.043103598058223724,
      "fcm_dpo/margin": 227.61962890625,
      "fcm_dpo/q_t": 0.36520931124687195,
      "grad_norm": 60.421783447265625,
      "learning_rate": 7.052201923388953e-08,
      "logits/chosen": -0.5076245069503784,
      "logits/rejected": -0.478635311126709,
      "logps/chosen": -414.247314453125,
      "logps/ref_chosen": -70.28601837158203,
      "logps/ref_rejected": -86.5913314819336,
      "logps/rejected": -658.1722412109375,
      "loss": 0.998,
      "margin_dpo/margin_mean": 227.61961364746094,
      "margin_dpo/margin_std": 282.49871826171875,
      "step": 532
    },
    {
      "epoch": 0.7826725403817915,
      "fcm_dpo/beta": 0.0028768163174390793,
      "fcm_dpo/delta": 0.06540602445602417,
      "fcm_dpo/margin": 186.47171020507812,
      "fcm_dpo/q_t": 0.38702893257141113,
      "grad_norm": 41.66936111450195,
      "learning_rate": 6.963101805503646e-08,
      "logits/chosen": -0.4856870174407959,
      "logits/rejected": -0.45800524950027466,
      "logps/chosen": -395.514892578125,
      "logps/ref_chosen": -64.8551025390625,
      "logps/ref_rejected": -76.58805847167969,
      "logps/rejected": -593.7196044921875,
      "loss": 1.0558,
      "margin_dpo/margin_mean": 186.47171020507812,
      "margin_dpo/margin_std": 268.0407409667969,
      "step": 533
    },
    {
      "epoch": 0.7841409691629956,
      "fcm_dpo/beta": 0.0028785793110728264,
      "fcm_dpo/delta": 0.0326891764998436,
      "fcm_dpo/margin": 197.72434997558594,
      "fcm_dpo/q_t": 0.37530991435050964,
      "grad_norm": 38.83979034423828,
      "learning_rate": 6.874476976660184e-08,
      "logits/chosen": -0.510763943195343,
      "logits/rejected": -0.49589037895202637,
      "logps/chosen": -392.6068115234375,
      "logps/ref_chosen": -60.119388580322266,
      "logps/ref_rejected": -78.54347229003906,
      "logps/rejected": -608.7552490234375,
      "loss": 1.0056,
      "margin_dpo/margin_mean": 197.724365234375,
      "margin_dpo/margin_std": 245.38973999023438,
      "step": 534
    },
    {
      "epoch": 0.7856093979441997,
      "fcm_dpo/beta": 0.0028537982143461704,
      "fcm_dpo/delta": -0.11794188618659973,
      "fcm_dpo/margin": 248.56651306152344,
      "fcm_dpo/q_t": 0.34491124749183655,
      "grad_norm": 37.186885833740234,
      "learning_rate": 6.786329772205246e-08,
      "logits/chosen": -0.5178133845329285,
      "logits/rejected": -0.49928510189056396,
      "logps/chosen": -351.3582763671875,
      "logps/ref_chosen": -54.330238342285156,
      "logps/ref_rejected": -96.30763244628906,
      "logps/rejected": -641.9022216796875,
      "loss": 0.899,
      "margin_dpo/margin_mean": 248.56649780273438,
      "margin_dpo/margin_std": 232.72039794921875,
      "step": 535
    },
    {
      "epoch": 0.7870778267254038,
      "fcm_dpo/beta": 0.002728927880525589,
      "fcm_dpo/delta": -0.21693715453147888,
      "fcm_dpo/margin": 293.05352783203125,
      "fcm_dpo/q_t": 0.33220839500427246,
      "grad_norm": 31.354650497436523,
      "learning_rate": 6.698662514899638e-08,
      "logits/chosen": -0.5063174962997437,
      "logits/rejected": -0.5071820616722107,
      "logps/chosen": -320.23358154296875,
      "logps/ref_chosen": -47.08053207397461,
      "logps/ref_rejected": -89.09783935546875,
      "logps/rejected": -655.304443359375,
      "loss": 0.8628,
      "margin_dpo/margin_mean": 293.05352783203125,
      "margin_dpo/margin_std": 300.212890625,
      "step": 536
    },
    {
      "epoch": 0.788546255506608,
      "fcm_dpo/beta": 0.0027053924277424812,
      "fcm_dpo/delta": -0.014062870293855667,
      "fcm_dpo/margin": 226.46954345703125,
      "fcm_dpo/q_t": 0.3674009442329407,
      "grad_norm": 38.02225875854492,
      "learning_rate": 6.611477514857114e-08,
      "logits/chosen": -0.47698020935058594,
      "logits/rejected": -0.4474596381187439,
      "logps/chosen": -329.1002197265625,
      "logps/ref_chosen": -57.747467041015625,
      "logps/ref_rejected": -70.43838500976562,
      "logps/rejected": -568.2606811523438,
      "loss": 0.9875,
      "margin_dpo/margin_mean": 226.46954345703125,
      "margin_dpo/margin_std": 272.7801513671875,
      "step": 537
    },
    {
      "epoch": 0.7900146842878121,
      "fcm_dpo/beta": 0.0026679965667426586,
      "fcm_dpo/delta": -0.04222983494400978,
      "fcm_dpo/margin": 239.66842651367188,
      "fcm_dpo/q_t": 0.35988813638687134,
      "grad_norm": 38.96491241455078,
      "learning_rate": 6.524777069483525e-08,
      "logits/chosen": -0.5299701690673828,
      "logits/rejected": -0.5038875341415405,
      "logps/chosen": -405.4573669433594,
      "logps/ref_chosen": -66.41594696044922,
      "logps/ref_rejected": -84.22808837890625,
      "logps/rejected": -662.9379272460938,
      "loss": 0.9442,
      "margin_dpo/margin_mean": 239.66842651367188,
      "margin_dpo/margin_std": 251.71209716796875,
      "step": 538
    },
    {
      "epoch": 0.7914831130690162,
      "fcm_dpo/beta": 0.0026902840472757816,
      "fcm_dpo/delta": 0.07448217272758484,
      "fcm_dpo/margin": 196.82864379882812,
      "fcm_dpo/q_t": 0.3829989731311798,
      "grad_norm": 29.287826538085938,
      "learning_rate": 6.438563463416221e-08,
      "logits/chosen": -0.5244389176368713,
      "logits/rejected": -0.4966394901275635,
      "logps/chosen": -368.57196044921875,
      "logps/ref_chosen": -58.492855072021484,
      "logps/ref_rejected": -91.85395050048828,
      "logps/rejected": -598.76171875,
      "loss": 1.0163,
      "margin_dpo/margin_mean": 196.82864379882812,
      "margin_dpo/margin_std": 235.43844604492188,
      "step": 539
    },
    {
      "epoch": 0.7929515418502202,
      "fcm_dpo/beta": 0.002647720742970705,
      "fcm_dpo/delta": -0.18606501817703247,
      "fcm_dpo/margin": 291.5929260253906,
      "fcm_dpo/q_t": 0.33625712990760803,
      "grad_norm": 37.2298698425293,
      "learning_rate": 6.352838968463919e-08,
      "logits/chosen": -0.5106328129768372,
      "logits/rejected": -0.5080406665802002,
      "logps/chosen": -361.15386962890625,
      "logps/ref_chosen": -63.482513427734375,
      "logps/ref_rejected": -116.42999267578125,
      "logps/rejected": -705.6942749023438,
      "loss": 0.8864,
      "margin_dpo/margin_mean": 291.5928955078125,
      "margin_dpo/margin_std": 289.0450744628906,
      "step": 540
    },
    {
      "epoch": 0.7944199706314243,
      "fcm_dpo/beta": 0.0026333509013056755,
      "fcm_dpo/delta": 0.12476900219917297,
      "fcm_dpo/margin": 182.57830810546875,
      "fcm_dpo/q_t": 0.3981369137763977,
      "grad_norm": 36.667198181152344,
      "learning_rate": 6.267605843546767e-08,
      "logits/chosen": -0.5467498302459717,
      "logits/rejected": -0.5339305400848389,
      "logps/chosen": -430.84771728515625,
      "logps/ref_chosen": -78.28036499023438,
      "logps/ref_rejected": -103.273681640625,
      "logps/rejected": -638.4193725585938,
      "loss": 1.0807,
      "margin_dpo/margin_mean": 182.57830810546875,
      "margin_dpo/margin_std": 271.44097900390625,
      "step": 541
    },
    {
      "epoch": 0.7958883994126285,
      "fcm_dpo/beta": 0.002633909694850445,
      "fcm_dpo/delta": -0.05343947932124138,
      "fcm_dpo/margin": 246.30075073242188,
      "fcm_dpo/q_t": 0.3598957359790802,
      "grad_norm": 37.33884811401367,
      "learning_rate": 6.182866334636888e-08,
      "logits/chosen": -0.5174318552017212,
      "logits/rejected": -0.5168588161468506,
      "logps/chosen": -388.119384765625,
      "logps/ref_chosen": -57.48497009277344,
      "logps/ref_rejected": -96.47506713867188,
      "logps/rejected": -673.4102783203125,
      "loss": 0.9588,
      "margin_dpo/margin_mean": 246.30075073242188,
      "margin_dpo/margin_std": 279.5341796875,
      "step": 542
    },
    {
      "epoch": 0.7973568281938326,
      "fcm_dpo/beta": 0.0026966403238475323,
      "fcm_dpo/delta": 0.13497576117515564,
      "fcm_dpo/margin": 174.83229064941406,
      "fcm_dpo/q_t": 0.40193819999694824,
      "grad_norm": 33.94786834716797,
      "learning_rate": 6.098622674699147e-08,
      "logits/chosen": -0.519938588142395,
      "logits/rejected": -0.5251273512840271,
      "logps/chosen": -390.4471435546875,
      "logps/ref_chosen": -60.61750793457031,
      "logps/ref_rejected": -105.59896850585938,
      "logps/rejected": -610.2608642578125,
      "loss": 1.099,
      "margin_dpo/margin_mean": 174.83230590820312,
      "margin_dpo/margin_std": 286.04852294921875,
      "step": 543
    },
    {
      "epoch": 0.7988252569750367,
      "fcm_dpo/beta": 0.002676740288734436,
      "fcm_dpo/delta": -0.09121545404195786,
      "fcm_dpo/margin": 255.8384246826172,
      "fcm_dpo/q_t": 0.3496483564376831,
      "grad_norm": 39.29571533203125,
      "learning_rate": 6.01487708363232e-08,
      "logits/chosen": -0.4991987347602844,
      "logits/rejected": -0.4953753650188446,
      "logps/chosen": -407.407470703125,
      "logps/ref_chosen": -59.642303466796875,
      "logps/ref_rejected": -100.95469665527344,
      "logps/rejected": -704.5582275390625,
      "loss": 0.9054,
      "margin_dpo/margin_mean": 255.8384246826172,
      "margin_dpo/margin_std": 249.53448486328125,
      "step": 544
    },
    {
      "epoch": 0.8002936857562408,
      "fcm_dpo/beta": 0.0026431018486618996,
      "fcm_dpo/delta": -0.0723901242017746,
      "fcm_dpo/margin": 252.58734130859375,
      "fcm_dpo/q_t": 0.3531061112880707,
      "grad_norm": 27.791589736938477,
      "learning_rate": 5.9316317682106294e-08,
      "logits/chosen": -0.49018025398254395,
      "logits/rejected": -0.4865438938140869,
      "logps/chosen": -402.95513916015625,
      "logps/ref_chosen": -67.64859771728516,
      "logps/ref_rejected": -95.90800476074219,
      "logps/rejected": -683.8018798828125,
      "loss": 0.9299,
      "margin_dpo/margin_mean": 252.58734130859375,
      "margin_dpo/margin_std": 258.5457763671875,
      "step": 545
    },
    {
      "epoch": 0.801762114537445,
      "fcm_dpo/beta": 0.0026444008108228445,
      "fcm_dpo/delta": 0.05075571686029434,
      "fcm_dpo/margin": 208.76841735839844,
      "fcm_dpo/q_t": 0.37396320700645447,
      "grad_norm": 34.60557174682617,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": -0.48540985584259033,
      "logits/rejected": -0.47076770663261414,
      "logps/chosen": -358.11322021484375,
      "logps/ref_chosen": -50.744232177734375,
      "logps/ref_rejected": -81.86622619628906,
      "logps/rejected": -598.003662109375,
      "loss": 0.9881,
      "margin_dpo/margin_mean": 208.7684326171875,
      "margin_dpo/margin_std": 221.3150634765625,
      "step": 546
    },
    {
      "epoch": 0.8032305433186491,
      "fcm_dpo/beta": 0.002690389985218644,
      "fcm_dpo/delta": 0.04553506523370743,
      "fcm_dpo/margin": 206.67840576171875,
      "fcm_dpo/q_t": 0.37883201241493225,
      "grad_norm": 47.69929122924805,
      "learning_rate": 5.7666507254280265e-08,
      "logits/chosen": -0.5229824185371399,
      "logits/rejected": -0.5123274326324463,
      "logps/chosen": -416.7408447265625,
      "logps/ref_chosen": -73.6877212524414,
      "logps/ref_rejected": -90.76136779785156,
      "logps/rejected": -640.492919921875,
      "loss": 1.0268,
      "margin_dpo/margin_mean": 206.67840576171875,
      "margin_dpo/margin_std": 272.1181945800781,
      "step": 547
    },
    {
      "epoch": 0.8046989720998532,
      "fcm_dpo/beta": 0.0026915257330983877,
      "fcm_dpo/delta": 0.021523747593164444,
      "fcm_dpo/margin": 215.3766632080078,
      "fcm_dpo/q_t": 0.37684351205825806,
      "grad_norm": 32.53667449951172,
      "learning_rate": 5.684919345471029e-08,
      "logits/chosen": -0.5300377011299133,
      "logits/rejected": -0.512967586517334,
      "logps/chosen": -406.23773193359375,
      "logps/ref_chosen": -65.24634552001953,
      "logps/ref_rejected": -94.11807250976562,
      "logps/rejected": -650.486083984375,
      "loss": 1.0016,
      "margin_dpo/margin_mean": 215.3766632080078,
      "margin_dpo/margin_std": 280.05401611328125,
      "step": 548
    },
    {
      "epoch": 0.8061674008810573,
      "fcm_dpo/beta": 0.00273230392485857,
      "fcm_dpo/delta": 0.08910378813743591,
      "fcm_dpo/margin": 188.5994873046875,
      "fcm_dpo/q_t": 0.38830626010894775,
      "grad_norm": 39.737220764160156,
      "learning_rate": 5.603696935852426e-08,
      "logits/chosen": -0.4929801821708679,
      "logits/rejected": -0.4741705656051636,
      "logps/chosen": -383.95849609375,
      "logps/ref_chosen": -49.21235656738281,
      "logps/ref_rejected": -73.91031646728516,
      "logps/rejected": -597.2559814453125,
      "loss": 1.0374,
      "margin_dpo/margin_mean": 188.5994873046875,
      "margin_dpo/margin_std": 250.06410217285156,
      "step": 549
    },
    {
      "epoch": 0.8076358296622613,
      "fcm_dpo/beta": 0.0027629181277006865,
      "fcm_dpo/delta": 0.0815601795911789,
      "fcm_dpo/margin": 189.18296813964844,
      "fcm_dpo/q_t": 0.38146719336509705,
      "grad_norm": 38.10478973388672,
      "learning_rate": 5.5229856368582376e-08,
      "logits/chosen": -0.4668349623680115,
      "logits/rejected": -0.4747786521911621,
      "logps/chosen": -374.70623779296875,
      "logps/ref_chosen": -56.80695343017578,
      "logps/ref_rejected": -95.12580871582031,
      "logps/rejected": -602.2080688476562,
      "loss": 1.0057,
      "margin_dpo/margin_mean": 189.18296813964844,
      "margin_dpo/margin_std": 203.63247680664062,
      "step": 550
    },
    {
      "epoch": 0.8091042584434655,
      "fcm_dpo/beta": 0.0027016932144761086,
      "fcm_dpo/delta": -0.18426547944545746,
      "fcm_dpo/margin": 284.74884033203125,
      "fcm_dpo/q_t": 0.3298325836658478,
      "grad_norm": 41.22262191772461,
      "learning_rate": 5.4427875753062734e-08,
      "logits/chosen": -0.4827745258808136,
      "logits/rejected": -0.48983511328697205,
      "logps/chosen": -374.94500732421875,
      "logps/ref_chosen": -59.10633087158203,
      "logps/ref_rejected": -111.67280578613281,
      "logps/rejected": -712.2603149414062,
      "loss": 0.8561,
      "margin_dpo/margin_mean": 284.74884033203125,
      "margin_dpo/margin_std": 247.297607421875,
      "step": 551
    },
    {
      "epoch": 0.8105726872246696,
      "fcm_dpo/beta": 0.002569342264905572,
      "fcm_dpo/delta": -0.22732627391815186,
      "fcm_dpo/margin": 313.23541259765625,
      "fcm_dpo/q_t": 0.32994934916496277,
      "grad_norm": 25.86652946472168,
      "learning_rate": 5.363104864490034e-08,
      "logits/chosen": -0.4956032335758209,
      "logits/rejected": -0.4835580885410309,
      "logps/chosen": -360.0677490234375,
      "logps/ref_chosen": -62.35459899902344,
      "logps/ref_rejected": -104.56210327148438,
      "logps/rejected": -715.5106201171875,
      "loss": 0.8604,
      "margin_dpo/margin_mean": 313.23541259765625,
      "margin_dpo/margin_std": 302.6081237792969,
      "step": 552
    },
    {
      "epoch": 0.8120411160058737,
      "fcm_dpo/beta": 0.0025847742799669504,
      "fcm_dpo/delta": 0.07955043762922287,
      "fcm_dpo/margin": 202.94935607910156,
      "fcm_dpo/q_t": 0.38608789443969727,
      "grad_norm": 36.25725555419922,
      "learning_rate": 5.2839396041230415e-08,
      "logits/chosen": -0.4990794062614441,
      "logits/rejected": -0.48621460795402527,
      "logps/chosen": -384.55615234375,
      "logps/ref_chosen": -68.25881958007812,
      "logps/ref_rejected": -98.0971450805664,
      "logps/rejected": -617.3438110351562,
      "loss": 1.0248,
      "margin_dpo/margin_mean": 202.94935607910156,
      "margin_dpo/margin_std": 253.7549285888672,
      "step": 553
    },
    {
      "epoch": 0.8135095447870778,
      "fcm_dpo/beta": 0.0025886285584419966,
      "fcm_dpo/delta": -0.09040947258472443,
      "fcm_dpo/margin": 264.222900390625,
      "fcm_dpo/q_t": 0.35030698776245117,
      "grad_norm": 31.871034622192383,
      "learning_rate": 5.205293880283551e-08,
      "logits/chosen": -0.5079976916313171,
      "logits/rejected": -0.4783214330673218,
      "logps/chosen": -389.82989501953125,
      "logps/ref_chosen": -67.94767761230469,
      "logps/ref_rejected": -89.78272247314453,
      "logps/rejected": -675.8878173828125,
      "loss": 0.9239,
      "margin_dpo/margin_mean": 264.222900390625,
      "margin_dpo/margin_std": 268.23663330078125,
      "step": 554
    },
    {
      "epoch": 0.8149779735682819,
      "fcm_dpo/beta": 0.0025238655507564545,
      "fcm_dpo/delta": -0.07741110026836395,
      "fcm_dpo/margin": 266.2066650390625,
      "fcm_dpo/q_t": 0.35779866576194763,
      "grad_norm": 37.653892517089844,
      "learning_rate": 5.127169765359515e-08,
      "logits/chosen": -0.5288249254226685,
      "logits/rejected": -0.5379273891448975,
      "logps/chosen": -376.24407958984375,
      "logps/ref_chosen": -53.33049011230469,
      "logps/ref_rejected": -108.47937774658203,
      "logps/rejected": -697.599609375,
      "loss": 0.9471,
      "margin_dpo/margin_mean": 266.2066650390625,
      "margin_dpo/margin_std": 305.3093566894531,
      "step": 555
    },
    {
      "epoch": 0.8164464023494861,
      "fcm_dpo/beta": 0.002562987618148327,
      "fcm_dpo/delta": 0.11464083939790726,
      "fcm_dpo/margin": 191.5848388671875,
      "fcm_dpo/q_t": 0.394670307636261,
      "grad_norm": 48.968658447265625,
      "learning_rate": 5.049569317994012e-08,
      "logits/chosen": -0.45059263706207275,
      "logits/rejected": -0.43395426869392395,
      "logps/chosen": -389.5137939453125,
      "logps/ref_chosen": -58.64447021484375,
      "logps/ref_rejected": -101.34040832519531,
      "logps/rejected": -623.7945556640625,
      "loss": 1.0517,
      "margin_dpo/margin_mean": 191.5848388671875,
      "margin_dpo/margin_std": 260.8348388671875,
      "step": 556
    },
    {
      "epoch": 0.8179148311306902,
      "fcm_dpo/beta": 0.002571512944996357,
      "fcm_dpo/delta": 0.0015640445053577423,
      "fcm_dpo/margin": 232.67929077148438,
      "fcm_dpo/q_t": 0.37047192454338074,
      "grad_norm": 40.12662887573242,
      "learning_rate": 4.9724945830310144e-08,
      "logits/chosen": -0.49551674723625183,
      "logits/rejected": -0.49951496720314026,
      "logps/chosen": -406.4974365234375,
      "logps/ref_chosen": -67.84066009521484,
      "logps/ref_rejected": -109.93965911865234,
      "logps/rejected": -681.2757568359375,
      "loss": 0.9819,
      "margin_dpo/margin_mean": 232.6792755126953,
      "margin_dpo/margin_std": 272.00067138671875,
      "step": 557
    },
    {
      "epoch": 0.8193832599118943,
      "fcm_dpo/beta": 0.0024929968640208244,
      "fcm_dpo/delta": -0.20179688930511475,
      "fcm_dpo/margin": 314.78643798828125,
      "fcm_dpo/q_t": 0.3285095691680908,
      "grad_norm": 42.07298278808594,
      "learning_rate": 4.8959475914614554e-08,
      "logits/chosen": -0.5093335509300232,
      "logits/rejected": -0.4982109069824219,
      "logps/chosen": -371.7452392578125,
      "logps/ref_chosen": -62.36824035644531,
      "logps/ref_rejected": -102.16102600097656,
      "logps/rejected": -726.324462890625,
      "loss": 0.866,
      "margin_dpo/margin_mean": 314.78643798828125,
      "margin_dpo/margin_std": 296.3161926269531,
      "step": 558
    },
    {
      "epoch": 0.8208516886930984,
      "fcm_dpo/beta": 0.0024765529669821262,
      "fcm_dpo/delta": 0.002801853232085705,
      "fcm_dpo/margin": 241.2025146484375,
      "fcm_dpo/q_t": 0.3673396706581116,
      "grad_norm": 28.609155654907227,
      "learning_rate": 4.8199303603697614e-08,
      "logits/chosen": -0.5868000984191895,
      "logits/rejected": -0.5612949728965759,
      "logps/chosen": -414.205322265625,
      "logps/ref_chosen": -60.752323150634766,
      "logps/ref_rejected": -93.44229125976562,
      "logps/rejected": -688.0977783203125,
      "loss": 0.9684,
      "margin_dpo/margin_mean": 241.2025146484375,
      "margin_dpo/margin_std": 260.7628173828125,
      "step": 559
    },
    {
      "epoch": 0.8223201174743024,
      "fcm_dpo/beta": 0.0025273123756051064,
      "fcm_dpo/delta": 0.15692362189292908,
      "fcm_dpo/margin": 178.22430419921875,
      "fcm_dpo/q_t": 0.3967198431491852,
      "grad_norm": 32.44375991821289,
      "learning_rate": 4.7444448928806615e-08,
      "logits/chosen": -0.5102473497390747,
      "logits/rejected": -0.4792303442955017,
      "logps/chosen": -362.985595703125,
      "logps/ref_chosen": -58.10382080078125,
      "logps/ref_rejected": -79.99122619628906,
      "logps/rejected": -563.0972900390625,
      "loss": 1.0412,
      "margin_dpo/margin_mean": 178.22430419921875,
      "margin_dpo/margin_std": 187.90447998046875,
      "step": 560
    },
    {
      "epoch": 0.8237885462555066,
      "fcm_dpo/beta": 0.0026212262455374002,
      "fcm_dpo/delta": 0.1312703937292099,
      "fcm_dpo/margin": 180.79673767089844,
      "fcm_dpo/q_t": 0.3930835723876953,
      "grad_norm": 34.945152282714844,
      "learning_rate": 4.669493178106432e-08,
      "logits/chosen": -0.46148979663848877,
      "logits/rejected": -0.4639427661895752,
      "logps/chosen": -401.359619140625,
      "logps/ref_chosen": -50.912879943847656,
      "logps/ref_rejected": -99.06856536865234,
      "logps/rejected": -630.3120727539062,
      "loss": 1.0486,
      "margin_dpo/margin_mean": 180.79672241210938,
      "margin_dpo/margin_std": 226.11021423339844,
      "step": 561
    },
    {
      "epoch": 0.8252569750367107,
      "fcm_dpo/beta": 0.0025770908687263727,
      "fcm_dpo/delta": -0.08462567627429962,
      "fcm_dpo/margin": 262.9320068359375,
      "fcm_dpo/q_t": 0.35398316383361816,
      "grad_norm": 43.5445442199707,
      "learning_rate": 4.5950771910944596e-08,
      "logits/chosen": -0.5065794587135315,
      "logits/rejected": -0.4925631284713745,
      "logps/chosen": -369.410400390625,
      "logps/ref_chosen": -59.46440124511719,
      "logps/ref_rejected": -96.54266357421875,
      "logps/rejected": -669.420654296875,
      "loss": 0.9218,
      "margin_dpo/margin_mean": 262.9320068359375,
      "margin_dpo/margin_std": 273.57208251953125,
      "step": 562
    },
    {
      "epoch": 0.8267254038179148,
      "fcm_dpo/beta": 0.002649601548910141,
      "fcm_dpo/delta": 0.11059555411338806,
      "fcm_dpo/margin": 185.759521484375,
      "fcm_dpo/q_t": 0.38974693417549133,
      "grad_norm": 38.421958923339844,
      "learning_rate": 4.521198892775202e-08,
      "logits/chosen": -0.4199938178062439,
      "logits/rejected": -0.41303732991218567,
      "logps/chosen": -402.4117736816406,
      "logps/ref_chosen": -60.60819625854492,
      "logps/ref_rejected": -94.56770324707031,
      "logps/rejected": -622.1307983398438,
      "loss": 1.0664,
      "margin_dpo/margin_mean": 185.759521484375,
      "margin_dpo/margin_std": 253.02633666992188,
      "step": 563
    },
    {
      "epoch": 0.8281938325991189,
      "fcm_dpo/beta": 0.002637513680383563,
      "fcm_dpo/delta": -0.013331831432878971,
      "fcm_dpo/margin": 232.2291717529297,
      "fcm_dpo/q_t": 0.35960835218429565,
      "grad_norm": 34.64234161376953,
      "learning_rate": 4.447860229910544e-08,
      "logits/chosen": -0.5244062542915344,
      "logits/rejected": -0.4892103672027588,
      "logps/chosen": -395.12603759765625,
      "logps/ref_chosen": -74.26837921142578,
      "logps/ref_rejected": -93.23818969726562,
      "logps/rejected": -646.3250732421875,
      "loss": 0.9258,
      "margin_dpo/margin_mean": 232.22918701171875,
      "margin_dpo/margin_std": 195.64129638671875,
      "step": 564
    },
    {
      "epoch": 0.8296622613803231,
      "fcm_dpo/beta": 0.0026156511157751083,
      "fcm_dpo/delta": -0.02939961664378643,
      "fcm_dpo/margin": 239.86215209960938,
      "fcm_dpo/q_t": 0.37034720182418823,
      "grad_norm": 41.29433059692383,
      "learning_rate": 4.375063135042445e-08,
      "logits/chosen": -0.5047638416290283,
      "logits/rejected": -0.48947322368621826,
      "logps/chosen": -412.850830078125,
      "logps/ref_chosen": -69.0199203491211,
      "logps/ref_rejected": -85.7789306640625,
      "logps/rejected": -669.4719848632812,
      "loss": 0.9974,
      "margin_dpo/margin_mean": 239.86215209960938,
      "margin_dpo/margin_std": 316.59368896484375,
      "step": 565
    },
    {
      "epoch": 0.8311306901615272,
      "fcm_dpo/beta": 0.002625175751745701,
      "fcm_dpo/delta": -0.04155733436346054,
      "fcm_dpo/margin": 242.91168212890625,
      "fcm_dpo/q_t": 0.36505499482154846,
      "grad_norm": 34.86541748046875,
      "learning_rate": 4.3028095264420525e-08,
      "logits/chosen": -0.46465927362442017,
      "logits/rejected": -0.46675699949264526,
      "logps/chosen": -391.67828369140625,
      "logps/ref_chosen": -66.5453109741211,
      "logps/ref_rejected": -103.86932373046875,
      "logps/rejected": -671.9140014648438,
      "loss": 0.9961,
      "margin_dpo/margin_mean": 242.91168212890625,
      "margin_dpo/margin_std": 304.49200439453125,
      "step": 566
    },
    {
      "epoch": 0.8325991189427313,
      "fcm_dpo/beta": 0.0026026610285043716,
      "fcm_dpo/delta": 0.01975536160171032,
      "fcm_dpo/margin": 223.38568115234375,
      "fcm_dpo/q_t": 0.3706703186035156,
      "grad_norm": 39.21678924560547,
      "learning_rate": 4.231101308059165e-08,
      "logits/chosen": -0.4942345917224884,
      "logits/rejected": -0.4810565710067749,
      "logps/chosen": -371.0935974121094,
      "logps/ref_chosen": -52.85829544067383,
      "logps/ref_rejected": -85.37095642089844,
      "logps/rejected": -626.991943359375,
      "loss": 0.9786,
      "margin_dpo/margin_mean": 223.38568115234375,
      "margin_dpo/margin_std": 245.55088806152344,
      "step": 567
    },
    {
      "epoch": 0.8340675477239354,
      "fcm_dpo/beta": 0.00256351288408041,
      "fcm_dpo/delta": -0.08483800292015076,
      "fcm_dpo/margin": 264.6658630371094,
      "fcm_dpo/q_t": 0.34867846965789795,
      "grad_norm": 28.661598205566406,
      "learning_rate": 4.1599403694720145e-08,
      "logits/chosen": -0.44072189927101135,
      "logits/rejected": -0.44052451848983765,
      "logps/chosen": -348.7388916015625,
      "logps/ref_chosen": -45.1923828125,
      "logps/ref_rejected": -89.09236907958984,
      "logps/rejected": -657.3046875,
      "loss": 0.9103,
      "margin_dpo/margin_mean": 264.66583251953125,
      "margin_dpo/margin_std": 249.04701232910156,
      "step": 568
    },
    {
      "epoch": 0.8355359765051396,
      "fcm_dpo/beta": 0.0026110014878213406,
      "fcm_dpo/delta": 0.09269944578409195,
      "fcm_dpo/margin": 195.74871826171875,
      "fcm_dpo/q_t": 0.3880962133407593,
      "grad_norm": 41.08034896850586,
      "learning_rate": 4.089328585837512e-08,
      "logits/chosen": -0.4750403165817261,
      "logits/rejected": -0.46098893880844116,
      "logps/chosen": -413.0689697265625,
      "logps/ref_chosen": -63.72056198120117,
      "logps/ref_rejected": -79.10325622558594,
      "logps/rejected": -624.2003173828125,
      "loss": 1.0448,
      "margin_dpo/margin_mean": 195.74871826171875,
      "margin_dpo/margin_std": 259.3326110839844,
      "step": 569
    },
    {
      "epoch": 0.8370044052863436,
      "fcm_dpo/beta": 0.002593663055449724,
      "fcm_dpo/delta": -0.01997148059308529,
      "fcm_dpo/margin": 238.44491577148438,
      "fcm_dpo/q_t": 0.3628031015396118,
      "grad_norm": 33.436527252197266,
      "learning_rate": 4.019267817841834e-08,
      "logits/chosen": -0.5121582746505737,
      "logits/rejected": -0.4927957057952881,
      "logps/chosen": -382.0532531738281,
      "logps/ref_chosen": -61.61454391479492,
      "logps/ref_rejected": -82.14186096191406,
      "logps/rejected": -641.0254516601562,
      "loss": 0.9467,
      "margin_dpo/margin_mean": 238.44491577148438,
      "margin_dpo/margin_std": 236.92184448242188,
      "step": 570
    },
    {
      "epoch": 0.8384728340675477,
      "fcm_dpo/beta": 0.002562489826232195,
      "fcm_dpo/delta": -0.06662685424089432,
      "fcm_dpo/margin": 258.15692138671875,
      "fcm_dpo/q_t": 0.3554573655128479,
      "grad_norm": 45.022804260253906,
      "learning_rate": 3.9497599116513705e-08,
      "logits/chosen": -0.4584212005138397,
      "logits/rejected": -0.4587005376815796,
      "logps/chosen": -373.08074951171875,
      "logps/ref_chosen": -53.05406188964844,
      "logps/ref_rejected": -91.33682250976562,
      "logps/rejected": -669.5204467773438,
      "loss": 0.9544,
      "margin_dpo/margin_mean": 258.15692138671875,
      "margin_dpo/margin_std": 291.7584228515625,
      "step": 571
    },
    {
      "epoch": 0.8399412628487518,
      "fcm_dpo/beta": 0.0025576986372470856,
      "fcm_dpo/delta": -0.0027488693594932556,
      "fcm_dpo/margin": 235.54122924804688,
      "fcm_dpo/q_t": 0.3760080337524414,
      "grad_norm": 36.91169357299805,
      "learning_rate": 3.880806698864086e-08,
      "logits/chosen": -0.4423763155937195,
      "logits/rejected": -0.43832147121429443,
      "logps/chosen": -395.4063720703125,
      "logps/ref_chosen": -48.45928955078125,
      "logps/ref_rejected": -83.55703735351562,
      "logps/rejected": -666.0453491210938,
      "loss": 1.0158,
      "margin_dpo/margin_mean": 235.54122924804688,
      "margin_dpo/margin_std": 325.2923889160156,
      "step": 572
    },
    {
      "epoch": 0.8414096916299559,
      "fcm_dpo/beta": 0.002582044806331396,
      "fcm_dpo/delta": 0.0030916333198547363,
      "fcm_dpo/margin": 230.921142578125,
      "fcm_dpo/q_t": 0.36895716190338135,
      "grad_norm": 34.18012237548828,
      "learning_rate": 3.812409996461275e-08,
      "logits/chosen": -0.4861369729042053,
      "logits/rejected": -0.4677738547325134,
      "logps/chosen": -373.53863525390625,
      "logps/ref_chosen": -51.62262725830078,
      "logps/ref_rejected": -85.32499694824219,
      "logps/rejected": -638.162109375,
      "loss": 0.9723,
      "margin_dpo/margin_mean": 230.921142578125,
      "margin_dpo/margin_std": 252.47198486328125,
      "step": 573
    },
    {
      "epoch": 0.8428781204111601,
      "fcm_dpo/beta": 0.0025438209995627403,
      "fcm_dpo/delta": -0.07464863359928131,
      "fcm_dpo/margin": 263.26629638671875,
      "fcm_dpo/q_t": 0.3528655171394348,
      "grad_norm": 30.160154342651367,
      "learning_rate": 3.74457160675965e-08,
      "logits/chosen": -0.5087519288063049,
      "logits/rejected": -0.5088317394256592,
      "logps/chosen": -369.17791748046875,
      "logps/ref_chosen": -51.04446029663086,
      "logps/ref_rejected": -92.80640411376953,
      "logps/rejected": -674.2061157226562,
      "loss": 0.9268,
      "margin_dpo/margin_mean": 263.26629638671875,
      "margin_dpo/margin_std": 267.4271240234375,
      "step": 574
    },
    {
      "epoch": 0.8443465491923642,
      "fcm_dpo/beta": 0.0025178524665534496,
      "fcm_dpo/delta": 0.048557281494140625,
      "fcm_dpo/margin": 219.31741333007812,
      "fcm_dpo/q_t": 0.3802528381347656,
      "grad_norm": 39.644771575927734,
      "learning_rate": 3.677293317363864e-08,
      "logits/chosen": -0.4346257448196411,
      "logits/rejected": -0.4225711226463318,
      "logps/chosen": -427.237060546875,
      "logps/ref_chosen": -71.7901382446289,
      "logps/ref_rejected": -95.38619995117188,
      "logps/rejected": -670.1505737304688,
      "loss": 1.0547,
      "margin_dpo/margin_mean": 219.31741333007812,
      "margin_dpo/margin_std": 301.30224609375,
      "step": 575
    },
    {
      "epoch": 0.8458149779735683,
      "fcm_dpo/beta": 0.0025599338114261627,
      "fcm_dpo/delta": 0.06943172216415405,
      "fcm_dpo/margin": 208.48135375976562,
      "fcm_dpo/q_t": 0.38438957929611206,
      "grad_norm": 35.327213287353516,
      "learning_rate": 3.6105769011194224e-08,
      "logits/chosen": -0.4814299941062927,
      "logits/rejected": -0.4900047481060028,
      "logps/chosen": -380.19384765625,
      "logps/ref_chosen": -54.262962341308594,
      "logps/ref_rejected": -100.75428009033203,
      "logps/rejected": -635.16650390625,
      "loss": 1.0487,
      "margin_dpo/margin_mean": 208.48135375976562,
      "margin_dpo/margin_std": 286.2848205566406,
      "step": 576
    },
    {
      "epoch": 0.8472834067547724,
      "fcm_dpo/beta": 0.0025888322852551937,
      "fcm_dpo/delta": -0.0014976090751588345,
      "fcm_dpo/margin": 232.30636596679688,
      "fcm_dpo/q_t": 0.36417728662490845,
      "grad_norm": 31.62501335144043,
      "learning_rate": 3.5444241160659304e-08,
      "logits/chosen": -0.4902263581752777,
      "logits/rejected": -0.4602411389350891,
      "logps/chosen": -351.259033203125,
      "logps/ref_chosen": -61.909706115722656,
      "logps/ref_rejected": -84.07069396972656,
      "logps/rejected": -605.7263793945312,
      "loss": 0.9617,
      "margin_dpo/margin_mean": 232.30638122558594,
      "margin_dpo/margin_std": 235.41848754882812,
      "step": 577
    },
    {
      "epoch": 0.8487518355359766,
      "fcm_dpo/beta": 0.0025621510576456785,
      "fcm_dpo/delta": -0.015007011592388153,
      "fcm_dpo/margin": 239.20761108398438,
      "fcm_dpo/q_t": 0.3611851930618286,
      "grad_norm": 30.414794921875,
      "learning_rate": 3.478836705390808e-08,
      "logits/chosen": -0.3913380205631256,
      "logits/rejected": -0.3857693076133728,
      "logps/chosen": -340.91937255859375,
      "logps/ref_chosen": -49.26368713378906,
      "logps/ref_rejected": -83.4362564086914,
      "logps/rejected": -614.299560546875,
      "loss": 0.9374,
      "margin_dpo/margin_mean": 239.20761108398438,
      "margin_dpo/margin_std": 213.71923828125,
      "step": 578
    },
    {
      "epoch": 0.8502202643171806,
      "fcm_dpo/beta": 0.0026131728664040565,
      "fcm_dpo/delta": 0.09434732794761658,
      "fcm_dpo/margin": 195.37240600585938,
      "fcm_dpo/q_t": 0.3860264718532562,
      "grad_norm": 37.3227424621582,
      "learning_rate": 3.41381639738331e-08,
      "logits/chosen": -0.47116217017173767,
      "logits/rejected": -0.4621252417564392,
      "logps/chosen": -367.91162109375,
      "logps/ref_chosen": -58.88581848144531,
      "logps/ref_rejected": -94.78762817382812,
      "logps/rejected": -599.185791015625,
      "loss": 1.0322,
      "margin_dpo/margin_mean": 195.37240600585938,
      "margin_dpo/margin_std": 244.83892822265625,
      "step": 579
    },
    {
      "epoch": 0.8516886930983847,
      "fcm_dpo/beta": 0.0025751434732228518,
      "fcm_dpo/delta": -0.1056961864233017,
      "fcm_dpo/margin": 270.815673828125,
      "fcm_dpo/q_t": 0.34934502840042114,
      "grad_norm": 27.840450286865234,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -0.4681975841522217,
      "logits/rejected": -0.4637313485145569,
      "logps/chosen": -287.3629150390625,
      "logps/ref_chosen": -48.70683670043945,
      "logps/ref_rejected": -81.7583999633789,
      "logps/rejected": -591.2301635742188,
      "loss": 0.9179,
      "margin_dpo/margin_mean": 270.815673828125,
      "margin_dpo/margin_std": 285.8489685058594,
      "step": 580
    },
    {
      "epoch": 0.8531571218795888,
      "fcm_dpo/beta": 0.002611548872664571,
      "fcm_dpo/delta": 0.10047388076782227,
      "fcm_dpo/margin": 193.2278289794922,
      "fcm_dpo/q_t": 0.3879375159740448,
      "grad_norm": 31.592443466186523,
      "learning_rate": 3.285483927764726e-08,
      "logits/chosen": -0.5031832456588745,
      "logits/rejected": -0.4919954836368561,
      "logps/chosen": -390.05816650390625,
      "logps/ref_chosen": -62.22235107421875,
      "logps/ref_rejected": -91.73568725585938,
      "logps/rejected": -612.79931640625,
      "loss": 1.0351,
      "margin_dpo/margin_mean": 193.22781372070312,
      "margin_dpo/margin_std": 241.17733764648438,
      "step": 581
    },
    {
      "epoch": 0.8546255506607929,
      "fcm_dpo/beta": 0.002609074581414461,
      "fcm_dpo/delta": -0.10906706750392914,
      "fcm_dpo/margin": 268.527587890625,
      "fcm_dpo/q_t": 0.34740155935287476,
      "grad_norm": 30.773542404174805,
      "learning_rate": 3.222175147833556e-08,
      "logits/chosen": -0.4649215638637543,
      "logits/rejected": -0.4725267291069031,
      "logps/chosen": -334.1369323730469,
      "logps/ref_chosen": -58.228660583496094,
      "logps/ref_rejected": -110.06959533691406,
      "logps/rejected": -654.50537109375,
      "loss": 0.9329,
      "margin_dpo/margin_mean": 268.527587890625,
      "margin_dpo/margin_std": 281.925048828125,
      "step": 582
    },
    {
      "epoch": 0.856093979441997,
      "fcm_dpo/beta": 0.0025950425770133734,
      "fcm_dpo/delta": 0.12001574784517288,
      "fcm_dpo/margin": 187.0833282470703,
      "fcm_dpo/q_t": 0.39638835191726685,
      "grad_norm": 38.84540557861328,
      "learning_rate": 3.159440233840763e-08,
      "logits/chosen": -0.4593685269355774,
      "logits/rejected": -0.44617611169815063,
      "logps/chosen": -376.9676513671875,
      "logps/ref_chosen": -56.86286163330078,
      "logps/ref_rejected": -88.4039306640625,
      "logps/rejected": -595.5921020507812,
      "loss": 1.0772,
      "margin_dpo/margin_mean": 187.0833282470703,
      "margin_dpo/margin_std": 274.46728515625,
      "step": 583
    },
    {
      "epoch": 0.8575624082232012,
      "fcm_dpo/beta": 0.0025689757894724607,
      "fcm_dpo/delta": -0.1522265374660492,
      "fcm_dpo/margin": 288.31964111328125,
      "fcm_dpo/q_t": 0.335731565952301,
      "grad_norm": 27.584877014160156,
      "learning_rate": 3.0972808389096635e-08,
      "logits/chosen": -0.48634427785873413,
      "logits/rejected": -0.47195422649383545,
      "logps/chosen": -321.5301818847656,
      "logps/ref_chosen": -56.90068054199219,
      "logps/ref_rejected": -97.63606262207031,
      "logps/rejected": -650.585205078125,
      "loss": 0.8585,
      "margin_dpo/margin_mean": 288.3196105957031,
      "margin_dpo/margin_std": 241.53466796875,
      "step": 584
    },
    {
      "epoch": 0.8590308370044053,
      "fcm_dpo/beta": 0.0025610639713704586,
      "fcm_dpo/delta": 0.02514045313000679,
      "fcm_dpo/margin": 225.02957153320312,
      "fcm_dpo/q_t": 0.37413886189460754,
      "grad_norm": 35.07202911376953,
      "learning_rate": 3.035698600998121e-08,
      "logits/chosen": -0.49202650785446167,
      "logits/rejected": -0.4775129556655884,
      "logps/chosen": -386.05023193359375,
      "logps/ref_chosen": -60.973968505859375,
      "logps/ref_rejected": -84.16952514648438,
      "logps/rejected": -634.275390625,
      "loss": 0.9943,
      "margin_dpo/margin_mean": 225.02957153320312,
      "margin_dpo/margin_std": 258.579345703125,
      "step": 585
    },
    {
      "epoch": 0.8604992657856094,
      "fcm_dpo/beta": 0.0026022857055068016,
      "fcm_dpo/delta": 0.09514912962913513,
      "fcm_dpo/margin": 195.86416625976562,
      "fcm_dpo/q_t": 0.38909363746643066,
      "grad_norm": 41.226749420166016,
      "learning_rate": 2.974695142855388e-08,
      "logits/chosen": -0.44712215662002563,
      "logits/rejected": -0.4477439820766449,
      "logps/chosen": -386.0147705078125,
      "logps/ref_chosen": -56.85559844970703,
      "logps/ref_rejected": -91.80261993408203,
      "logps/rejected": -616.825927734375,
      "loss": 1.0511,
      "margin_dpo/margin_mean": 195.86416625976562,
      "margin_dpo/margin_std": 270.68035888671875,
      "step": 586
    },
    {
      "epoch": 0.8619676945668135,
      "fcm_dpo/beta": 0.0026168739423155785,
      "fcm_dpo/delta": 0.0030817219521850348,
      "fcm_dpo/margin": 228.1727294921875,
      "fcm_dpo/q_t": 0.368298202753067,
      "grad_norm": 30.468931198120117,
      "learning_rate": 2.9142720719793122e-08,
      "logits/chosen": -0.48182883858680725,
      "logits/rejected": -0.48729079961776733,
      "logps/chosen": -289.46923828125,
      "logps/ref_chosen": -44.69159698486328,
      "logps/ref_rejected": -82.62385559082031,
      "logps/rejected": -555.57421875,
      "loss": 0.9793,
      "margin_dpo/margin_mean": 228.17274475097656,
      "margin_dpo/margin_std": 261.90203857421875,
      "step": 587
    },
    {
      "epoch": 0.8634361233480177,
      "fcm_dpo/beta": 0.0026173333171755075,
      "fcm_dpo/delta": 0.0325515940785408,
      "fcm_dpo/margin": 217.41366577148438,
      "fcm_dpo/q_t": 0.3760989308357239,
      "grad_norm": 35.79490280151367,
      "learning_rate": 2.8544309805740018e-08,
      "logits/chosen": -0.5010935664176941,
      "logits/rejected": -0.5093272924423218,
      "logps/chosen": -365.68310546875,
      "logps/ref_chosen": -50.29494857788086,
      "logps/ref_rejected": -107.36988067626953,
      "logps/rejected": -640.1717529296875,
      "loss": 0.9967,
      "margin_dpo/margin_mean": 217.41366577148438,
      "margin_dpo/margin_std": 257.3772277832031,
      "step": 588
    },
    {
      "epoch": 0.8649045521292217,
      "fcm_dpo/beta": 0.002576986327767372,
      "fcm_dpo/delta": -0.15281669795513153,
      "fcm_dpo/margin": 287.77593994140625,
      "fcm_dpo/q_t": 0.34083396196365356,
      "grad_norm": 29.585044860839844,
      "learning_rate": 2.7951734455078786e-08,
      "logits/chosen": -0.48489910364151,
      "logits/rejected": -0.47208333015441895,
      "logps/chosen": -363.57073974609375,
      "logps/ref_chosen": -59.929908752441406,
      "logps/ref_rejected": -111.65534973144531,
      "logps/rejected": -703.0721435546875,
      "loss": 0.8936,
      "margin_dpo/margin_mean": 287.77593994140625,
      "margin_dpo/margin_std": 287.7421875,
      "step": 589
    },
    {
      "epoch": 0.8663729809104258,
      "fcm_dpo/beta": 0.0025251221377402544,
      "fcm_dpo/delta": -0.0640040785074234,
      "fcm_dpo/margin": 261.2105712890625,
      "fcm_dpo/q_t": 0.3543946146965027,
      "grad_norm": 30.528892517089844,
      "learning_rate": 2.736501028272095e-08,
      "logits/chosen": -0.5115870237350464,
      "logits/rejected": -0.5127223134040833,
      "logps/chosen": -343.80242919921875,
      "logps/ref_chosen": -55.80979537963867,
      "logps/ref_rejected": -106.06282043457031,
      "logps/rejected": -655.2659912109375,
      "loss": 0.9296,
      "margin_dpo/margin_mean": 261.2105712890625,
      "margin_dpo/margin_std": 264.1900939941406,
      "step": 590
    },
    {
      "epoch": 0.8678414096916299,
      "fcm_dpo/beta": 0.002512441948056221,
      "fcm_dpo/delta": -0.01745918020606041,
      "fcm_dpo/margin": 245.31549072265625,
      "fcm_dpo/q_t": 0.3611946403980255,
      "grad_norm": 34.154884338378906,
      "learning_rate": 2.678415274939408e-08,
      "logits/chosen": -0.4548659920692444,
      "logits/rejected": -0.4189767837524414,
      "logps/chosen": -348.8348693847656,
      "logps/ref_chosen": -56.24061965942383,
      "logps/ref_rejected": -83.78629302978516,
      "logps/rejected": -621.696044921875,
      "loss": 0.9438,
      "margin_dpo/margin_mean": 245.3155059814453,
      "margin_dpo/margin_std": 240.31198120117188,
      "step": 591
    },
    {
      "epoch": 0.869309838472834,
      "fcm_dpo/beta": 0.0025183933321386576,
      "fcm_dpo/delta": 0.03698636591434479,
      "fcm_dpo/margin": 224.35015869140625,
      "fcm_dpo/q_t": 0.37596046924591064,
      "grad_norm": 36.619808197021484,
      "learning_rate": 2.6209177161234442e-08,
      "logits/chosen": -0.41440728306770325,
      "logits/rejected": -0.39510399103164673,
      "logps/chosen": -373.01190185546875,
      "logps/ref_chosen": -47.94025421142578,
      "logps/ref_rejected": -75.73287963867188,
      "logps/rejected": -625.1546630859375,
      "loss": 1.0503,
      "margin_dpo/margin_mean": 224.35015869140625,
      "margin_dpo/margin_std": 322.49993896484375,
      "step": 592
    },
    {
      "epoch": 0.8707782672540382,
      "fcm_dpo/beta": 0.002595474012196064,
      "fcm_dpo/delta": 0.17214420437812805,
      "fcm_dpo/margin": 167.77935791015625,
      "fcm_dpo/q_t": 0.4035273790359497,
      "grad_norm": 44.139076232910156,
      "learning_rate": 2.564009866938349e-08,
      "logits/chosen": -0.4011484682559967,
      "logits/rejected": -0.3781154155731201,
      "logps/chosen": -341.73126220703125,
      "logps/ref_chosen": -48.690757751464844,
      "logps/ref_rejected": -60.90800094604492,
      "logps/rejected": -521.7278442382812,
      "loss": 1.0966,
      "margin_dpo/margin_mean": 167.7793426513672,
      "margin_dpo/margin_std": 248.49221801757812,
      "step": 593
    },
    {
      "epoch": 0.8722466960352423,
      "fcm_dpo/beta": 0.00264896173030138,
      "fcm_dpo/delta": -0.018487006425857544,
      "fcm_dpo/margin": 231.80612182617188,
      "fcm_dpo/q_t": 0.36759766936302185,
      "grad_norm": 32.71717071533203,
      "learning_rate": 2.5076932269588708e-08,
      "logits/chosen": -0.5203255414962769,
      "logits/rejected": -0.49811041355133057,
      "logps/chosen": -332.0072021484375,
      "logps/ref_chosen": -54.93488693237305,
      "logps/ref_rejected": -86.09967803955078,
      "logps/rejected": -594.9780883789062,
      "loss": 0.9936,
      "margin_dpo/margin_mean": 231.80612182617188,
      "margin_dpo/margin_std": 272.37933349609375,
      "step": 594
    },
    {
      "epoch": 0.8737151248164464,
      "fcm_dpo/beta": 0.002612186362966895,
      "fcm_dpo/delta": 0.022652659565210342,
      "fcm_dpo/margin": 221.48776245117188,
      "fcm_dpo/q_t": 0.37508952617645264,
      "grad_norm": 33.132080078125,
      "learning_rate": 2.451969280180849e-08,
      "logits/chosen": -0.4913725256919861,
      "logits/rejected": -0.4721420407295227,
      "logps/chosen": -338.8521728515625,
      "logps/ref_chosen": -49.4204216003418,
      "logps/ref_rejected": -80.62731170654297,
      "logps/rejected": -591.546875,
      "loss": 0.9833,
      "margin_dpo/margin_mean": 221.48776245117188,
      "margin_dpo/margin_std": 257.7921142578125,
      "step": 595
    },
    {
      "epoch": 0.8751835535976505,
      "fcm_dpo/beta": 0.002664618194103241,
      "fcm_dpo/delta": 0.08821612596511841,
      "fcm_dpo/margin": 193.6279296875,
      "fcm_dpo/q_t": 0.38701266050338745,
      "grad_norm": 41.59341049194336,
      "learning_rate": 2.396839494982103e-08,
      "logits/chosen": -0.5178302526473999,
      "logits/rejected": -0.48967409133911133,
      "logps/chosen": -358.09478759765625,
      "logps/ref_chosen": -59.791683197021484,
      "logps/ref_rejected": -80.09111785888672,
      "logps/rejected": -572.0221557617188,
      "loss": 1.0508,
      "margin_dpo/margin_mean": 193.6279296875,
      "margin_dpo/margin_std": 269.0942687988281,
      "step": 596
    },
    {
      "epoch": 0.8766519823788547,
      "fcm_dpo/beta": 0.0026114017236977816,
      "fcm_dpo/delta": -0.08313372731208801,
      "fcm_dpo/margin": 258.4861755371094,
      "fcm_dpo/q_t": 0.3541925251483917,
      "grad_norm": 33.791534423828125,
      "learning_rate": 2.3423053240837514e-08,
      "logits/chosen": -0.43104076385498047,
      "logits/rejected": -0.4429478645324707,
      "logps/chosen": -359.1304931640625,
      "logps/ref_chosen": -57.26078796386719,
      "logps/ref_rejected": -100.6937255859375,
      "logps/rejected": -661.049560546875,
      "loss": 0.9329,
      "margin_dpo/margin_mean": 258.4861755371094,
      "margin_dpo/margin_std": 268.7314758300781,
      "step": 597
    },
    {
      "epoch": 0.8781204111600588,
      "fcm_dpo/beta": 0.002603329950943589,
      "fcm_dpo/delta": 0.007749959826469421,
      "fcm_dpo/margin": 227.01988220214844,
      "fcm_dpo/q_t": 0.36725500226020813,
      "grad_norm": 40.18560791015625,
      "learning_rate": 2.2883682045119062e-08,
      "logits/chosen": -0.5023446083068848,
      "logits/rejected": -0.4874638319015503,
      "logps/chosen": -370.9491882324219,
      "logps/ref_chosen": -52.51850509643555,
      "logps/ref_rejected": -89.44385528564453,
      "logps/rejected": -634.8944091796875,
      "loss": 0.9904,
      "margin_dpo/margin_mean": 227.01986694335938,
      "margin_dpo/margin_std": 255.60830688476562,
      "step": 598
    },
    {
      "epoch": 0.8795888399412628,
      "fcm_dpo/beta": 0.002633871277794242,
      "fcm_dpo/delta": 0.049679867923259735,
      "fcm_dpo/margin": 209.84701538085938,
      "fcm_dpo/q_t": 0.3737853169441223,
      "grad_norm": 29.854141235351562,
      "learning_rate": 2.2350295575598367e-08,
      "logits/chosen": -0.4743267297744751,
      "logits/rejected": -0.46898123621940613,
      "logps/chosen": -339.2664794921875,
      "logps/ref_chosen": -49.802677154541016,
      "logps/ref_rejected": -82.978515625,
      "logps/rejected": -582.2893676757812,
      "loss": 0.9804,
      "margin_dpo/margin_mean": 209.8470458984375,
      "margin_dpo/margin_std": 206.9398193359375,
      "step": 599
    },
    {
      "epoch": 0.8810572687224669,
      "fcm_dpo/beta": 0.0026514409109950066,
      "fcm_dpo/delta": 0.004689887166023254,
      "fcm_dpo/margin": 224.59609985351562,
      "fcm_dpo/q_t": 0.3712194561958313,
      "grad_norm": 36.80059051513672,
      "learning_rate": 2.1822907887504932e-08,
      "logits/chosen": -0.5273237824440002,
      "logits/rejected": -0.5121384263038635,
      "logps/chosen": -400.91802978515625,
      "logps/ref_chosen": -66.43487548828125,
      "logps/ref_rejected": -85.45649719238281,
      "logps/rejected": -644.5357666015625,
      "loss": 0.998,
      "margin_dpo/margin_mean": 224.59609985351562,
      "margin_dpo/margin_std": 278.3258056640625,
      "step": 600
    },
    {
      "epoch": 0.8810572687224669,
      "eval_fcm_dpo/beta": 0.002657976234331727,
      "eval_logits/chosen": -0.5060350298881531,
      "eval_logits/rejected": -0.4877537786960602,
      "eval_logps/chosen": -436.8470153808594,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -597.329345703125,
      "eval_loss": 0.569061279296875,
      "eval_margin_dpo/margin_mean": 152.7354736328125,
      "eval_margin_dpo/margin_std": 265.09954833984375,
      "eval_runtime": 39.2811,
      "eval_samples_per_second": 59.545,
      "eval_steps_per_second": 1.884,
      "step": 600
    },
    {
      "epoch": 0.882525697503671,
      "fcm_dpo/beta": 0.002635272918269038,
      "fcm_dpo/delta": -0.03896784782409668,
      "fcm_dpo/margin": 241.43040466308594,
      "fcm_dpo/q_t": 0.3611557185649872,
      "grad_norm": 41.90599822998047,
      "learning_rate": 2.1301532877994742e-08,
      "logits/chosen": -0.47748884558677673,
      "logits/rejected": -0.46581321954727173,
      "logps/chosen": -405.7626953125,
      "logps/ref_chosen": -59.13361358642578,
      "logps/ref_rejected": -94.69093322753906,
      "logps/rejected": -682.7504272460938,
      "loss": 0.9629,
      "margin_dpo/margin_mean": 241.43038940429688,
      "margin_dpo/margin_std": 275.83514404296875,
      "step": 601
    },
    {
      "epoch": 0.8839941262848752,
      "fcm_dpo/beta": 0.002600977662950754,
      "fcm_dpo/delta": -0.12249189615249634,
      "fcm_dpo/margin": 274.4678955078125,
      "fcm_dpo/q_t": 0.34391018748283386,
      "grad_norm": 34.37300491333008,
      "learning_rate": 2.0786184285784298e-08,
      "logits/chosen": -0.4986855983734131,
      "logits/rejected": -0.4927312731742859,
      "logps/chosen": -303.8367919921875,
      "logps/ref_chosen": -48.59352111816406,
      "logps/ref_rejected": -87.6685562133789,
      "logps/rejected": -617.3796997070312,
      "loss": 0.8916,
      "margin_dpo/margin_mean": 274.4678955078125,
      "margin_dpo/margin_std": 258.61602783203125,
      "step": 602
    },
    {
      "epoch": 0.8854625550660793,
      "fcm_dpo/beta": 0.002573822159320116,
      "fcm_dpo/delta": -0.00591304013505578,
      "fcm_dpo/margin": 235.269287109375,
      "fcm_dpo/q_t": 0.3697740435600281,
      "grad_norm": 44.532958984375,
      "learning_rate": 2.0276875690788204e-08,
      "logits/chosen": -0.5375059843063354,
      "logits/rejected": -0.5155242681503296,
      "logps/chosen": -360.972900390625,
      "logps/ref_chosen": -70.41461944580078,
      "logps/ref_rejected": -100.32559967041016,
      "logps/rejected": -626.1531982421875,
      "loss": 0.9847,
      "margin_dpo/margin_mean": 235.269287109375,
      "margin_dpo/margin_std": 282.26904296875,
      "step": 603
    },
    {
      "epoch": 0.8869309838472834,
      "fcm_dpo/beta": 0.002521086484193802,
      "fcm_dpo/delta": -0.11240590363740921,
      "fcm_dpo/margin": 279.26800537109375,
      "fcm_dpo/q_t": 0.34469854831695557,
      "grad_norm": 37.802711486816406,
      "learning_rate": 1.977362051376158e-08,
      "logits/chosen": -0.4898557662963867,
      "logits/rejected": -0.4911998510360718,
      "logps/chosen": -313.27166748046875,
      "logps/ref_chosen": -46.45808029174805,
      "logps/ref_rejected": -91.8544921875,
      "logps/rejected": -637.9360961914062,
      "loss": 0.9107,
      "margin_dpo/margin_mean": 279.26800537109375,
      "margin_dpo/margin_std": 277.7120056152344,
      "step": 604
    },
    {
      "epoch": 0.8883994126284875,
      "fcm_dpo/beta": 0.0025093178264796734,
      "fcm_dpo/delta": 0.017898954451084137,
      "fcm_dpo/margin": 232.27915954589844,
      "fcm_dpo/q_t": 0.37571078538894653,
      "grad_norm": 30.63051414489746,
      "learning_rate": 1.9276432015946446e-08,
      "logits/chosen": -0.5367037057876587,
      "logits/rejected": -0.5347219109535217,
      "logps/chosen": -390.2449645996094,
      "logps/ref_chosen": -66.24933624267578,
      "logps/ref_rejected": -102.30496978759766,
      "logps/rejected": -658.5797119140625,
      "loss": 0.9934,
      "margin_dpo/margin_mean": 232.27914428710938,
      "margin_dpo/margin_std": 289.82879638671875,
      "step": 605
    },
    {
      "epoch": 0.8898678414096917,
      "fcm_dpo/beta": 0.002527826000005007,
      "fcm_dpo/delta": -0.03301798552274704,
      "fcm_dpo/margin": 249.30398559570312,
      "fcm_dpo/q_t": 0.36113643646240234,
      "grad_norm": 36.33147430419922,
      "learning_rate": 1.8785323298722093e-08,
      "logits/chosen": -0.5017037391662598,
      "logits/rejected": -0.4989688992500305,
      "logps/chosen": -350.18060302734375,
      "logps/ref_chosen": -54.819122314453125,
      "logps/ref_rejected": -98.37146759033203,
      "logps/rejected": -643.0369873046875,
      "loss": 0.9437,
      "margin_dpo/margin_mean": 249.30398559570312,
      "margin_dpo/margin_std": 251.91043090820312,
      "step": 606
    },
    {
      "epoch": 0.8913362701908958,
      "fcm_dpo/beta": 0.002542435657233,
      "fcm_dpo/delta": 0.07603956758975983,
      "fcm_dpo/margin": 207.49200439453125,
      "fcm_dpo/q_t": 0.382717490196228,
      "grad_norm": 28.992673873901367,
      "learning_rate": 1.8300307303259904e-08,
      "logits/chosen": -0.5028606057167053,
      "logits/rejected": -0.48178744316101074,
      "logps/chosen": -371.967041015625,
      "logps/ref_chosen": -58.08403778076172,
      "logps/ref_rejected": -79.777099609375,
      "logps/rejected": -601.1521606445312,
      "loss": 1.0122,
      "margin_dpo/margin_mean": 207.4920196533203,
      "margin_dpo/margin_std": 238.70074462890625,
      "step": 607
    },
    {
      "epoch": 0.8928046989720999,
      "fcm_dpo/beta": 0.0025535556487739086,
      "fcm_dpo/delta": 0.010348204523324966,
      "fcm_dpo/margin": 231.10964965820312,
      "fcm_dpo/q_t": 0.3670482039451599,
      "grad_norm": 27.258190155029297,
      "learning_rate": 1.7821396810182437e-08,
      "logits/chosen": -0.536124587059021,
      "logits/rejected": -0.5197678208351135,
      "logps/chosen": -358.027099609375,
      "logps/ref_chosen": -57.450836181640625,
      "logps/ref_rejected": -94.77339172363281,
      "logps/rejected": -626.4592895507812,
      "loss": 0.9573,
      "margin_dpo/margin_mean": 231.10964965820312,
      "margin_dpo/margin_std": 227.2630615234375,
      "step": 608
    },
    {
      "epoch": 0.8942731277533039,
      "fcm_dpo/beta": 0.0025187768042087555,
      "fcm_dpo/delta": -0.08650342375040054,
      "fcm_dpo/margin": 270.2357177734375,
      "fcm_dpo/q_t": 0.3506140112876892,
      "grad_norm": 41.33991241455078,
      "learning_rate": 1.7348604439226617e-08,
      "logits/chosen": -0.5273410677909851,
      "logits/rejected": -0.516228199005127,
      "logps/chosen": -351.49822998046875,
      "logps/ref_chosen": -58.805355072021484,
      "logps/ref_rejected": -88.81600952148438,
      "logps/rejected": -651.74462890625,
      "loss": 0.9151,
      "margin_dpo/margin_mean": 270.2357177734375,
      "margin_dpo/margin_std": 274.7491760253906,
      "step": 609
    },
    {
      "epoch": 0.895741556534508,
      "fcm_dpo/beta": 0.002538463566452265,
      "fcm_dpo/delta": 0.07885053008794785,
      "fcm_dpo/margin": 206.88258361816406,
      "fcm_dpo/q_t": 0.3827626705169678,
      "grad_norm": 40.440711975097656,
      "learning_rate": 1.6881942648911074e-08,
      "logits/chosen": -0.5288474559783936,
      "logits/rejected": -0.4975678324699402,
      "logps/chosen": -368.1615905761719,
      "logps/ref_chosen": -65.69503784179688,
      "logps/ref_rejected": -83.40538787841797,
      "logps/rejected": -592.7545166015625,
      "loss": 1.0173,
      "margin_dpo/margin_mean": 206.88258361816406,
      "margin_dpo/margin_std": 242.43508911132812,
      "step": 610
    },
    {
      "epoch": 0.8972099853157122,
      "fcm_dpo/beta": 0.002536176238209009,
      "fcm_dpo/delta": -0.05313975363969803,
      "fcm_dpo/margin": 256.1167907714844,
      "fcm_dpo/q_t": 0.35713207721710205,
      "grad_norm": 35.82498550415039,
      "learning_rate": 1.6421423736208e-08,
      "logits/chosen": -0.5013567209243774,
      "logits/rejected": -0.5022546052932739,
      "logps/chosen": -356.3680725097656,
      "logps/ref_chosen": -52.59946823120117,
      "logps/ref_rejected": -86.33099365234375,
      "logps/rejected": -646.2163696289062,
      "loss": 0.9377,
      "margin_dpo/margin_mean": 256.1167907714844,
      "margin_dpo/margin_std": 261.8753662109375,
      "step": 611
    },
    {
      "epoch": 0.8986784140969163,
      "fcm_dpo/beta": 0.0024938895367085934,
      "fcm_dpo/delta": -0.07635810971260071,
      "fcm_dpo/margin": 269.17584228515625,
      "fcm_dpo/q_t": 0.3497973084449768,
      "grad_norm": 25.489120483398438,
      "learning_rate": 1.5967059836219042e-08,
      "logits/chosen": -0.49843016266822815,
      "logits/rejected": -0.48337119817733765,
      "logps/chosen": -366.25396728515625,
      "logps/ref_chosen": -59.32372283935547,
      "logps/ref_rejected": -88.31239318847656,
      "logps/rejected": -664.41845703125,
      "loss": 0.9038,
      "margin_dpo/margin_mean": 269.17584228515625,
      "margin_dpo/margin_std": 241.643310546875,
      "step": 612
    },
    {
      "epoch": 0.9001468428781204,
      "fcm_dpo/beta": 0.0024755061604082584,
      "fcm_dpo/delta": -0.025314215570688248,
      "fcm_dpo/margin": 251.95530700683594,
      "fcm_dpo/q_t": 0.3586321473121643,
      "grad_norm": 35.185272216796875,
      "learning_rate": 1.551886292185553e-08,
      "logits/chosen": -0.5439824461936951,
      "logits/rejected": -0.5609041452407837,
      "logps/chosen": -351.8963317871094,
      "logps/ref_chosen": -59.72996520996094,
      "logps/ref_rejected": -105.10752868652344,
      "logps/rejected": -649.229248046875,
      "loss": 0.9211,
      "margin_dpo/margin_mean": 251.95529174804688,
      "margin_dpo/margin_std": 217.81700134277344,
      "step": 613
    },
    {
      "epoch": 0.9016152716593245,
      "fcm_dpo/beta": 0.002444417215883732,
      "fcm_dpo/delta": -0.04796981438994408,
      "fcm_dpo/margin": 263.7240905761719,
      "fcm_dpo/q_t": 0.35601553320884705,
      "grad_norm": 34.938133239746094,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": -0.4674204885959625,
      "logits/rejected": -0.49593475461006165,
      "logps/chosen": -351.73431396484375,
      "logps/ref_chosen": -52.93898010253906,
      "logps/ref_rejected": -104.67938232421875,
      "logps/rejected": -667.1987915039062,
      "loss": 0.932,
      "margin_dpo/margin_mean": 263.7240905761719,
      "margin_dpo/margin_std": 258.8504638671875,
      "step": 614
    },
    {
      "epoch": 0.9030837004405287,
      "fcm_dpo/beta": 0.0024889023043215275,
      "fcm_dpo/delta": 0.09434535354375839,
      "fcm_dpo/margin": 204.8232421875,
      "fcm_dpo/q_t": 0.38655292987823486,
      "grad_norm": 31.439695358276367,
      "learning_rate": 1.4641017128809801e-08,
      "logits/chosen": -0.5380944609642029,
      "logits/rejected": -0.5339647531509399,
      "logps/chosen": -367.2992248535156,
      "logps/ref_chosen": -65.81727600097656,
      "logps/ref_rejected": -95.17749786376953,
      "logps/rejected": -601.482666015625,
      "loss": 1.0194,
      "margin_dpo/margin_mean": 204.8232421875,
      "margin_dpo/margin_std": 236.53875732421875,
      "step": 615
    },
    {
      "epoch": 0.9045521292217328,
      "fcm_dpo/beta": 0.002577736508101225,
      "fcm_dpo/delta": 0.2006855607032776,
      "fcm_dpo/margin": 157.602294921875,
      "fcm_dpo/q_t": 0.4117230772972107,
      "grad_norm": 39.64345932006836,
      "learning_rate": 1.4211391382180637e-08,
      "logits/chosen": -0.5193255543708801,
      "logits/rejected": -0.4902590811252594,
      "logps/chosen": -436.3590393066406,
      "logps/ref_chosen": -65.13285827636719,
      "logps/ref_rejected": -74.70050048828125,
      "logps/rejected": -603.5289916992188,
      "loss": 1.1199,
      "margin_dpo/margin_mean": 157.602294921875,
      "margin_dpo/margin_std": 252.67433166503906,
      "step": 616
    },
    {
      "epoch": 0.9060205580029369,
      "fcm_dpo/beta": 0.0026704384945333004,
      "fcm_dpo/delta": 0.1679898351430893,
      "fcm_dpo/margin": 164.12026977539062,
      "fcm_dpo/q_t": 0.4053176939487457,
      "grad_norm": 46.60618209838867,
      "learning_rate": 1.378797888467345e-08,
      "logits/chosen": -0.5051152110099792,
      "logits/rejected": -0.47557491064071655,
      "logps/chosen": -400.8250732421875,
      "logps/ref_chosen": -63.005550384521484,
      "logps/ref_rejected": -64.234130859375,
      "logps/rejected": -566.1739501953125,
      "loss": 1.0936,
      "margin_dpo/margin_mean": 164.12026977539062,
      "margin_dpo/margin_std": 247.26060485839844,
      "step": 617
    },
    {
      "epoch": 0.9074889867841409,
      "fcm_dpo/beta": 0.002672924194484949,
      "fcm_dpo/delta": -0.059617504477500916,
      "fcm_dpo/margin": 245.20562744140625,
      "fcm_dpo/q_t": 0.3566618263721466,
      "grad_norm": 40.86361312866211,
      "learning_rate": 1.3370790793601371e-08,
      "logits/chosen": -0.4829975962638855,
      "logits/rejected": -0.4715951681137085,
      "logps/chosen": -403.0682373046875,
      "logps/ref_chosen": -67.10134887695312,
      "logps/ref_rejected": -92.15340423583984,
      "logps/rejected": -673.325927734375,
      "loss": 0.9513,
      "margin_dpo/margin_mean": 245.2056427001953,
      "margin_dpo/margin_std": 266.8808898925781,
      "step": 618
    },
    {
      "epoch": 0.908957415565345,
      "fcm_dpo/beta": 0.0026871771551668644,
      "fcm_dpo/delta": 0.07144356518983841,
      "fcm_dpo/margin": 197.93942260742188,
      "fcm_dpo/q_t": 0.38919079303741455,
      "grad_norm": 40.54890441894531,
      "learning_rate": 1.2959838102258535e-08,
      "logits/chosen": -0.5091539025306702,
      "logits/rejected": -0.5012193918228149,
      "logps/chosen": -386.0754699707031,
      "logps/ref_chosen": -55.978233337402344,
      "logps/ref_rejected": -93.1854019165039,
      "logps/rejected": -621.2220458984375,
      "loss": 1.0637,
      "margin_dpo/margin_mean": 197.93943786621094,
      "margin_dpo/margin_std": 297.71466064453125,
      "step": 619
    },
    {
      "epoch": 0.9104258443465492,
      "fcm_dpo/beta": 0.0026975993532687426,
      "fcm_dpo/delta": 0.0638522133231163,
      "fcm_dpo/margin": 199.90643310546875,
      "fcm_dpo/q_t": 0.3846666216850281,
      "grad_norm": 46.63349914550781,
      "learning_rate": 1.2555131639630567e-08,
      "logits/chosen": -0.5148506164550781,
      "logits/rejected": -0.4951644539833069,
      "logps/chosen": -366.35540771484375,
      "logps/ref_chosen": -59.79750061035156,
      "logps/ref_rejected": -78.41075134277344,
      "logps/rejected": -584.8751220703125,
      "loss": 1.0283,
      "margin_dpo/margin_mean": 199.90643310546875,
      "margin_dpo/margin_std": 263.07427978515625,
      "step": 620
    },
    {
      "epoch": 0.9118942731277533,
      "fcm_dpo/beta": 0.0026699635200202465,
      "fcm_dpo/delta": -0.17680245637893677,
      "fcm_dpo/margin": 285.97552490234375,
      "fcm_dpo/q_t": 0.33413994312286377,
      "grad_norm": 35.8352165222168,
      "learning_rate": 1.2156682070109086e-08,
      "logits/chosen": -0.5005359649658203,
      "logits/rejected": -0.5020206570625305,
      "logps/chosen": -341.9903564453125,
      "logps/ref_chosen": -53.93375778198242,
      "logps/ref_rejected": -88.36951446533203,
      "logps/rejected": -662.401611328125,
      "loss": 0.8752,
      "margin_dpo/margin_mean": 285.97552490234375,
      "margin_dpo/margin_std": 268.96636962890625,
      "step": 621
    },
    {
      "epoch": 0.9133627019089574,
      "fcm_dpo/beta": 0.0026113735511898994,
      "fcm_dpo/delta": 0.0012344121932983398,
      "fcm_dpo/margin": 229.01025390625,
      "fcm_dpo/q_t": 0.3693368434906006,
      "grad_norm": 33.05309295654297,
      "learning_rate": 1.1764499893210878e-08,
      "logits/chosen": -0.48245564103126526,
      "logits/rejected": -0.4655565917491913,
      "logps/chosen": -363.42919921875,
      "logps/ref_chosen": -60.28582000732422,
      "logps/ref_rejected": -85.51873779296875,
      "logps/rejected": -617.67236328125,
      "loss": 0.9679,
      "margin_dpo/margin_mean": 229.01025390625,
      "margin_dpo/margin_std": 250.83233642578125,
      "step": 622
    },
    {
      "epoch": 0.9148311306901615,
      "fcm_dpo/beta": 0.002670406363904476,
      "fcm_dpo/delta": 0.12260958552360535,
      "fcm_dpo/margin": 181.0537109375,
      "fcm_dpo/q_t": 0.39631474018096924,
      "grad_norm": 40.9214973449707,
      "learning_rate": 1.1378595443300998e-08,
      "logits/chosen": -0.48193132877349854,
      "logits/rejected": -0.4616912603378296,
      "logps/chosen": -400.8602294921875,
      "logps/ref_chosen": -64.1569595336914,
      "logps/ref_rejected": -85.08304595947266,
      "logps/rejected": -602.840087890625,
      "loss": 1.0693,
      "margin_dpo/margin_mean": 181.05372619628906,
      "margin_dpo/margin_std": 264.8545227050781,
      "step": 623
    },
    {
      "epoch": 0.9162995594713657,
      "fcm_dpo/beta": 0.0026587401516735554,
      "fcm_dpo/delta": -0.08800569176673889,
      "fcm_dpo/margin": 256.51275634765625,
      "fcm_dpo/q_t": 0.34991389513015747,
      "grad_norm": 39.98982620239258,
      "learning_rate": 1.0998978889320582e-08,
      "logits/chosen": -0.5772846937179565,
      "logits/rejected": -0.5503325462341309,
      "logps/chosen": -415.0184020996094,
      "logps/ref_chosen": -71.91862487792969,
      "logps/ref_rejected": -97.13203430175781,
      "logps/rejected": -696.7445678710938,
      "loss": 0.9264,
      "margin_dpo/margin_mean": 256.51275634765625,
      "margin_dpo/margin_std": 264.0021057128906,
      "step": 624
    },
    {
      "epoch": 0.9177679882525698,
      "fcm_dpo/beta": 0.0026332507841289043,
      "fcm_dpo/delta": -0.023220915347337723,
      "fcm_dpo/margin": 236.09141540527344,
      "fcm_dpo/q_t": 0.36515945196151733,
      "grad_norm": 42.99685287475586,
      "learning_rate": 1.0625660234518913e-08,
      "logits/chosen": -0.4856789708137512,
      "logits/rejected": -0.468561589717865,
      "logps/chosen": -366.0799560546875,
      "logps/ref_chosen": -58.342071533203125,
      "logps/ref_rejected": -86.09038543701172,
      "logps/rejected": -629.919677734375,
      "loss": 0.9517,
      "margin_dpo/margin_mean": 236.09140014648438,
      "margin_dpo/margin_std": 255.3983154296875,
      "step": 625
    },
    {
      "epoch": 0.9192364170337739,
      "fcm_dpo/beta": 0.0026790527626872063,
      "fcm_dpo/delta": 0.09501801431179047,
      "fcm_dpo/margin": 190.15899658203125,
      "fcm_dpo/q_t": 0.3896622657775879,
      "grad_norm": 35.053749084472656,
      "learning_rate": 1.0258649316189721e-08,
      "logits/chosen": -0.516830325126648,
      "logits/rejected": -0.4992465078830719,
      "logps/chosen": -417.03143310546875,
      "logps/ref_chosen": -75.11260986328125,
      "logps/ref_rejected": -99.188720703125,
      "logps/rejected": -631.2666015625,
      "loss": 1.0376,
      "margin_dpo/margin_mean": 190.15899658203125,
      "margin_dpo/margin_std": 242.58265686035156,
      "step": 626
    },
    {
      "epoch": 0.920704845814978,
      "fcm_dpo/beta": 0.0026376736350357533,
      "fcm_dpo/delta": -0.13061486184597015,
      "fcm_dpo/margin": 273.473388671875,
      "fcm_dpo/q_t": 0.35445448756217957,
      "grad_norm": 37.661746978759766,
      "learning_rate": 9.897955805412e-09,
      "logits/chosen": -0.4934673309326172,
      "logits/rejected": -0.5177198648452759,
      "logps/chosen": -314.9818115234375,
      "logps/ref_chosen": -47.74314880371094,
      "logps/ref_rejected": -106.75448608398438,
      "logps/rejected": -647.466552734375,
      "loss": 0.9501,
      "margin_dpo/margin_mean": 273.4734191894531,
      "margin_dpo/margin_std": 342.03204345703125,
      "step": 627
    },
    {
      "epoch": 0.922173274596182,
      "fcm_dpo/beta": 0.0025675692595541477,
      "fcm_dpo/delta": -0.09277446568012238,
      "fcm_dpo/margin": 267.0002746582031,
      "fcm_dpo/q_t": 0.34974387288093567,
      "grad_norm": 32.36410140991211,
      "learning_rate": 9.543589206795238e-09,
      "logits/chosen": -0.5133162140846252,
      "logits/rejected": -0.5017916560173035,
      "logps/chosen": -382.24267578125,
      "logps/ref_chosen": -60.182945251464844,
      "logps/ref_rejected": -101.55467224121094,
      "logps/rejected": -690.6146240234375,
      "loss": 0.9174,
      "margin_dpo/margin_mean": 267.0002746582031,
      "margin_dpo/margin_std": 266.8199768066406,
      "step": 628
    },
    {
      "epoch": 0.9236417033773862,
      "fcm_dpo/beta": 0.002568417927250266,
      "fcm_dpo/delta": 0.013333270326256752,
      "fcm_dpo/margin": 228.7070770263672,
      "fcm_dpo/q_t": 0.3673741817474365,
      "grad_norm": 39.50351333618164,
      "learning_rate": 9.19555885822887e-09,
      "logits/chosen": -0.4918622374534607,
      "logits/rejected": -0.47989362478256226,
      "logps/chosen": -393.0993957519531,
      "logps/ref_chosen": -64.21354675292969,
      "logps/ref_rejected": -91.65367126464844,
      "logps/rejected": -649.24658203125,
      "loss": 0.9515,
      "margin_dpo/margin_mean": 228.70706176757812,
      "margin_dpo/margin_std": 213.66384887695312,
      "step": 629
    },
    {
      "epoch": 0.9251101321585903,
      "fcm_dpo/beta": 0.0026454501785337925,
      "fcm_dpo/delta": 0.16232366859912872,
      "fcm_dpo/margin": 167.9291534423828,
      "fcm_dpo/q_t": 0.4042319357395172,
      "grad_norm": 37.71936798095703,
      "learning_rate": 8.85387393063622e-09,
      "logits/chosen": -0.5329476594924927,
      "logits/rejected": -0.5061321258544922,
      "logps/chosen": -354.07684326171875,
      "logps/ref_chosen": -59.29100036621094,
      "logps/ref_rejected": -83.59829711914062,
      "logps/rejected": -546.3132934570312,
      "loss": 1.0856,
      "margin_dpo/margin_mean": 167.9291534423828,
      "margin_dpo/margin_std": 248.06668090820312,
      "step": 630
    },
    {
      "epoch": 0.9265785609397944,
      "fcm_dpo/beta": 0.002692385343834758,
      "fcm_dpo/delta": 0.05998341366648674,
      "fcm_dpo/margin": 201.62759399414062,
      "fcm_dpo/q_t": 0.3826253414154053,
      "grad_norm": 39.2678337097168,
      "learning_rate": 8.518543427732949e-09,
      "logits/chosen": -0.5022942423820496,
      "logits/rejected": -0.48305743932724,
      "logps/chosen": -390.76043701171875,
      "logps/ref_chosen": -59.45360565185547,
      "logps/ref_rejected": -80.95156860351562,
      "logps/rejected": -613.885986328125,
      "loss": 1.0448,
      "margin_dpo/margin_mean": 201.62759399414062,
      "margin_dpo/margin_std": 280.7120361328125,
      "step": 631
    },
    {
      "epoch": 0.9280469897209985,
      "fcm_dpo/beta": 0.0027071628719568253,
      "fcm_dpo/delta": 0.05815380439162254,
      "fcm_dpo/margin": 201.3222198486328,
      "fcm_dpo/q_t": 0.3803539276123047,
      "grad_norm": 38.19340133666992,
      "learning_rate": 8.189576185789637e-09,
      "logits/chosen": -0.4966999888420105,
      "logits/rejected": -0.47419142723083496,
      "logps/chosen": -389.7144775390625,
      "logps/ref_chosen": -61.35155487060547,
      "logps/ref_rejected": -86.16017150878906,
      "logps/rejected": -615.8453369140625,
      "loss": 1.0424,
      "margin_dpo/margin_mean": 201.3222198486328,
      "margin_dpo/margin_std": 276.93914794921875,
      "step": 632
    },
    {
      "epoch": 0.9295154185022027,
      "fcm_dpo/beta": 0.0027928436174988747,
      "fcm_dpo/delta": 0.12002822011709213,
      "fcm_dpo/margin": 173.24588012695312,
      "fcm_dpo/q_t": 0.39484164118766785,
      "grad_norm": 57.099609375,
      "learning_rate": 7.866980873399015e-09,
      "logits/chosen": -0.5218715667724609,
      "logits/rejected": -0.5239961743354797,
      "logps/chosen": -393.3670349121094,
      "logps/ref_chosen": -57.27816390991211,
      "logps/ref_rejected": -91.58395385742188,
      "logps/rejected": -600.918701171875,
      "loss": 1.0778,
      "margin_dpo/margin_mean": 173.24588012695312,
      "margin_dpo/margin_std": 253.44149780273438,
      "step": 633
    },
    {
      "epoch": 0.9309838472834068,
      "fcm_dpo/beta": 0.002813713625073433,
      "fcm_dpo/delta": 0.08731327205896378,
      "fcm_dpo/margin": 183.8138885498047,
      "fcm_dpo/q_t": 0.3883476257324219,
      "grad_norm": 39.00724411010742,
      "learning_rate": 7.550765991247654e-09,
      "logits/chosen": -0.5172115564346313,
      "logits/rejected": -0.5073580741882324,
      "logps/chosen": -427.5379333496094,
      "logps/ref_chosen": -66.61896514892578,
      "logps/ref_rejected": -107.12564849853516,
      "logps/rejected": -651.8585205078125,
      "loss": 1.0418,
      "margin_dpo/margin_mean": 183.8138885498047,
      "margin_dpo/margin_std": 243.1580047607422,
      "step": 634
    },
    {
      "epoch": 0.9324522760646109,
      "fcm_dpo/beta": 0.0028590408619493246,
      "fcm_dpo/delta": 0.05354948714375496,
      "fcm_dpo/margin": 192.16888427734375,
      "fcm_dpo/q_t": 0.38193368911743164,
      "grad_norm": 41.086822509765625,
      "learning_rate": 7.240939871891699e-09,
      "logits/chosen": -0.5438894629478455,
      "logits/rejected": -0.5025356411933899,
      "logps/chosen": -382.0107421875,
      "logps/ref_chosen": -73.95551300048828,
      "logps/ref_rejected": -82.50045776367188,
      "logps/rejected": -582.724609375,
      "loss": 1.0099,
      "margin_dpo/margin_mean": 192.16888427734375,
      "margin_dpo/margin_std": 238.52110290527344,
      "step": 635
    },
    {
      "epoch": 0.933920704845815,
      "fcm_dpo/beta": 0.0028688067104667425,
      "fcm_dpo/delta": -0.04967202991247177,
      "fcm_dpo/margin": 225.1034698486328,
      "fcm_dpo/q_t": 0.35912269353866577,
      "grad_norm": 42.04436492919922,
      "learning_rate": 6.937510679537628e-09,
      "logits/chosen": -0.4912013113498688,
      "logits/rejected": -0.4742831587791443,
      "logps/chosen": -358.6704406738281,
      "logps/ref_chosen": -59.628910064697266,
      "logps/ref_rejected": -81.97883605957031,
      "logps/rejected": -606.1238403320312,
      "loss": 0.9427,
      "margin_dpo/margin_mean": 225.10345458984375,
      "margin_dpo/margin_std": 237.44015502929688,
      "step": 636
    },
    {
      "epoch": 0.9353891336270191,
      "fcm_dpo/beta": 0.002806810662150383,
      "fcm_dpo/delta": -0.06823452562093735,
      "fcm_dpo/margin": 236.39791870117188,
      "fcm_dpo/q_t": 0.3558032512664795,
      "grad_norm": 40.245174407958984,
      "learning_rate": 6.640486409826785e-09,
      "logits/chosen": -0.4979478716850281,
      "logits/rejected": -0.496917724609375,
      "logps/chosen": -365.4517822265625,
      "logps/ref_chosen": -49.652687072753906,
      "logps/ref_rejected": -98.40513610839844,
      "logps/rejected": -650.6021728515625,
      "loss": 0.9336,
      "margin_dpo/margin_mean": 236.39791870117188,
      "margin_dpo/margin_std": 248.04531860351562,
      "step": 637
    },
    {
      "epoch": 0.9368575624082232,
      "fcm_dpo/beta": 0.0028112411964684725,
      "fcm_dpo/delta": -0.03967977315187454,
      "fcm_dpo/margin": 226.1969757080078,
      "fcm_dpo/q_t": 0.357632577419281,
      "grad_norm": 33.084903717041016,
      "learning_rate": 6.349874889624962e-09,
      "logits/chosen": -0.4676443338394165,
      "logits/rejected": -0.4407140016555786,
      "logps/chosen": -347.86865234375,
      "logps/ref_chosen": -58.156639099121094,
      "logps/ref_rejected": -79.3014907836914,
      "logps/rejected": -595.2105102539062,
      "loss": 0.9258,
      "margin_dpo/margin_mean": 226.1969757080078,
      "margin_dpo/margin_std": 198.74960327148438,
      "step": 638
    },
    {
      "epoch": 0.9383259911894273,
      "fcm_dpo/beta": 0.0028432803228497505,
      "fcm_dpo/delta": 0.1298457235097885,
      "fcm_dpo/margin": 167.3184356689453,
      "fcm_dpo/q_t": 0.3954669237136841,
      "grad_norm": 51.73390197753906,
      "learning_rate": 6.065683776815933e-09,
      "logits/chosen": -0.4740576148033142,
      "logits/rejected": -0.434129536151886,
      "logps/chosen": -424.7391357421875,
      "logps/ref_chosen": -72.32319641113281,
      "logps/ref_rejected": -74.2749252319336,
      "logps/rejected": -594.00927734375,
      "loss": 1.0733,
      "margin_dpo/margin_mean": 167.3184356689453,
      "margin_dpo/margin_std": 239.7473602294922,
      "step": 639
    },
    {
      "epoch": 0.9397944199706314,
      "fcm_dpo/beta": 0.002794375643134117,
      "fcm_dpo/delta": -0.18512314558029175,
      "fcm_dpo/margin": 275.9643249511719,
      "fcm_dpo/q_t": 0.3381561040878296,
      "grad_norm": 32.32111358642578,
      "learning_rate": 5.7879205600998296e-09,
      "logits/chosen": -0.4964972138404846,
      "logits/rejected": -0.4891301393508911,
      "logps/chosen": -369.9661560058594,
      "logps/ref_chosen": -56.13436508178711,
      "logps/ref_rejected": -108.60014343261719,
      "logps/rejected": -698.396240234375,
      "loss": 0.8868,
      "margin_dpo/margin_mean": 275.9643249511719,
      "margin_dpo/margin_std": 286.1241149902344,
      "step": 640
    },
    {
      "epoch": 0.9412628487518355,
      "fcm_dpo/beta": 0.002765617100521922,
      "fcm_dpo/delta": 0.06337472051382065,
      "fcm_dpo/margin": 195.2603759765625,
      "fcm_dpo/q_t": 0.38529226183891296,
      "grad_norm": 54.78212356567383,
      "learning_rate": 5.516592558795746e-09,
      "logits/chosen": -0.49673527479171753,
      "logits/rejected": -0.4844779968261719,
      "logps/chosen": -419.86749267578125,
      "logps/ref_chosen": -64.99689483642578,
      "logps/ref_rejected": -86.99232482910156,
      "logps/rejected": -637.123291015625,
      "loss": 1.0383,
      "margin_dpo/margin_mean": 195.26036071777344,
      "margin_dpo/margin_std": 274.37164306640625,
      "step": 641
    },
    {
      "epoch": 0.9427312775330396,
      "fcm_dpo/beta": 0.002786886878311634,
      "fcm_dpo/delta": -0.06086990237236023,
      "fcm_dpo/margin": 235.2342987060547,
      "fcm_dpo/q_t": 0.3679524064064026,
      "grad_norm": 43.42346954345703,
      "learning_rate": 5.251706922648868e-09,
      "logits/chosen": -0.5038265585899353,
      "logits/rejected": -0.5031158328056335,
      "logps/chosen": -400.65997314453125,
      "logps/ref_chosen": -65.68924713134766,
      "logps/ref_rejected": -110.24205017089844,
      "logps/rejected": -680.4470825195312,
      "loss": 0.9975,
      "margin_dpo/margin_mean": 235.23428344726562,
      "margin_dpo/margin_std": 318.88861083984375,
      "step": 642
    },
    {
      "epoch": 0.9441997063142438,
      "fcm_dpo/beta": 0.002727922983467579,
      "fcm_dpo/delta": 0.0013057217001914978,
      "fcm_dpo/margin": 218.98019409179688,
      "fcm_dpo/q_t": 0.36833906173706055,
      "grad_norm": 39.67738342285156,
      "learning_rate": 4.993270631642038e-09,
      "logits/chosen": -0.5181025266647339,
      "logits/rejected": -0.5136141180992126,
      "logps/chosen": -359.638427734375,
      "logps/ref_chosen": -51.94999694824219,
      "logps/ref_rejected": -87.46833801269531,
      "logps/rejected": -614.136962890625,
      "loss": 0.9763,
      "margin_dpo/margin_mean": 218.98019409179688,
      "margin_dpo/margin_std": 238.99154663085938,
      "step": 643
    },
    {
      "epoch": 0.9456681350954479,
      "fcm_dpo/beta": 0.0027766618877649307,
      "fcm_dpo/delta": 0.07305951416492462,
      "fcm_dpo/margin": 191.19378662109375,
      "fcm_dpo/q_t": 0.38764306902885437,
      "grad_norm": 49.11812210083008,
      "learning_rate": 4.741290495811873e-09,
      "logits/chosen": -0.511053204536438,
      "logits/rejected": -0.5009315013885498,
      "logps/chosen": -381.0382080078125,
      "logps/ref_chosen": -59.017662048339844,
      "logps/ref_rejected": -87.13668823242188,
      "logps/rejected": -600.3509521484375,
      "loss": 1.0642,
      "margin_dpo/margin_mean": 191.19378662109375,
      "margin_dpo/margin_std": 282.96331787109375,
      "step": 644
    },
    {
      "epoch": 0.947136563876652,
      "fcm_dpo/beta": 0.0028436644934117794,
      "fcm_dpo/delta": 0.16144075989723206,
      "fcm_dpo/margin": 156.83892822265625,
      "fcm_dpo/q_t": 0.40858685970306396,
      "grad_norm": 49.89131546020508,
      "learning_rate": 4.495773155069299e-09,
      "logits/chosen": -0.4825410842895508,
      "logits/rejected": -0.47511184215545654,
      "logps/chosen": -378.183349609375,
      "logps/ref_chosen": -55.87602233886719,
      "logps/ref_rejected": -97.78080749511719,
      "logps/rejected": -576.9270629882812,
      "loss": 1.1329,
      "margin_dpo/margin_mean": 156.83892822265625,
      "margin_dpo/margin_std": 280.7100524902344,
      "step": 645
    },
    {
      "epoch": 0.9486049926578561,
      "fcm_dpo/beta": 0.002883891575038433,
      "fcm_dpo/delta": 0.029555249959230423,
      "fcm_dpo/margin": 198.30703735351562,
      "fcm_dpo/q_t": 0.3727361261844635,
      "grad_norm": 39.009521484375,
      "learning_rate": 4.256725079024553e-09,
      "logits/chosen": -0.48562538623809814,
      "logits/rejected": -0.46007299423217773,
      "logps/chosen": -364.4145812988281,
      "logps/ref_chosen": -61.275787353515625,
      "logps/ref_rejected": -77.50580596923828,
      "logps/rejected": -578.95166015625,
      "loss": 0.9785,
      "margin_dpo/margin_mean": 198.30703735351562,
      "margin_dpo/margin_std": 211.89578247070312,
      "step": 646
    },
    {
      "epoch": 0.9500734214390602,
      "fcm_dpo/beta": 0.002909876639023423,
      "fcm_dpo/delta": 0.007972326129674911,
      "fcm_dpo/margin": 203.5854949951172,
      "fcm_dpo/q_t": 0.3704141676425934,
      "grad_norm": 34.949928283691406,
      "learning_rate": 4.024152566816791e-09,
      "logits/chosen": -0.4611695408821106,
      "logits/rejected": -0.4655107855796814,
      "logps/chosen": -345.0274353027344,
      "logps/ref_chosen": -54.8524169921875,
      "logps/ref_rejected": -93.5194091796875,
      "logps/rejected": -587.2799072265625,
      "loss": 0.9958,
      "margin_dpo/margin_mean": 203.5854949951172,
      "margin_dpo/margin_std": 246.224853515625,
      "step": 647
    },
    {
      "epoch": 0.9515418502202643,
      "fcm_dpo/beta": 0.002797810360789299,
      "fcm_dpo/delta": -0.25719988346099854,
      "fcm_dpo/margin": 298.71856689453125,
      "fcm_dpo/q_t": 0.3337755799293518,
      "grad_norm": 35.38324737548828,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": -0.4994831681251526,
      "logits/rejected": -0.5130796432495117,
      "logps/chosen": -361.3632507324219,
      "logps/ref_chosen": -54.17146682739258,
      "logps/ref_rejected": -98.7127914428711,
      "logps/rejected": -704.6231689453125,
      "loss": 0.8967,
      "margin_dpo/margin_mean": 298.71856689453125,
      "margin_dpo/margin_std": 349.7194519042969,
      "step": 648
    },
    {
      "epoch": 0.9530102790014684,
      "fcm_dpo/beta": 0.002828485332429409,
      "fcm_dpo/delta": 0.1316825896501541,
      "fcm_dpo/margin": 167.37564086914062,
      "fcm_dpo/q_t": 0.40139663219451904,
      "grad_norm": 42.975502014160156,
      "learning_rate": 3.5784585771215235e-09,
      "logits/chosen": -0.5309435725212097,
      "logits/rejected": -0.5060242414474487,
      "logps/chosen": -379.24853515625,
      "logps/ref_chosen": -62.480350494384766,
      "logps/ref_rejected": -80.07717895507812,
      "logps/rejected": -564.2210083007812,
      "loss": 1.09,
      "margin_dpo/margin_mean": 167.37564086914062,
      "margin_dpo/margin_std": 259.5540466308594,
      "step": 649
    },
    {
      "epoch": 0.9544787077826725,
      "fcm_dpo/beta": 0.002783937146887183,
      "fcm_dpo/delta": -0.10079372674226761,
      "fcm_dpo/margin": 249.080322265625,
      "fcm_dpo/q_t": 0.3533610701560974,
      "grad_norm": 34.299888610839844,
      "learning_rate": 3.3653488440851253e-09,
      "logits/chosen": -0.48052090406417847,
      "logits/rejected": -0.47680503129959106,
      "logps/chosen": -385.91802978515625,
      "logps/ref_chosen": -56.09281921386719,
      "logps/ref_rejected": -98.26483917236328,
      "logps/rejected": -677.1703491210938,
      "loss": 0.9576,
      "margin_dpo/margin_mean": 249.08029174804688,
      "margin_dpo/margin_std": 299.54718017578125,
      "step": 650
    },
    {
      "epoch": 0.9559471365638766,
      "fcm_dpo/beta": 0.0027128136716783047,
      "fcm_dpo/delta": -0.12499568611383438,
      "fcm_dpo/margin": 263.5880126953125,
      "fcm_dpo/q_t": 0.34237995743751526,
      "grad_norm": 38.067604064941406,
      "learning_rate": 3.158738163478475e-09,
      "logits/chosen": -0.5105220079421997,
      "logits/rejected": -0.5287940502166748,
      "logps/chosen": -301.7446594238281,
      "logps/ref_chosen": -43.42544937133789,
      "logps/ref_rejected": -99.95791625976562,
      "logps/rejected": -621.8651123046875,
      "loss": 0.8879,
      "margin_dpo/margin_mean": 263.5880126953125,
      "margin_dpo/margin_std": 240.21612548828125,
      "step": 651
    },
    {
      "epoch": 0.9574155653450808,
      "fcm_dpo/beta": 0.0027055356185883284,
      "fcm_dpo/delta": -0.019109947606921196,
      "fcm_dpo/margin": 228.38198852539062,
      "fcm_dpo/q_t": 0.36907005310058594,
      "grad_norm": 36.77167510986328,
      "learning_rate": 2.9586319796851555e-09,
      "logits/chosen": -0.5113554000854492,
      "logits/rejected": -0.504870593547821,
      "logps/chosen": -353.9315185546875,
      "logps/ref_chosen": -62.57680892944336,
      "logps/ref_rejected": -111.76779174804688,
      "logps/rejected": -631.5044555664062,
      "loss": 0.9818,
      "margin_dpo/margin_mean": 228.38198852539062,
      "margin_dpo/margin_std": 280.3888244628906,
      "step": 652
    },
    {
      "epoch": 0.9588839941262849,
      "fcm_dpo/beta": 0.0026833987794816494,
      "fcm_dpo/delta": -0.041953183710575104,
      "fcm_dpo/margin": 238.23910522460938,
      "fcm_dpo/q_t": 0.3634333610534668,
      "grad_norm": 42.57564163208008,
      "learning_rate": 2.7650355656892166e-09,
      "logits/chosen": -0.5182297825813293,
      "logits/rejected": -0.517985999584198,
      "logps/chosen": -397.2801818847656,
      "logps/ref_chosen": -61.11295700073242,
      "logps/ref_rejected": -103.24960327148438,
      "logps/rejected": -677.6558837890625,
      "loss": 0.9644,
      "margin_dpo/margin_mean": 238.23910522460938,
      "margin_dpo/margin_std": 277.93377685546875,
      "step": 653
    },
    {
      "epoch": 0.960352422907489,
      "fcm_dpo/beta": 0.002681623911485076,
      "fcm_dpo/delta": 0.03697919845581055,
      "fcm_dpo/margin": 210.70355224609375,
      "fcm_dpo/q_t": 0.3804095983505249,
      "grad_norm": 35.654144287109375,
      "learning_rate": 2.577954022936174e-09,
      "logits/chosen": -0.49398699402809143,
      "logits/rejected": -0.4834080934524536,
      "logps/chosen": -394.8516845703125,
      "logps/ref_chosen": -61.7281379699707,
      "logps/ref_rejected": -98.7738037109375,
      "logps/rejected": -642.600830078125,
      "loss": 1.0183,
      "margin_dpo/margin_mean": 210.70355224609375,
      "margin_dpo/margin_std": 280.0821533203125,
      "step": 654
    },
    {
      "epoch": 0.9618208516886931,
      "fcm_dpo/beta": 0.002694209571927786,
      "fcm_dpo/delta": 0.017087338492274284,
      "fcm_dpo/margin": 216.68463134765625,
      "fcm_dpo/q_t": 0.3784465789794922,
      "grad_norm": 33.220149993896484,
      "learning_rate": 2.397392281198729e-09,
      "logits/chosen": -0.45762312412261963,
      "logits/rejected": -0.4692486524581909,
      "logps/chosen": -356.33929443359375,
      "logps/ref_chosen": -49.576812744140625,
      "logps/ref_rejected": -98.29183197021484,
      "logps/rejected": -621.7389526367188,
      "loss": 1.0201,
      "margin_dpo/margin_mean": 216.6846466064453,
      "margin_dpo/margin_std": 295.111083984375,
      "step": 655
    },
    {
      "epoch": 0.9632892804698973,
      "fcm_dpo/beta": 0.002611855510622263,
      "fcm_dpo/delta": -0.25648045539855957,
      "fcm_dpo/margin": 319.9178771972656,
      "fcm_dpo/q_t": 0.3181704878807068,
      "grad_norm": 54.623451232910156,
      "learning_rate": 2.223355098446622e-09,
      "logits/chosen": -0.4365518093109131,
      "logits/rejected": -0.44737708568573,
      "logps/chosen": -353.51763916015625,
      "logps/ref_chosen": -52.54943084716797,
      "logps/ref_rejected": -113.67464447021484,
      "logps/rejected": -734.560791015625,
      "loss": 0.8148,
      "margin_dpo/margin_mean": 319.91790771484375,
      "margin_dpo/margin_std": 263.46356201171875,
      "step": 656
    },
    {
      "epoch": 0.9647577092511013,
      "fcm_dpo/beta": 0.002502152230590582,
      "fcm_dpo/delta": -0.15946225821971893,
      "fcm_dpo/margin": 298.534423828125,
      "fcm_dpo/q_t": 0.33715760707855225,
      "grad_norm": 33.79457473754883,
      "learning_rate": 2.055847060721566e-09,
      "logits/chosen": -0.5489492416381836,
      "logits/rejected": -0.5547805428504944,
      "logps/chosen": -332.32037353515625,
      "logps/ref_chosen": -46.700538635253906,
      "logps/ref_rejected": -97.91487121582031,
      "logps/rejected": -682.0692138671875,
      "loss": 0.8675,
      "margin_dpo/margin_mean": 298.534423828125,
      "margin_dpo/margin_std": 273.04095458984375,
      "step": 657
    },
    {
      "epoch": 0.9662261380323054,
      "fcm_dpo/beta": 0.0024999987799674273,
      "fcm_dpo/delta": 0.03055243194103241,
      "fcm_dpo/margin": 228.47642517089844,
      "fcm_dpo/q_t": 0.37107935547828674,
      "grad_norm": 32.88031768798828,
      "learning_rate": 1.8948725820160662e-09,
      "logits/chosen": -0.4807060956954956,
      "logits/rejected": -0.4754422903060913,
      "logps/chosen": -388.13995361328125,
      "logps/ref_chosen": -60.95820999145508,
      "logps/ref_rejected": -95.93949127197266,
      "logps/rejected": -651.59765625,
      "loss": 0.9682,
      "margin_dpo/margin_mean": 228.4764404296875,
      "margin_dpo/margin_std": 229.4976806640625,
      "step": 658
    },
    {
      "epoch": 0.9676945668135095,
      "fcm_dpo/beta": 0.002503772731870413,
      "fcm_dpo/delta": 0.029256850481033325,
      "fcm_dpo/margin": 228.52227783203125,
      "fcm_dpo/q_t": 0.37436485290527344,
      "grad_norm": 36.82291030883789,
      "learning_rate": 1.7404359041573723e-09,
      "logits/chosen": -0.509990394115448,
      "logits/rejected": -0.4739938974380493,
      "logps/chosen": -371.61309814453125,
      "logps/ref_chosen": -76.74298095703125,
      "logps/ref_rejected": -87.4709701538086,
      "logps/rejected": -610.8634033203125,
      "loss": 0.9845,
      "margin_dpo/margin_mean": 228.52227783203125,
      "margin_dpo/margin_std": 254.27716064453125,
      "step": 659
    },
    {
      "epoch": 0.9691629955947136,
      "fcm_dpo/beta": 0.002496623434126377,
      "fcm_dpo/delta": -0.0633494034409523,
      "fcm_dpo/margin": 264.0384521484375,
      "fcm_dpo/q_t": 0.35319751501083374,
      "grad_norm": 33.35771942138672,
      "learning_rate": 1.592541096695571e-09,
      "logits/chosen": -0.49027347564697266,
      "logits/rejected": -0.4790500998497009,
      "logps/chosen": -362.43536376953125,
      "logps/ref_chosen": -59.04788589477539,
      "logps/ref_rejected": -75.96005249023438,
      "logps/rejected": -643.385986328125,
      "loss": 0.9142,
      "margin_dpo/margin_mean": 264.0384521484375,
      "margin_dpo/margin_std": 247.07907104492188,
      "step": 660
    },
    {
      "epoch": 0.9706314243759178,
      "fcm_dpo/beta": 0.0024961913004517555,
      "fcm_dpo/delta": -0.03317072242498398,
      "fcm_dpo/margin": 252.34422302246094,
      "fcm_dpo/q_t": 0.3640047013759613,
      "grad_norm": 30.544677734375,
      "learning_rate": 1.4511920567963908e-09,
      "logits/chosen": -0.5381925702095032,
      "logits/rejected": -0.519443154335022,
      "logps/chosen": -326.90234375,
      "logps/ref_chosen": -50.673973083496094,
      "logps/ref_rejected": -86.00569152832031,
      "logps/rejected": -614.5782470703125,
      "loss": 0.9623,
      "margin_dpo/margin_mean": 252.34423828125,
      "margin_dpo/margin_std": 285.3419189453125,
      "step": 661
    },
    {
      "epoch": 0.9720998531571219,
      "fcm_dpo/beta": 0.0025062900967895985,
      "fcm_dpo/delta": 0.08712703734636307,
      "fcm_dpo/margin": 206.3653106689453,
      "fcm_dpo/q_t": 0.3861844539642334,
      "grad_norm": 34.15724563598633,
      "learning_rate": 1.3163925091384532e-09,
      "logits/chosen": -0.5257574319839478,
      "logits/rejected": -0.5023434162139893,
      "logps/chosen": -401.38140869140625,
      "logps/ref_chosen": -69.26106262207031,
      "logps/ref_rejected": -89.05593872070312,
      "logps/rejected": -627.5416259765625,
      "loss": 1.0401,
      "margin_dpo/margin_mean": 206.36529541015625,
      "margin_dpo/margin_std": 271.6153564453125,
      "step": 662
    },
    {
      "epoch": 0.973568281938326,
      "fcm_dpo/beta": 0.002501696813851595,
      "fcm_dpo/delta": -0.016493409872055054,
      "fcm_dpo/margin": 245.9669189453125,
      "fcm_dpo/q_t": 0.3665703535079956,
      "grad_norm": 30.06166648864746,
      "learning_rate": 1.1881460058152382e-09,
      "logits/chosen": -0.5630896687507629,
      "logits/rejected": -0.5610999464988708,
      "logps/chosen": -358.4510498046875,
      "logps/ref_chosen": -64.87890625,
      "logps/ref_rejected": -113.92536926269531,
      "logps/rejected": -653.4644165039062,
      "loss": 0.969,
      "margin_dpo/margin_mean": 245.9669189453125,
      "margin_dpo/margin_std": 283.71075439453125,
      "step": 663
    },
    {
      "epoch": 0.9750367107195301,
      "fcm_dpo/beta": 0.0024649640545248985,
      "fcm_dpo/delta": -0.09488554298877716,
      "fcm_dpo/margin": 279.1371765136719,
      "fcm_dpo/q_t": 0.35125353932380676,
      "grad_norm": 34.83463668823242,
      "learning_rate": 1.066455926241383e-09,
      "logits/chosen": -0.4723320007324219,
      "logits/rejected": -0.4694727063179016,
      "logps/chosen": -382.6121826171875,
      "logps/ref_chosen": -60.88847351074219,
      "logps/ref_rejected": -105.521728515625,
      "logps/rejected": -706.382568359375,
      "loss": 0.9418,
      "margin_dpo/margin_mean": 279.1371765136719,
      "margin_dpo/margin_std": 316.78741455078125,
      "step": 664
    },
    {
      "epoch": 0.9765051395007343,
      "fcm_dpo/beta": 0.0024584224447607994,
      "fcm_dpo/delta": 0.006961781997233629,
      "fcm_dpo/margin": 241.38986206054688,
      "fcm_dpo/q_t": 0.3676924705505371,
      "grad_norm": 32.79265594482422,
      "learning_rate": 9.513254770636137e-10,
      "logits/chosen": -0.4960812032222748,
      "logits/rejected": -0.4816419780254364,
      "logps/chosen": -358.48394775390625,
      "logps/ref_chosen": -60.56413269042969,
      "logps/ref_rejected": -84.80882263183594,
      "logps/rejected": -624.1184692382812,
      "loss": 0.9606,
      "margin_dpo/margin_mean": 241.38986206054688,
      "margin_dpo/margin_std": 244.98712158203125,
      "step": 665
    },
    {
      "epoch": 0.9779735682819384,
      "fcm_dpo/beta": 0.002456535119563341,
      "fcm_dpo/delta": 0.005466273054480553,
      "fcm_dpo/margin": 242.0909423828125,
      "fcm_dpo/q_t": 0.36501848697662354,
      "grad_norm": 38.639163970947266,
      "learning_rate": 8.427576920763956e-10,
      "logits/chosen": -0.4671856760978699,
      "logits/rejected": -0.4504069983959198,
      "logps/chosen": -372.948486328125,
      "logps/ref_chosen": -64.41996002197266,
      "logps/ref_rejected": -95.8916244506836,
      "logps/rejected": -646.5111083984375,
      "loss": 0.9541,
      "margin_dpo/margin_mean": 242.0909423828125,
      "margin_dpo/margin_std": 233.2517852783203,
      "step": 666
    },
    {
      "epoch": 0.9794419970631424,
      "fcm_dpo/beta": 0.002473963890224695,
      "fcm_dpo/delta": 0.02551562525331974,
      "fcm_dpo/margin": 232.80560302734375,
      "fcm_dpo/q_t": 0.3697671890258789,
      "grad_norm": 38.66985321044922,
      "learning_rate": 7.407554321417764e-10,
      "logits/chosen": -0.47391679883003235,
      "logits/rejected": -0.4410842955112457,
      "logps/chosen": -402.3163146972656,
      "logps/ref_chosen": -69.27702331542969,
      "logps/ref_rejected": -87.83549499511719,
      "logps/rejected": -653.680419921875,
      "loss": 0.9638,
      "margin_dpo/margin_mean": 232.80560302734375,
      "margin_dpo/margin_std": 229.169677734375,
      "step": 667
    },
    {
      "epoch": 0.9809104258443465,
      "fcm_dpo/beta": 0.00251585291698575,
      "fcm_dpo/delta": 0.06850365549325943,
      "fcm_dpo/margin": 212.354736328125,
      "fcm_dpo/q_t": 0.3838024139404297,
      "grad_norm": 52.99601745605469,
      "learning_rate": 6.453213851142225e-10,
      "logits/chosen": -0.5375820398330688,
      "logits/rejected": -0.529564380645752,
      "logps/chosen": -411.6194763183594,
      "logps/ref_chosen": -72.60400390625,
      "logps/ref_rejected": -103.73905944824219,
      "logps/rejected": -655.1092529296875,
      "loss": 1.0277,
      "margin_dpo/margin_mean": 212.354736328125,
      "margin_dpo/margin_std": 273.1512145996094,
      "step": 668
    },
    {
      "epoch": 0.9823788546255506,
      "fcm_dpo/beta": 0.0024861646816134453,
      "fcm_dpo/delta": -0.06441953778266907,
      "fcm_dpo/margin": 265.5224609375,
      "fcm_dpo/q_t": 0.35320982336997986,
      "grad_norm": 32.41237258911133,
      "learning_rate": 5.564580657695939e-10,
      "logits/chosen": -0.49402615427970886,
      "logits/rejected": -0.4743247926235199,
      "logps/chosen": -311.386962890625,
      "logps/ref_chosen": -46.116416931152344,
      "logps/ref_rejected": -77.92434692382812,
      "logps/rejected": -608.7173461914062,
      "loss": 0.9308,
      "margin_dpo/margin_mean": 265.5224609375,
      "margin_dpo/margin_std": 272.043701171875,
      "step": 669
    },
    {
      "epoch": 0.9838472834067548,
      "fcm_dpo/beta": 0.0024540587328374386,
      "fcm_dpo/delta": -0.09894061833620071,
      "fcm_dpo/margin": 282.0305480957031,
      "fcm_dpo/q_t": 0.3481866717338562,
      "grad_norm": 27.054269790649414,
      "learning_rate": 4.741678157389739e-10,
      "logits/chosen": -0.4560254216194153,
      "logits/rejected": -0.4461694657802582,
      "logps/chosen": -328.38433837890625,
      "logps/ref_chosen": -62.34575271606445,
      "logps/ref_rejected": -96.9405517578125,
      "logps/rejected": -645.0096435546875,
      "loss": 0.9109,
      "margin_dpo/margin_mean": 282.0305480957031,
      "margin_dpo/margin_std": 275.9473876953125,
      "step": 670
    },
    {
      "epoch": 0.9853157121879589,
      "fcm_dpo/beta": 0.0024509113281965256,
      "fcm_dpo/delta": 0.019564248621463776,
      "fcm_dpo/margin": 237.01344299316406,
      "fcm_dpo/q_t": 0.3713776171207428,
      "grad_norm": 38.91750717163086,
      "learning_rate": 3.9845280344705245e-10,
      "logits/chosen": -0.47259026765823364,
      "logits/rejected": -0.46925845742225647,
      "logps/chosen": -369.33929443359375,
      "logps/ref_chosen": -48.00010681152344,
      "logps/ref_rejected": -83.81932067871094,
      "logps/rejected": -642.1719970703125,
      "loss": 0.9848,
      "margin_dpo/margin_mean": 237.013427734375,
      "margin_dpo/margin_std": 265.4302673339844,
      "step": 671
    },
    {
      "epoch": 0.986784140969163,
      "fcm_dpo/beta": 0.0024676902685314417,
      "fcm_dpo/delta": 0.07117144763469696,
      "fcm_dpo/margin": 215.81497192382812,
      "fcm_dpo/q_t": 0.3837113380432129,
      "grad_norm": 31.851991653442383,
      "learning_rate": 3.293150240547549e-10,
      "logits/chosen": -0.5248799920082092,
      "logits/rejected": -0.5065342783927917,
      "logps/chosen": -408.06689453125,
      "logps/ref_chosen": -58.58328628540039,
      "logps/ref_rejected": -93.14015197753906,
      "logps/rejected": -658.438720703125,
      "loss": 1.0327,
      "margin_dpo/margin_mean": 215.81497192382812,
      "margin_dpo/margin_std": 279.0846252441406,
      "step": 672
    },
    {
      "epoch": 0.9882525697503671,
      "fcm_dpo/beta": 0.0024534822441637516,
      "fcm_dpo/delta": -0.05598102882504463,
      "fcm_dpo/margin": 265.86773681640625,
      "fcm_dpo/q_t": 0.3518218994140625,
      "grad_norm": 31.22127914428711,
      "learning_rate": 2.6675629940689504e-10,
      "logits/chosen": -0.4801812767982483,
      "logits/rejected": -0.4714806079864502,
      "logps/chosen": -353.5696105957031,
      "logps/ref_chosen": -46.72320556640625,
      "logps/ref_rejected": -85.29623413085938,
      "logps/rejected": -658.0103759765625,
      "loss": 0.9048,
      "margin_dpo/margin_mean": 265.86773681640625,
      "margin_dpo/margin_std": 224.6098175048828,
      "step": 673
    },
    {
      "epoch": 0.9897209985315712,
      "fcm_dpo/beta": 0.002430280437693,
      "fcm_dpo/delta": -0.05990447849035263,
      "fcm_dpo/margin": 269.932861328125,
      "fcm_dpo/q_t": 0.3552195131778717,
      "grad_norm": 27.59811019897461,
      "learning_rate": 2.1077827798404725e-10,
      "logits/chosen": -0.45021355152130127,
      "logits/rejected": -0.43945032358169556,
      "logps/chosen": -328.8429870605469,
      "logps/ref_chosen": -45.445526123046875,
      "logps/ref_rejected": -70.04593658447266,
      "logps/rejected": -623.3762817382812,
      "loss": 0.9216,
      "margin_dpo/margin_mean": 269.932861328125,
      "margin_dpo/margin_std": 263.62127685546875,
      "step": 674
    },
    {
      "epoch": 0.9911894273127754,
      "fcm_dpo/beta": 0.0023772665299475193,
      "fcm_dpo/delta": -0.08401606976985931,
      "fcm_dpo/margin": 285.0059814453125,
      "fcm_dpo/q_t": 0.3517681956291199,
      "grad_norm": 29.97161865234375,
      "learning_rate": 1.6138243485910863e-10,
      "logits/chosen": -0.4851476550102234,
      "logits/rejected": -0.47465410828590393,
      "logps/chosen": -329.2435607910156,
      "logps/ref_chosen": -44.17628479003906,
      "logps/ref_rejected": -74.09197998046875,
      "logps/rejected": -644.165283203125,
      "loss": 0.9136,
      "margin_dpo/margin_mean": 285.0060119628906,
      "margin_dpo/margin_std": 279.90576171875,
      "step": 675
    },
    {
      "epoch": 0.9926578560939795,
      "fcm_dpo/beta": 0.0023738746531307697,
      "fcm_dpo/delta": 0.0118600744754076,
      "fcm_dpo/margin": 247.96432495117188,
      "fcm_dpo/q_t": 0.3671633303165436,
      "grad_norm": 31.170412063598633,
      "learning_rate": 1.1857007165852472e-10,
      "logits/chosen": -0.5280739068984985,
      "logits/rejected": -0.5125676393508911,
      "logps/chosen": -399.6715087890625,
      "logps/ref_chosen": -71.39852905273438,
      "logps/ref_rejected": -88.3587646484375,
      "logps/rejected": -664.5960693359375,
      "loss": 0.9577,
      "margin_dpo/margin_mean": 247.96432495117188,
      "margin_dpo/margin_std": 239.91629028320312,
      "step": 676
    },
    {
      "epoch": 0.9941262848751835,
      "fcm_dpo/beta": 0.0024010459892451763,
      "fcm_dpo/delta": 0.033741872757673264,
      "fcm_dpo/margin": 236.56593322753906,
      "fcm_dpo/q_t": 0.37002164125442505,
      "grad_norm": 29.955596923828125,
      "learning_rate": 8.23423165278725e-11,
      "logits/chosen": -0.4702920615673065,
      "logits/rejected": -0.43533623218536377,
      "logps/chosen": -371.1539306640625,
      "logps/ref_chosen": -56.527435302734375,
      "logps/ref_rejected": -78.22654724121094,
      "logps/rejected": -629.4189453125,
      "loss": 0.9604,
      "margin_dpo/margin_mean": 236.56593322753906,
      "margin_dpo/margin_std": 216.29861450195312,
      "step": 677
    },
    {
      "epoch": 0.9955947136563876,
      "fcm_dpo/beta": 0.0023945937864482403,
      "fcm_dpo/delta": -0.025561237707734108,
      "fcm_dpo/margin": 260.56817626953125,
      "fcm_dpo/q_t": 0.36279577016830444,
      "grad_norm": 30.772653579711914,
      "learning_rate": 5.270012410216185e-11,
      "logits/chosen": -0.4658196270465851,
      "logits/rejected": -0.4596235156059265,
      "logps/chosen": -331.82763671875,
      "logps/ref_chosen": -46.13447570800781,
      "logps/ref_rejected": -80.60462951660156,
      "logps/rejected": -626.865966796875,
      "loss": 0.96,
      "margin_dpo/margin_mean": 260.5681457519531,
      "margin_dpo/margin_std": 285.284423828125,
      "step": 678
    },
    {
      "epoch": 0.9970631424375918,
      "fcm_dpo/beta": 0.002417163457721472,
      "fcm_dpo/delta": 0.0805218368768692,
      "fcm_dpo/margin": 216.64544677734375,
      "fcm_dpo/q_t": 0.3846209943294525,
      "grad_norm": 31.97832489013672,
      "learning_rate": 2.9644275480772416e-11,
      "logits/chosen": -0.498549222946167,
      "logits/rejected": -0.4737178087234497,
      "logps/chosen": -354.89007568359375,
      "logps/ref_chosen": -50.294921875,
      "logps/ref_rejected": -76.59813690185547,
      "logps/rejected": -597.8387451171875,
      "loss": 1.0171,
      "margin_dpo/margin_mean": 216.6454315185547,
      "margin_dpo/margin_std": 261.185546875,
      "step": 679
    },
    {
      "epoch": 0.9985315712187959,
      "fcm_dpo/beta": 0.002414638176560402,
      "fcm_dpo/delta": -0.01086747832596302,
      "fcm_dpo/margin": 252.61300659179688,
      "fcm_dpo/q_t": 0.36631903052330017,
      "grad_norm": 35.76852798461914,
      "learning_rate": 1.31753782067201e-11,
      "logits/chosen": -0.5048670768737793,
      "logits/rejected": -0.5000746250152588,
      "logps/chosen": -395.61883544921875,
      "logps/ref_chosen": -76.91569519042969,
      "logps/ref_rejected": -112.384765625,
      "logps/rejected": -683.700927734375,
      "loss": 0.9757,
      "margin_dpo/margin_mean": 252.61300659179688,
      "margin_dpo/margin_std": 294.8744201660156,
      "step": 680
    },
    {
      "epoch": 1.0,
      "fcm_dpo/beta": 0.002459340263158083,
      "fcm_dpo/delta": 0.04384385794401169,
      "fcm_dpo/margin": 226.318603515625,
      "fcm_dpo/q_t": 0.3781166672706604,
      "grad_norm": 32.83413314819336,
      "learning_rate": 3.2938662507808745e-12,
      "logits/chosen": -0.53011155128479,
      "logits/rejected": -0.5230584740638733,
      "logps/chosen": -388.7578125,
      "logps/ref_chosen": -60.957279205322266,
      "logps/ref_rejected": -88.55797576904297,
      "logps/rejected": -642.6771240234375,
      "loss": 1.0058,
      "margin_dpo/margin_mean": 226.318603515625,
      "margin_dpo/margin_std": 260.7975769042969,
      "step": 681
    },
    {
      "epoch": 1.0,
      "step": 681,
      "total_flos": 0.0,
      "train_loss": 0.9836619676647382,
      "train_runtime": 1868.3255,
      "train_samples_per_second": 23.335,
      "train_steps_per_second": 0.364
    }
  ],
  "logging_steps": 1,
  "max_steps": 681,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}