{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.04536247253417969,
      "KL/mean": -0.05389120429754257,
      "KL/rejected_KL_mean": -0.06241607666015625,
      "KL/std": 0.5513200759887695,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01704825460910797,
      "fcm_dpo/q_t": 0.499957412481308,
      "grad_norm": 14.61890697479248,
      "learning_rate": 0.0,
      "logits/chosen": 1.9854813814163208,
      "logits/rejected": 1.7737188339233398,
      "logps/chosen": -267.0936584472656,
      "logps/ref_chosen": -267.04827880859375,
      "logps/ref_rejected": -219.76657104492188,
      "logps/rejected": -219.82899475097656,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.01704716682434082,
      "margin_dpo/margin_std": 0.7467707991600037,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.07550811767578125,
      "KL/mean": 0.03060000389814377,
      "KL/rejected_KL_mean": -0.014301300048828125,
      "KL/std": 0.6403332352638245,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08979703485965729,
      "fcm_dpo/q_t": 0.4997755289077759,
      "grad_norm": 15.141301155090332,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 2.1993117332458496,
      "logits/rejected": 2.1368322372436523,
      "logps/chosen": -256.5101623535156,
      "logps/ref_chosen": -256.585693359375,
      "logps/ref_rejected": -239.92742919921875,
      "logps/rejected": -239.94171142578125,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08979731798171997,
      "margin_dpo/margin_std": 0.9745746850967407,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": 0.011964797973632812,
      "KL/mean": -0.034976981580257416,
      "KL/rejected_KL_mean": -0.08191871643066406,
      "KL/std": 0.6281730532646179,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09388147294521332,
      "fcm_dpo/q_t": 0.4997653067111969,
      "grad_norm": 14.710558891296387,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": 2.256382942199707,
      "logits/rejected": 2.2959229946136475,
      "logps/chosen": -265.6568908691406,
      "logps/ref_chosen": -265.66888427734375,
      "logps/ref_rejected": -284.3778991699219,
      "logps/rejected": -284.4598083496094,
      "loss": 5.5415,
      "margin_dpo/margin_mean": 0.09388026595115662,
      "margin_dpo/margin_std": 0.8301907181739807,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.04450225830078125,
      "KL/mean": -0.025189697742462158,
      "KL/rejected_KL_mean": -0.09487533569335938,
      "KL/std": 0.7065908312797546,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13937604427337646,
      "fcm_dpo/q_t": 0.4996515214443207,
      "grad_norm": 16.939228057861328,
      "learning_rate": 3.125e-08,
      "logits/chosen": 1.7918097972869873,
      "logits/rejected": 1.7272919416427612,
      "logps/chosen": -319.4629821777344,
      "logps/ref_chosen": -319.50750732421875,
      "logps/ref_rejected": -296.22314453125,
      "logps/rejected": -296.3179931640625,
      "loss": 5.5397,
      "margin_dpo/margin_mean": 0.1393764317035675,
      "margin_dpo/margin_std": 0.9487665295600891,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.018970489501953125,
      "KL/mean": 0.014476306736469269,
      "KL/rejected_KL_mean": 0.009992599487304688,
      "KL/std": 0.6308712959289551,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.008981391787528992,
      "fcm_dpo/q_t": 0.49997755885124207,
      "grad_norm": 15.7199125289917,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": 1.9322835206985474,
      "logits/rejected": 1.9453833103179932,
      "logps/chosen": -292.8540344238281,
      "logps/ref_chosen": -292.87298583984375,
      "logps/ref_rejected": -270.3802795410156,
      "logps/rejected": -270.3702697753906,
      "loss": 5.5449,
      "margin_dpo/margin_mean": 0.008981779217720032,
      "margin_dpo/margin_std": 0.8075746297836304,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": -0.03542518615722656,
      "KL/mean": 0.004847772419452667,
      "KL/rejected_KL_mean": 0.0451202392578125,
      "KL/std": 0.5974876880645752,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.08054231107234955,
      "fcm_dpo/q_t": 0.5002013444900513,
      "grad_norm": 15.444246292114258,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 2.1559972763061523,
      "logits/rejected": 2.094434976577759,
      "logps/chosen": -280.3208923339844,
      "logps/ref_chosen": -280.28546142578125,
      "logps/ref_rejected": -242.7611541748047,
      "logps/rejected": -242.71603393554688,
      "loss": 5.5485,
      "margin_dpo/margin_mean": -0.08054298162460327,
      "margin_dpo/margin_std": 0.8793975114822388,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": -0.040313720703125,
      "KL/mean": -0.004289403557777405,
      "KL/rejected_KL_mean": 0.03173828125,
      "KL/std": 0.6521949768066406,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07205775380134583,
      "fcm_dpo/q_t": 0.5001801252365112,
      "grad_norm": 15.699423789978027,
      "learning_rate": 6.25e-08,
      "logits/chosen": 2.0065722465515137,
      "logits/rejected": 1.9197604656219482,
      "logps/chosen": -255.14602661132812,
      "logps/ref_chosen": -255.10572814941406,
      "logps/ref_rejected": -251.15402221679688,
      "logps/rejected": -251.12228393554688,
      "loss": 5.5482,
      "margin_dpo/margin_mean": -0.07205848395824432,
      "margin_dpo/margin_std": 0.9182904362678528,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.017284393310546875,
      "KL/mean": 0.022129356861114502,
      "KL/rejected_KL_mean": 0.06154823303222656,
      "KL/std": 0.7080288529396057,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0788525640964508,
      "fcm_dpo/q_t": 0.500197172164917,
      "grad_norm": 15.929621696472168,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": 2.284201145172119,
      "logits/rejected": 2.089193820953369,
      "logps/chosen": -345.1988525390625,
      "logps/ref_chosen": -345.18157958984375,
      "logps/ref_rejected": -277.3522644042969,
      "logps/rejected": -277.2907409667969,
      "loss": 5.5484,
      "margin_dpo/margin_mean": -0.07885238528251648,
      "margin_dpo/margin_std": 0.9571667313575745,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.023832321166992188,
      "KL/mean": -0.024019300937652588,
      "KL/rejected_KL_mean": -0.024202346801757812,
      "KL/std": 0.6660038232803345,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.00037151575088500977,
      "fcm_dpo/q_t": 0.499999076128006,
      "grad_norm": 14.122302055358887,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": 1.9752345085144043,
      "logits/rejected": 1.9466899633407593,
      "logps/chosen": -276.7541809082031,
      "logps/ref_chosen": -276.7303161621094,
      "logps/ref_rejected": -266.19952392578125,
      "logps/rejected": -266.22369384765625,
      "loss": 5.5453,
      "margin_dpo/margin_mean": 0.0003725886344909668,
      "margin_dpo/margin_std": 0.9617262482643127,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": -0.0417022705078125,
      "KL/mean": -0.024755097925662994,
      "KL/rejected_KL_mean": -0.0078105926513671875,
      "KL/std": 0.5764731168746948,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03389562666416168,
      "fcm_dpo/q_t": 0.5000846982002258,
      "grad_norm": 14.002037048339844,
      "learning_rate": 9.375e-08,
      "logits/chosen": 2.015033721923828,
      "logits/rejected": 2.0728096961975098,
      "logps/chosen": -246.6039581298828,
      "logps/ref_chosen": -246.562255859375,
      "logps/ref_rejected": -224.70401000976562,
      "logps/rejected": -224.7117919921875,
      "loss": 5.5466,
      "margin_dpo/margin_mean": -0.03389531373977661,
      "margin_dpo/margin_std": 0.7300001382827759,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": -0.019695281982421875,
      "KL/mean": -0.050078097730875015,
      "KL/rejected_KL_mean": -0.08045196533203125,
      "KL/std": 0.6451322436332703,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06075254827737808,
      "fcm_dpo/q_t": 0.4998481273651123,
      "grad_norm": 15.389554977416992,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 2.0427331924438477,
      "logits/rejected": 1.987351417541504,
      "logps/chosen": -290.5207824707031,
      "logps/ref_chosen": -290.5010986328125,
      "logps/ref_rejected": -253.9559326171875,
      "logps/rejected": -254.03634643554688,
      "loss": 5.5428,
      "margin_dpo/margin_mean": 0.06075235456228256,
      "margin_dpo/margin_std": 0.8239589929580688,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": -0.022192001342773438,
      "KL/mean": 0.014331161975860596,
      "KL/rejected_KL_mean": 0.0508575439453125,
      "KL/std": 0.5983895063400269,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07305729389190674,
      "fcm_dpo/q_t": 0.5001826286315918,
      "grad_norm": 14.805671691894531,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": 1.5643898248672485,
      "logits/rejected": 1.5478968620300293,
      "logps/chosen": -278.54071044921875,
      "logps/ref_chosen": -278.5185241699219,
      "logps/ref_rejected": -273.56207275390625,
      "logps/rejected": -273.51123046875,
      "loss": 5.5482,
      "margin_dpo/margin_mean": -0.07305683195590973,
      "margin_dpo/margin_std": 0.8199501633644104,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.0550079345703125,
      "KL/mean": -0.06133158504962921,
      "KL/rejected_KL_mean": -0.06765556335449219,
      "KL/std": 0.6161543726921082,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01265329122543335,
      "fcm_dpo/q_t": 0.49996840953826904,
      "grad_norm": 14.801441192626953,
      "learning_rate": 1.25e-07,
      "logits/chosen": 1.809030532836914,
      "logits/rejected": 1.9366787672042847,
      "logps/chosen": -264.7121276855469,
      "logps/ref_chosen": -264.6571350097656,
      "logps/ref_rejected": -274.3492736816406,
      "logps/rejected": -274.4169616699219,
      "loss": 5.5447,
      "margin_dpo/margin_mean": 0.012652680277824402,
      "margin_dpo/margin_std": 0.795184314250946,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.06851768493652344,
      "KL/mean": 0.045767880976200104,
      "KL/rejected_KL_mean": 0.023014068603515625,
      "KL/std": 0.6428378224372864,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04551221430301666,
      "fcm_dpo/q_t": 0.4998862147331238,
      "grad_norm": 15.195685386657715,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": 1.944555640220642,
      "logits/rejected": 1.968363642692566,
      "logps/chosen": -274.0460510253906,
      "logps/ref_chosen": -274.1145935058594,
      "logps/ref_rejected": -261.0951843261719,
      "logps/rejected": -261.0721740722656,
      "loss": 5.5435,
      "margin_dpo/margin_mean": 0.045512035489082336,
      "margin_dpo/margin_std": 0.9175758957862854,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": 0.016941070556640625,
      "KL/mean": 0.010041125118732452,
      "KL/rejected_KL_mean": 0.0031337738037109375,
      "KL/std": 0.6187252998352051,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01381199061870575,
      "fcm_dpo/q_t": 0.4999654293060303,
      "grad_norm": 16.057292938232422,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": 2.1034514904022217,
      "logits/rejected": 1.9798694849014282,
      "logps/chosen": -288.56781005859375,
      "logps/ref_chosen": -288.584716796875,
      "logps/ref_rejected": -224.8094482421875,
      "logps/rejected": -224.8063201904297,
      "loss": 5.5447,
      "margin_dpo/margin_mean": 0.01381191611289978,
      "margin_dpo/margin_std": 0.8627191185951233,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": -0.07950782775878906,
      "KL/mean": -0.09135378897190094,
      "KL/rejected_KL_mean": -0.10319900512695312,
      "KL/std": 0.6704068183898926,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02369016408920288,
      "fcm_dpo/q_t": 0.49994078278541565,
      "grad_norm": 16.58214569091797,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": 2.051470994949341,
      "logits/rejected": 1.9153982400894165,
      "logps/chosen": -283.9226379394531,
      "logps/ref_chosen": -283.8431091308594,
      "logps/ref_rejected": -248.63645935058594,
      "logps/rejected": -248.73968505859375,
      "loss": 5.5443,
      "margin_dpo/margin_mean": 0.023690536618232727,
      "margin_dpo/margin_std": 0.9911866784095764,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.05753326416015625,
      "KL/mean": 0.051471464335918427,
      "KL/rejected_KL_mean": 0.045406341552734375,
      "KL/std": 0.5948900580406189,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.012122318148612976,
      "fcm_dpo/q_t": 0.4999697208404541,
      "grad_norm": 15.230005264282227,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 1.9430903196334839,
      "logits/rejected": 1.9035202264785767,
      "logps/chosen": -254.57603454589844,
      "logps/ref_chosen": -254.6335906982422,
      "logps/ref_rejected": -245.3981170654297,
      "logps/rejected": -245.3527069091797,
      "loss": 5.5448,
      "margin_dpo/margin_mean": 0.012122467160224915,
      "margin_dpo/margin_std": 0.8353469967842102,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.046398162841796875,
      "KL/mean": 0.014631368219852448,
      "KL/rejected_KL_mean": 0.07564926147460938,
      "KL/std": 0.5732801556587219,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.12205047905445099,
      "fcm_dpo/q_t": 0.5003051161766052,
      "grad_norm": 15.087552070617676,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": 1.9591819047927856,
      "logits/rejected": 1.8706018924713135,
      "logps/chosen": -287.7959289550781,
      "logps/ref_chosen": -287.7495422363281,
      "logps/ref_rejected": -273.03363037109375,
      "logps/rejected": -272.9579772949219,
      "loss": 5.5501,
      "margin_dpo/margin_mean": -0.12205088138580322,
      "margin_dpo/margin_std": 0.7901231050491333,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.004154205322265625,
      "KL/mean": 0.018218830227851868,
      "KL/rejected_KL_mean": 0.040592193603515625,
      "KL/std": 0.6369118690490723,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.04474249482154846,
      "fcm_dpo/q_t": 0.5001118779182434,
      "grad_norm": 14.910365104675293,
      "learning_rate": 1.875e-07,
      "logits/chosen": 2.065115451812744,
      "logits/rejected": 2.033435821533203,
      "logps/chosen": -273.4080505371094,
      "logps/ref_chosen": -273.4039001464844,
      "logps/ref_rejected": -260.09722900390625,
      "logps/rejected": -260.0566101074219,
      "loss": 5.5471,
      "margin_dpo/margin_mean": -0.044742852449417114,
      "margin_dpo/margin_std": 0.8832864761352539,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.10010337829589844,
      "KL/mean": 0.050181642174720764,
      "KL/rejected_KL_mean": 0.0002651214599609375,
      "KL/std": 0.6293897032737732,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0998464822769165,
      "fcm_dpo/q_t": 0.49975040555000305,
      "grad_norm": 15.465063095092773,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.8417750597000122,
      "logits/rejected": 1.7885390520095825,
      "logps/chosen": -292.86199951171875,
      "logps/ref_chosen": -292.96209716796875,
      "logps/ref_rejected": -256.602783203125,
      "logps/rejected": -256.6025085449219,
      "loss": 5.5413,
      "margin_dpo/margin_mean": 0.09984631836414337,
      "margin_dpo/margin_std": 0.823845386505127,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.019054412841796875,
      "KL/mean": -0.006033807992935181,
      "KL/rejected_KL_mean": -0.031131744384765625,
      "KL/std": 0.656477153301239,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05018627643585205,
      "fcm_dpo/q_t": 0.4998745322227478,
      "grad_norm": 15.503681182861328,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 2.0545313358306885,
      "logits/rejected": 1.981313705444336,
      "logps/chosen": -292.30816650390625,
      "logps/ref_chosen": -292.3271789550781,
      "logps/ref_rejected": -274.20635986328125,
      "logps/rejected": -274.2375183105469,
      "loss": 5.5433,
      "margin_dpo/margin_mean": 0.05018548667430878,
      "margin_dpo/margin_std": 0.8881590366363525,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.10235786437988281,
      "KL/mean": 0.05713986977934837,
      "KL/rejected_KL_mean": 0.01192474365234375,
      "KL/std": 0.5605254173278809,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09043645113706589,
      "fcm_dpo/q_t": 0.49977391958236694,
      "grad_norm": 13.763216972351074,
      "learning_rate": 2.1875e-07,
      "logits/chosen": 2.0775809288024902,
      "logits/rejected": 2.085012197494507,
      "logps/chosen": -253.16384887695312,
      "logps/ref_chosen": -253.2662353515625,
      "logps/ref_rejected": -241.4681396484375,
      "logps/rejected": -241.4562225341797,
      "loss": 5.5416,
      "margin_dpo/margin_mean": 0.09043721854686737,
      "margin_dpo/margin_std": 0.6532580852508545,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.08785438537597656,
      "KL/mean": 0.010299596935510635,
      "KL/rejected_KL_mean": -0.06725692749023438,
      "KL/std": 0.5934419631958008,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15511207282543182,
      "fcm_dpo/q_t": 0.4996122419834137,
      "grad_norm": 15.19594669342041,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": 2.2065234184265137,
      "logits/rejected": 2.2063889503479004,
      "logps/chosen": -275.3337097167969,
      "logps/ref_chosen": -275.4216003417969,
      "logps/ref_rejected": -264.44024658203125,
      "logps/rejected": -264.50750732421875,
      "loss": 5.539,
      "margin_dpo/margin_mean": 0.15511269867420197,
      "margin_dpo/margin_std": 0.8116491436958313,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": -0.0317535400390625,
      "KL/mean": 0.021567709743976593,
      "KL/rejected_KL_mean": 0.07489204406738281,
      "KL/std": 0.6496813297271729,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.10664214193820953,
      "fcm_dpo/q_t": 0.5002665519714355,
      "grad_norm": 16.546751022338867,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": 1.8981962203979492,
      "logits/rejected": 1.9241142272949219,
      "logps/chosen": -293.4093322753906,
      "logps/ref_chosen": -293.3775634765625,
      "logps/ref_rejected": -301.59014892578125,
      "logps/rejected": -301.5152587890625,
      "loss": 5.5495,
      "margin_dpo/margin_mean": -0.10664233565330505,
      "margin_dpo/margin_std": 0.9528028964996338,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.06864356994628906,
      "KL/mean": 0.03217141330242157,
      "KL/rejected_KL_mean": -0.00429534912109375,
      "KL/std": 0.599852979183197,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07293584942817688,
      "fcm_dpo/q_t": 0.4998176693916321,
      "grad_norm": 14.550215721130371,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.8030617237091064,
      "logits/rejected": 1.8422044515609741,
      "logps/chosen": -240.8650360107422,
      "logps/ref_chosen": -240.93368530273438,
      "logps/ref_rejected": -235.3230743408203,
      "logps/rejected": -235.32736206054688,
      "loss": 5.5423,
      "margin_dpo/margin_mean": 0.07293592393398285,
      "margin_dpo/margin_std": 0.791696310043335,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": 0.05198097229003906,
      "KL/mean": 0.005136556923389435,
      "KL/rejected_KL_mean": -0.0417022705078125,
      "KL/std": 0.5901949405670166,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09368611872196198,
      "fcm_dpo/q_t": 0.4997658133506775,
      "grad_norm": 15.579866409301758,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": 1.8310253620147705,
      "logits/rejected": 1.8241422176361084,
      "logps/chosen": -273.5433349609375,
      "logps/ref_chosen": -273.5953369140625,
      "logps/ref_rejected": -234.49148559570312,
      "logps/rejected": -234.53318786621094,
      "loss": 5.5415,
      "margin_dpo/margin_mean": 0.09368638694286346,
      "margin_dpo/margin_std": 0.7569026947021484,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.065673828125,
      "KL/mean": -0.005172401666641235,
      "KL/rejected_KL_mean": -0.07601356506347656,
      "KL/std": 0.6471235156059265,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1416812539100647,
      "fcm_dpo/q_t": 0.49964576959609985,
      "grad_norm": 15.147098541259766,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": 2.173008441925049,
      "logits/rejected": 1.9318007230758667,
      "logps/chosen": -314.1982116699219,
      "logps/ref_chosen": -314.26385498046875,
      "logps/ref_rejected": -242.06735229492188,
      "logps/rejected": -242.1433563232422,
      "loss": 5.5396,
      "margin_dpo/margin_mean": 0.14168088138103485,
      "margin_dpo/margin_std": 0.9031927585601807,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.1670360565185547,
      "KL/mean": 0.13729628920555115,
      "KL/rejected_KL_mean": 0.10755538940429688,
      "KL/std": 0.6009537577629089,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.059474214911460876,
      "fcm_dpo/q_t": 0.4998513162136078,
      "grad_norm": 15.400225639343262,
      "learning_rate": 2.8125e-07,
      "logits/chosen": 2.012343406677246,
      "logits/rejected": 2.0195155143737793,
      "logps/chosen": -240.57177734375,
      "logps/ref_chosen": -240.7388153076172,
      "logps/ref_rejected": -237.8331298828125,
      "logps/rejected": -237.7255859375,
      "loss": 5.5429,
      "margin_dpo/margin_mean": 0.059473857283592224,
      "margin_dpo/margin_std": 0.7671529054641724,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.06951141357421875,
      "KL/mean": 0.023282930254936218,
      "KL/rejected_KL_mean": -0.02294158935546875,
      "KL/std": 0.6735391616821289,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09245225787162781,
      "fcm_dpo/q_t": 0.4997688829898834,
      "grad_norm": 14.727362632751465,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": 2.231151819229126,
      "logits/rejected": 2.161248207092285,
      "logps/chosen": -307.7598876953125,
      "logps/ref_chosen": -307.82940673828125,
      "logps/ref_rejected": -281.8109130859375,
      "logps/rejected": -281.8338317871094,
      "loss": 5.5416,
      "margin_dpo/margin_mean": 0.09245166182518005,
      "margin_dpo/margin_std": 0.9269174933433533,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.05083274841308594,
      "KL/mean": 0.01891952008008957,
      "KL/rejected_KL_mean": -0.012994766235351562,
      "KL/std": 0.5521615743637085,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06383581459522247,
      "fcm_dpo/q_t": 0.4998404383659363,
      "grad_norm": 14.398689270019531,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.843553066253662,
      "logits/rejected": 1.8417586088180542,
      "logps/chosen": -247.96295166015625,
      "logps/ref_chosen": -248.01376342773438,
      "logps/ref_rejected": -259.0014953613281,
      "logps/rejected": -259.0144958496094,
      "loss": 5.5427,
      "margin_dpo/margin_mean": 0.06383557617664337,
      "margin_dpo/margin_std": 0.7376356720924377,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.15418624877929688,
      "KL/mean": 0.07422264665365219,
      "KL/rejected_KL_mean": -0.0057430267333984375,
      "KL/std": 0.6132599711418152,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1599252074956894,
      "fcm_dpo/q_t": 0.49960023164749146,
      "grad_norm": 13.657106399536133,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 1.9598398208618164,
      "logits/rejected": 1.8985774517059326,
      "logps/chosen": -239.9086151123047,
      "logps/ref_chosen": -240.0628204345703,
      "logps/ref_rejected": -201.91021728515625,
      "logps/rejected": -201.91595458984375,
      "loss": 5.5389,
      "margin_dpo/margin_mean": 0.1599239706993103,
      "margin_dpo/margin_std": 0.762336015701294,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.06338882446289062,
      "KL/mean": 0.02764827385544777,
      "KL/rejected_KL_mean": -0.008085250854492188,
      "KL/std": 0.6432656645774841,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07146499305963516,
      "fcm_dpo/q_t": 0.49982136487960815,
      "grad_norm": 15.578837394714355,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": 1.9551210403442383,
      "logits/rejected": 1.8822119235992432,
      "logps/chosen": -294.06884765625,
      "logps/ref_chosen": -294.1322326660156,
      "logps/ref_rejected": -265.8150634765625,
      "logps/rejected": -265.8231201171875,
      "loss": 5.5424,
      "margin_dpo/margin_mean": 0.07146459817886353,
      "margin_dpo/margin_std": 0.79095458984375,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.26575279235839844,
      "KL/mean": 0.17796005308628082,
      "KL/rejected_KL_mean": 0.09015274047851562,
      "KL/std": 0.6747806668281555,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1756047010421753,
      "fcm_dpo/q_t": 0.49956098198890686,
      "grad_norm": 14.935774803161621,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 2.1671762466430664,
      "logits/rejected": 2.1500463485717773,
      "logps/chosen": -275.38934326171875,
      "logps/ref_chosen": -275.6551208496094,
      "logps/ref_rejected": -253.5972137451172,
      "logps/rejected": -253.50704956054688,
      "loss": 5.5383,
      "margin_dpo/margin_mean": 0.17560473084449768,
      "margin_dpo/margin_std": 0.9286333322525024,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.2560920715332031,
      "KL/mean": 0.1566370129585266,
      "KL/rejected_KL_mean": 0.057186126708984375,
      "KL/std": 0.7480279207229614,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1988913118839264,
      "fcm_dpo/q_t": 0.4995027780532837,
      "grad_norm": 17.69593048095703,
      "learning_rate": 3.4375e-07,
      "logits/chosen": 1.9501967430114746,
      "logits/rejected": 1.829155683517456,
      "logps/chosen": -332.7139892578125,
      "logps/ref_chosen": -332.9701232910156,
      "logps/ref_rejected": -278.2213439941406,
      "logps/rejected": -278.1641540527344,
      "loss": 5.5373,
      "margin_dpo/margin_mean": 0.19889086484909058,
      "margin_dpo/margin_std": 1.011539101600647,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.11819648742675781,
      "KL/mean": 0.09648744761943817,
      "KL/rejected_KL_mean": 0.07478523254394531,
      "KL/std": 0.6844741106033325,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04342344403266907,
      "fcm_dpo/q_t": 0.499891459941864,
      "grad_norm": 13.692011833190918,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": 1.7702629566192627,
      "logits/rejected": 1.8306357860565186,
      "logps/chosen": -258.3106689453125,
      "logps/ref_chosen": -258.42889404296875,
      "logps/ref_rejected": -259.5119934082031,
      "logps/rejected": -259.4371643066406,
      "loss": 5.5435,
      "margin_dpo/margin_mean": 0.043423622846603394,
      "margin_dpo/margin_std": 0.9360470175743103,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.33817481994628906,
      "KL/mean": 0.1802654266357422,
      "KL/rejected_KL_mean": 0.022365570068359375,
      "KL/std": 0.7815188765525818,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.315809428691864,
      "fcm_dpo/q_t": 0.49921050667762756,
      "grad_norm": 17.498188018798828,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": 1.9217441082000732,
      "logits/rejected": 1.8685119152069092,
      "logps/chosen": -290.28118896484375,
      "logps/ref_chosen": -290.6193542480469,
      "logps/ref_rejected": -280.2503967285156,
      "logps/rejected": -280.22802734375,
      "loss": 5.5327,
      "margin_dpo/margin_mean": 0.315809041261673,
      "margin_dpo/margin_std": 1.095875859260559,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.28701019287109375,
      "KL/mean": 0.22493845224380493,
      "KL/rejected_KL_mean": 0.1628570556640625,
      "KL/std": 0.7361248731613159,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12415733188390732,
      "fcm_dpo/q_t": 0.49968963861465454,
      "grad_norm": 15.15367603302002,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.9018648862838745,
      "logits/rejected": 1.893281102180481,
      "logps/chosen": -303.28582763671875,
      "logps/ref_chosen": -303.57281494140625,
      "logps/ref_rejected": -294.56878662109375,
      "logps/rejected": -294.4059143066406,
      "loss": 5.5403,
      "margin_dpo/margin_mean": 0.12415631860494614,
      "margin_dpo/margin_std": 1.0107661485671997,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.3103466033935547,
      "KL/mean": 0.22250132262706757,
      "KL/rejected_KL_mean": 0.1346607208251953,
      "KL/std": 0.7436400055885315,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1756768375635147,
      "fcm_dpo/q_t": 0.49956080317497253,
      "grad_norm": 15.119152069091797,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": 1.8437223434448242,
      "logits/rejected": 1.893830418586731,
      "logps/chosen": -272.2431640625,
      "logps/ref_chosen": -272.5534973144531,
      "logps/ref_rejected": -266.9136047363281,
      "logps/rejected": -266.7789611816406,
      "loss": 5.5383,
      "margin_dpo/margin_mean": 0.17567716538906097,
      "margin_dpo/margin_std": 1.004802942276001,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 0.500701904296875,
      "KL/mean": 0.2996448874473572,
      "KL/rejected_KL_mean": 0.09859275817871094,
      "KL/std": 0.7649118900299072,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4021117091178894,
      "fcm_dpo/q_t": 0.49899473786354065,
      "grad_norm": 15.766634941101074,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": 1.674131989479065,
      "logits/rejected": 1.599538803100586,
      "logps/chosen": -264.8858337402344,
      "logps/ref_chosen": -265.38653564453125,
      "logps/ref_rejected": -244.34774780273438,
      "logps/rejected": -244.24917602539062,
      "loss": 5.5292,
      "margin_dpo/margin_mean": 0.4021109938621521,
      "margin_dpo/margin_std": 0.866349458694458,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 0.5692138671875,
      "KL/mean": 0.32225462794303894,
      "KL/rejected_KL_mean": 0.07529640197753906,
      "KL/std": 0.7736038565635681,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.49391674995422363,
      "fcm_dpo/q_t": 0.4987652897834778,
      "grad_norm": 14.962098121643066,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.743635654449463,
      "logits/rejected": 1.7243411540985107,
      "logps/chosen": -274.3245849609375,
      "logps/ref_chosen": -274.893798828125,
      "logps/ref_rejected": -244.3120880126953,
      "logps/rejected": -244.23680114746094,
      "loss": 5.5256,
      "margin_dpo/margin_mean": 0.4939155876636505,
      "margin_dpo/margin_std": 1.04802405834198,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 0.3116493225097656,
      "KL/mean": 0.23255746066570282,
      "KL/rejected_KL_mean": 0.15346527099609375,
      "KL/std": 0.7285861372947693,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15818263590335846,
      "fcm_dpo/q_t": 0.49960455298423767,
      "grad_norm": 15.694361686706543,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 1.916085124015808,
      "logits/rejected": 1.8279353380203247,
      "logps/chosen": -284.48748779296875,
      "logps/ref_chosen": -284.7991027832031,
      "logps/ref_rejected": -248.8882598876953,
      "logps/rejected": -248.7347869873047,
      "loss": 5.5389,
      "margin_dpo/margin_mean": 0.15818291902542114,
      "margin_dpo/margin_std": 0.9550891518592834,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.4759540557861328,
      "KL/mean": 0.31227797269821167,
      "KL/rejected_KL_mean": 0.1486034393310547,
      "KL/std": 0.7599216103553772,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3273537755012512,
      "fcm_dpo/q_t": 0.4991816282272339,
      "grad_norm": 14.550949096679688,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": 2.0020487308502197,
      "logits/rejected": 2.0612542629241943,
      "logps/chosen": -245.3637237548828,
      "logps/ref_chosen": -245.83966064453125,
      "logps/ref_rejected": -259.2870788574219,
      "logps/rejected": -259.1385192871094,
      "loss": 5.5322,
      "margin_dpo/margin_mean": 0.32735365629196167,
      "margin_dpo/margin_std": 0.9645758271217346,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.4735698699951172,
      "KL/mean": 0.25026118755340576,
      "KL/rejected_KL_mean": 0.026958465576171875,
      "KL/std": 0.9156359434127808,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4466080069541931,
      "fcm_dpo/q_t": 0.4988834857940674,
      "grad_norm": 15.666962623596191,
      "learning_rate": 4.375e-07,
      "logits/chosen": 1.8349411487579346,
      "logits/rejected": 1.8126323223114014,
      "logps/chosen": -307.9239807128906,
      "logps/ref_chosen": -308.39752197265625,
      "logps/ref_rejected": -270.3797912597656,
      "logps/rejected": -270.35284423828125,
      "loss": 5.5275,
      "margin_dpo/margin_mean": 0.44660818576812744,
      "margin_dpo/margin_std": 1.2032585144042969,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 0.6691036224365234,
      "KL/mean": 0.3709958493709564,
      "KL/rejected_KL_mean": 0.07288551330566406,
      "KL/std": 0.9258967638015747,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5962182283401489,
      "fcm_dpo/q_t": 0.4985094964504242,
      "grad_norm": 15.777692794799805,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": 1.7514632940292358,
      "logits/rejected": 1.742828130722046,
      "logps/chosen": -274.692626953125,
      "logps/ref_chosen": -275.36175537109375,
      "logps/ref_rejected": -279.7540588378906,
      "logps/rejected": -279.6811828613281,
      "loss": 5.5216,
      "margin_dpo/margin_mean": 0.5962188243865967,
      "margin_dpo/margin_std": 1.333499550819397,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 0.8269519805908203,
      "KL/mean": 0.5170125961303711,
      "KL/rejected_KL_mean": 0.20707321166992188,
      "KL/std": 1.0235893726348877,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6198720932006836,
      "fcm_dpo/q_t": 0.4984504282474518,
      "grad_norm": 16.998027801513672,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": 1.7448126077651978,
      "logits/rejected": 1.7032006978988647,
      "logps/chosen": -314.2932434082031,
      "logps/ref_chosen": -315.1202087402344,
      "logps/ref_rejected": -290.34210205078125,
      "logps/rejected": -290.1350402832031,
      "loss": 5.5206,
      "margin_dpo/margin_mean": 0.6198710203170776,
      "margin_dpo/margin_std": 1.3211122751235962,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 0.7853775024414062,
      "KL/mean": 0.44616198539733887,
      "KL/rejected_KL_mean": 0.10694313049316406,
      "KL/std": 0.9420828819274902,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6784301996231079,
      "fcm_dpo/q_t": 0.49830397963523865,
      "grad_norm": 16.514205932617188,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": 1.724705696105957,
      "logits/rejected": 1.729569673538208,
      "logps/chosen": -292.15521240234375,
      "logps/ref_chosen": -292.94061279296875,
      "logps/ref_rejected": -264.0979309082031,
      "logps/rejected": -263.9909973144531,
      "loss": 5.5182,
      "margin_dpo/margin_mean": 0.6784305572509766,
      "margin_dpo/margin_std": 1.1612238883972168,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 0.5977306365966797,
      "KL/mean": 0.4410361647605896,
      "KL/rejected_KL_mean": 0.2843437194824219,
      "KL/std": 0.9705711603164673,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.31338584423065186,
      "fcm_dpo/q_t": 0.49921658635139465,
      "grad_norm": 13.563760757446289,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": 1.8230832815170288,
      "logits/rejected": 1.9280116558074951,
      "logps/chosen": -254.25497436523438,
      "logps/ref_chosen": -254.85269165039062,
      "logps/ref_rejected": -255.88380432128906,
      "logps/rejected": -255.59942626953125,
      "loss": 5.5329,
      "margin_dpo/margin_mean": 0.31338539719581604,
      "margin_dpo/margin_std": 1.3753273487091064,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 0.7555789947509766,
      "KL/mean": 0.47741976380348206,
      "KL/rejected_KL_mean": 0.19925689697265625,
      "KL/std": 1.064993143081665,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5563247203826904,
      "fcm_dpo/q_t": 0.498609334230423,
      "grad_norm": 14.84977912902832,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": 1.9120694398880005,
      "logits/rejected": 1.8959877490997314,
      "logps/chosen": -261.7521667480469,
      "logps/ref_chosen": -262.5077209472656,
      "logps/ref_rejected": -239.25320434570312,
      "logps/rejected": -239.053955078125,
      "loss": 5.5232,
      "margin_dpo/margin_mean": 0.5563240051269531,
      "margin_dpo/margin_std": 1.3645246028900146,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.9799823760986328,
      "KL/mean": 0.679954469203949,
      "KL/rejected_KL_mean": 0.37994384765625,
      "KL/std": 1.2010356187820435,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6000455617904663,
      "fcm_dpo/q_t": 0.4985000193119049,
      "grad_norm": 15.207971572875977,
      "learning_rate": 5e-07,
      "logits/chosen": 2.0270321369171143,
      "logits/rejected": 2.1388378143310547,
      "logps/chosen": -248.54791259765625,
      "logps/ref_chosen": -249.5278778076172,
      "logps/ref_rejected": -254.6465301513672,
      "logps/rejected": -254.26657104492188,
      "loss": 5.5215,
      "margin_dpo/margin_mean": 0.6000461578369141,
      "margin_dpo/margin_std": 1.6541342735290527,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 0.9637680053710938,
      "KL/mean": 0.7131020426750183,
      "KL/rejected_KL_mean": 0.4624195098876953,
      "KL/std": 1.1217046976089478,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5013490915298462,
      "fcm_dpo/q_t": 0.49874669313430786,
      "grad_norm": 14.636533737182617,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 1.9905931949615479,
      "logits/rejected": 2.002610445022583,
      "logps/chosen": -291.5379333496094,
      "logps/ref_chosen": -292.501708984375,
      "logps/ref_rejected": -294.6496887207031,
      "logps/rejected": -294.187255859375,
      "loss": 5.5254,
      "margin_dpo/margin_mean": 0.5013490915298462,
      "margin_dpo/margin_std": 1.3868173360824585,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 0.8939304351806641,
      "KL/mean": 0.5718977451324463,
      "KL/rejected_KL_mean": 0.24985885620117188,
      "KL/std": 1.308363437652588,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6440691351890564,
      "fcm_dpo/q_t": 0.49838995933532715,
      "grad_norm": 14.735676765441895,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 1.4649405479431152,
      "logits/rejected": 1.5940523147583008,
      "logps/chosen": -258.96795654296875,
      "logps/ref_chosen": -259.8619079589844,
      "logps/ref_rejected": -293.1994934082031,
      "logps/rejected": -292.94964599609375,
      "loss": 5.5198,
      "margin_dpo/margin_mean": 0.6440699100494385,
      "margin_dpo/margin_std": 1.7918469905853271,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 1.2152996063232422,
      "KL/mean": 0.7828458547592163,
      "KL/rejected_KL_mean": 0.35040283203125,
      "KL/std": 1.415105938911438,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8648933172225952,
      "fcm_dpo/q_t": 0.4978380799293518,
      "grad_norm": 17.10328483581543,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": 2.0653188228607178,
      "logits/rejected": 1.991447925567627,
      "logps/chosen": -297.21002197265625,
      "logps/ref_chosen": -298.42529296875,
      "logps/ref_rejected": -278.5128479003906,
      "logps/rejected": -278.16241455078125,
      "loss": 5.511,
      "margin_dpo/margin_mean": 0.8648929595947266,
      "margin_dpo/margin_std": 1.8851488828659058,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 1.2122745513916016,
      "KL/mean": 0.8257365822792053,
      "KL/rejected_KL_mean": 0.4391937255859375,
      "KL/std": 1.4535601139068604,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7730848789215088,
      "fcm_dpo/q_t": 0.4980675280094147,
      "grad_norm": 16.01097297668457,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": 1.791596531867981,
      "logits/rejected": 1.8592129945755005,
      "logps/chosen": -274.0962219238281,
      "logps/ref_chosen": -275.3085021972656,
      "logps/ref_rejected": -267.97381591796875,
      "logps/rejected": -267.5346374511719,
      "loss": 5.5147,
      "margin_dpo/margin_mean": 0.7730849385261536,
      "margin_dpo/margin_std": 1.8076071739196777,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 1.2410144805908203,
      "KL/mean": 0.9135382175445557,
      "KL/rejected_KL_mean": 0.5860595703125,
      "KL/std": 1.4959383010864258,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6549609899520874,
      "fcm_dpo/q_t": 0.49836283922195435,
      "grad_norm": 14.599188804626465,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": 1.8569557666778564,
      "logits/rejected": 1.9047002792358398,
      "logps/chosen": -296.90631103515625,
      "logps/ref_chosen": -298.1473388671875,
      "logps/ref_rejected": -269.7539367675781,
      "logps/rejected": -269.1678771972656,
      "loss": 5.5195,
      "margin_dpo/margin_mean": 0.6549609899520874,
      "margin_dpo/margin_std": 1.9896756410598755,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.122955322265625,
      "KL/mean": 0.8335964679718018,
      "KL/rejected_KL_mean": 0.5442428588867188,
      "KL/std": 1.3406589031219482,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5787116885185242,
      "fcm_dpo/q_t": 0.49855342507362366,
      "grad_norm": 12.769622802734375,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": 1.5937082767486572,
      "logits/rejected": 1.63937246799469,
      "logps/chosen": -215.62704467773438,
      "logps/ref_chosen": -216.74998474121094,
      "logps/ref_rejected": -202.7234649658203,
      "logps/rejected": -202.17921447753906,
      "loss": 5.5224,
      "margin_dpo/margin_mean": 0.5787118077278137,
      "margin_dpo/margin_std": 1.814623475074768,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 1.2741985321044922,
      "KL/mean": 0.7556896209716797,
      "KL/rejected_KL_mean": 0.23717880249023438,
      "KL/std": 1.8064461946487427,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.037016749382019,
      "fcm_dpo/q_t": 0.4974079132080078,
      "grad_norm": 14.412718772888184,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": 1.5860885381698608,
      "logits/rejected": 1.677718162536621,
      "logps/chosen": -242.41720581054688,
      "logps/ref_chosen": -243.69140625,
      "logps/ref_rejected": -234.970947265625,
      "logps/rejected": -234.7337646484375,
      "loss": 5.5044,
      "margin_dpo/margin_mean": 1.0370157957077026,
      "margin_dpo/margin_std": 2.2169723510742188,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 1.7492332458496094,
      "KL/mean": 1.028888463973999,
      "KL/rejected_KL_mean": 0.30855560302734375,
      "KL/std": 1.720902442932129,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4406651258468628,
      "fcm_dpo/q_t": 0.4963994026184082,
      "grad_norm": 17.54824447631836,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": 1.994687795639038,
      "logits/rejected": 1.8904571533203125,
      "logps/chosen": -308.9970703125,
      "logps/ref_chosen": -310.746337890625,
      "logps/ref_rejected": -270.6407165527344,
      "logps/rejected": -270.3321533203125,
      "loss": 5.4883,
      "margin_dpo/margin_mean": 1.4406648874282837,
      "margin_dpo/margin_std": 2.1863250732421875,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 1.3561058044433594,
      "KL/mean": 0.8420171737670898,
      "KL/rejected_KL_mean": 0.32793426513671875,
      "KL/std": 1.6369813680648804,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0281695127487183,
      "fcm_dpo/q_t": 0.49742981791496277,
      "grad_norm": 14.890371322631836,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": 1.6783607006072998,
      "logits/rejected": 1.660950779914856,
      "logps/chosen": -257.4568786621094,
      "logps/ref_chosen": -258.8130187988281,
      "logps/ref_rejected": -268.2101745605469,
      "logps/rejected": -267.88226318359375,
      "loss": 5.5046,
      "margin_dpo/margin_mean": 1.028170108795166,
      "margin_dpo/margin_std": 1.9679595232009888,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 1.4668464660644531,
      "KL/mean": 0.994315505027771,
      "KL/rejected_KL_mean": 0.52178955078125,
      "KL/std": 1.9703882932662964,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.945061981678009,
      "fcm_dpo/q_t": 0.4976379871368408,
      "grad_norm": 15.225805282592773,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": 1.6447336673736572,
      "logits/rejected": 1.5671443939208984,
      "logps/chosen": -275.77459716796875,
      "logps/ref_chosen": -277.2414245605469,
      "logps/ref_rejected": -259.13519287109375,
      "logps/rejected": -258.6134033203125,
      "loss": 5.5082,
      "margin_dpo/margin_mean": 0.9450607895851135,
      "margin_dpo/margin_std": 2.569849729537964,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 1.5412158966064453,
      "KL/mean": 0.9015890955924988,
      "KL/rejected_KL_mean": 0.26197242736816406,
      "KL/std": 1.6752173900604248,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2792445421218872,
      "fcm_dpo/q_t": 0.4968023896217346,
      "grad_norm": 14.698164939880371,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.8337934017181396,
      "logits/rejected": 1.8755024671554565,
      "logps/chosen": -249.80465698242188,
      "logps/ref_chosen": -251.34585571289062,
      "logps/ref_rejected": -251.4120635986328,
      "logps/rejected": -251.15008544921875,
      "loss": 5.4947,
      "margin_dpo/margin_mean": 1.2792444229125977,
      "margin_dpo/margin_std": 2.2290306091308594,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 1.707132339477539,
      "KL/mean": 1.1009700298309326,
      "KL/rejected_KL_mean": 0.4948081970214844,
      "KL/std": 1.8404216766357422,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2123188972473145,
      "fcm_dpo/q_t": 0.49696996808052063,
      "grad_norm": 15.042752265930176,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 1.9101613759994507,
      "logits/rejected": 1.8760406970977783,
      "logps/chosen": -267.27545166015625,
      "logps/ref_chosen": -268.98260498046875,
      "logps/ref_rejected": -249.34474182128906,
      "logps/rejected": -248.84996032714844,
      "loss": 5.4975,
      "margin_dpo/margin_mean": 1.2123186588287354,
      "margin_dpo/margin_std": 2.530561923980713,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 2.065876007080078,
      "KL/mean": 1.4827386140823364,
      "KL/rejected_KL_mean": 0.8996028900146484,
      "KL/std": 2.0084240436553955,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1662698984146118,
      "fcm_dpo/q_t": 0.4970850348472595,
      "grad_norm": 14.649291038513184,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 1.9618442058563232,
      "logits/rejected": 1.9401826858520508,
      "logps/chosen": -262.13763427734375,
      "logps/ref_chosen": -264.2034912109375,
      "logps/ref_rejected": -260.2911376953125,
      "logps/rejected": -259.39154052734375,
      "loss": 5.4994,
      "margin_dpo/margin_mean": 1.1662683486938477,
      "margin_dpo/margin_std": 2.6174564361572266,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 1.9210033416748047,
      "KL/mean": 1.2686817646026611,
      "KL/rejected_KL_mean": 0.6163482666015625,
      "KL/std": 2.0734527111053467,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3046634197235107,
      "fcm_dpo/q_t": 0.4967391788959503,
      "grad_norm": 14.783002853393555,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": 1.7934911251068115,
      "logits/rejected": 1.8195244073867798,
      "logps/chosen": -269.5472717285156,
      "logps/ref_chosen": -271.4682922363281,
      "logps/ref_rejected": -273.0804748535156,
      "logps/rejected": -272.4641418457031,
      "loss": 5.494,
      "margin_dpo/margin_mean": 1.3046636581420898,
      "margin_dpo/margin_std": 2.7045812606811523,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 2.450763702392578,
      "KL/mean": 1.5988849401474,
      "KL/rejected_KL_mean": 0.7470073699951172,
      "KL/std": 2.3729257583618164,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7037557363510132,
      "fcm_dpo/q_t": 0.49574196338653564,
      "grad_norm": 15.823236465454102,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": 1.8489965200424194,
      "logits/rejected": 1.8566608428955078,
      "logps/chosen": -271.1526184082031,
      "logps/ref_chosen": -273.6033935546875,
      "logps/ref_rejected": -259.54107666015625,
      "logps/rejected": -258.7940673828125,
      "loss": 5.4783,
      "margin_dpo/margin_mean": 1.703755259513855,
      "margin_dpo/margin_std": 2.9707090854644775,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 2.3244876861572266,
      "KL/mean": 1.7412693500518799,
      "KL/rejected_KL_mean": 1.1580543518066406,
      "KL/std": 2.1126232147216797,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1664327383041382,
      "fcm_dpo/q_t": 0.497084379196167,
      "grad_norm": 14.506269454956055,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": 1.7756383419036865,
      "logits/rejected": 1.875848412513733,
      "logps/chosen": -256.19317626953125,
      "logps/ref_chosen": -258.5176696777344,
      "logps/ref_rejected": -264.9944763183594,
      "logps/rejected": -263.83642578125,
      "loss": 5.4995,
      "margin_dpo/margin_mean": 1.1664323806762695,
      "margin_dpo/margin_std": 2.728050947189331,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 2.1610488891601562,
      "KL/mean": 1.501230239868164,
      "KL/rejected_KL_mean": 0.8414096832275391,
      "KL/std": 2.732363700866699,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.31963050365448,
      "fcm_dpo/q_t": 0.4967021048069,
      "grad_norm": 14.964388847351074,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": 2.03570294380188,
      "logits/rejected": 2.1434881687164307,
      "logps/chosen": -265.3483581542969,
      "logps/ref_chosen": -267.5094299316406,
      "logps/ref_rejected": -277.03228759765625,
      "logps/rejected": -276.19085693359375,
      "loss": 5.494,
      "margin_dpo/margin_mean": 1.3196306228637695,
      "margin_dpo/margin_std": 3.70042085647583,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 2.6605148315429688,
      "KL/mean": 2.0537405014038086,
      "KL/rejected_KL_mean": 1.44696044921875,
      "KL/std": 2.591015338897705,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2135499715805054,
      "fcm_dpo/q_t": 0.4969669580459595,
      "grad_norm": 14.236626625061035,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": 1.8384249210357666,
      "logits/rejected": 1.8645050525665283,
      "logps/chosen": -254.80441284179688,
      "logps/ref_chosen": -257.4649353027344,
      "logps/ref_rejected": -247.29161071777344,
      "logps/rejected": -245.8446502685547,
      "loss": 5.498,
      "margin_dpo/margin_mean": 1.2135504484176636,
      "margin_dpo/margin_std": 3.3108415603637695,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 2.535602569580078,
      "KL/mean": 1.720833659172058,
      "KL/rejected_KL_mean": 0.9060745239257812,
      "KL/std": 2.8693718910217285,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6295247077941895,
      "fcm_dpo/q_t": 0.49592864513397217,
      "grad_norm": 14.798855781555176,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": 1.879491925239563,
      "logits/rejected": 1.8447827100753784,
      "logps/chosen": -287.94854736328125,
      "logps/ref_chosen": -290.4841613769531,
      "logps/ref_rejected": -260.8398132324219,
      "logps/rejected": -259.9337463378906,
      "loss": 5.4818,
      "margin_dpo/margin_mean": 1.6295251846313477,
      "margin_dpo/margin_std": 3.7866528034210205,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 2.7788352966308594,
      "KL/mean": 1.7287815809249878,
      "KL/rejected_KL_mean": 0.6787204742431641,
      "KL/std": 3.004427194595337,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.100127696990967,
      "fcm_dpo/q_t": 0.49475181102752686,
      "grad_norm": 14.990151405334473,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": 1.7665235996246338,
      "logits/rejected": 1.7945448160171509,
      "logps/chosen": -254.85873413085938,
      "logps/ref_chosen": -257.6375732421875,
      "logps/ref_rejected": -235.6666717529297,
      "logps/rejected": -234.98794555664062,
      "loss": 5.4631,
      "margin_dpo/margin_mean": 2.1001274585723877,
      "margin_dpo/margin_std": 3.755279302597046,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 3.3880062103271484,
      "KL/mean": 2.342463254928589,
      "KL/rejected_KL_mean": 1.2969188690185547,
      "KL/std": 3.084852933883667,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.09108829498291,
      "fcm_dpo/q_t": 0.4947749376296997,
      "grad_norm": 15.591358184814453,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.8355705738067627,
      "logits/rejected": 1.8072062730789185,
      "logps/chosen": -284.0771789550781,
      "logps/ref_chosen": -287.4651794433594,
      "logps/ref_rejected": -253.4770965576172,
      "logps/rejected": -252.18016052246094,
      "loss": 5.4636,
      "margin_dpo/margin_mean": 2.09108829498291,
      "margin_dpo/margin_std": 3.8038573265075684,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 3.5444717407226562,
      "KL/mean": 2.5267648696899414,
      "KL/rejected_KL_mean": 1.5090618133544922,
      "KL/std": 3.2750277519226074,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.035414218902588,
      "fcm_dpo/q_t": 0.4949134886264801,
      "grad_norm": 14.950457572937012,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 1.9215449094772339,
      "logits/rejected": 1.9261125326156616,
      "logps/chosen": -277.39093017578125,
      "logps/ref_chosen": -280.9353942871094,
      "logps/ref_rejected": -275.998779296875,
      "logps/rejected": -274.48974609375,
      "loss": 5.4662,
      "margin_dpo/margin_mean": 2.0354151725769043,
      "margin_dpo/margin_std": 4.2240309715271,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 3.0440025329589844,
      "KL/mean": 2.009183883666992,
      "KL/rejected_KL_mean": 0.9743671417236328,
      "KL/std": 3.0175909996032715,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.069643020629883,
      "fcm_dpo/q_t": 0.494828462600708,
      "grad_norm": 14.556612968444824,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": 1.6940979957580566,
      "logits/rejected": 1.8324360847473145,
      "logps/chosen": -227.1015167236328,
      "logps/ref_chosen": -230.14553833007812,
      "logps/ref_rejected": -253.37661743164062,
      "logps/rejected": -252.40225219726562,
      "loss": 5.4645,
      "margin_dpo/margin_mean": 2.0696425437927246,
      "margin_dpo/margin_std": 3.891420841217041,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 3.796670913696289,
      "KL/mean": 2.7970566749572754,
      "KL/rejected_KL_mean": 1.797445297241211,
      "KL/std": 3.184476613998413,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9992173910140991,
      "fcm_dpo/q_t": 0.49500417709350586,
      "grad_norm": 14.884024620056152,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": 1.56602942943573,
      "logits/rejected": 1.5778162479400635,
      "logps/chosen": -261.0696105957031,
      "logps/ref_chosen": -264.86627197265625,
      "logps/ref_rejected": -245.21304321289062,
      "logps/rejected": -243.4155731201172,
      "loss": 5.4676,
      "margin_dpo/margin_mean": 1.9992163181304932,
      "margin_dpo/margin_std": 4.277068138122559,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.7560997009277344,
      "KL/mean": 2.791999101638794,
      "KL/rejected_KL_mean": 1.8279132843017578,
      "KL/std": 3.454188346862793,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9281823635101318,
      "fcm_dpo/q_t": 0.4951818585395813,
      "grad_norm": 15.709370613098145,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": 1.749987244606018,
      "logits/rejected": 1.802310585975647,
      "logps/chosen": -330.79742431640625,
      "logps/ref_chosen": -334.55352783203125,
      "logps/ref_rejected": -325.8403015136719,
      "logps/rejected": -324.01239013671875,
      "loss": 5.4706,
      "margin_dpo/margin_mean": 1.9281835556030273,
      "margin_dpo/margin_std": 4.405967712402344,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.523866653442383,
      "KL/mean": 3.2062339782714844,
      "KL/rejected_KL_mean": 1.8886032104492188,
      "KL/std": 3.594428062438965,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.6352691650390625,
      "fcm_dpo/q_t": 0.49341535568237305,
      "grad_norm": 15.226400375366211,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 1.6022204160690308,
      "logits/rejected": 1.6788519620895386,
      "logps/chosen": -255.8979949951172,
      "logps/ref_chosen": -260.421875,
      "logps/ref_rejected": -273.0882263183594,
      "logps/rejected": -271.1996154785156,
      "loss": 5.4425,
      "margin_dpo/margin_mean": 2.635268211364746,
      "margin_dpo/margin_std": 4.223102569580078,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.604757308959961,
      "KL/mean": 3.109954833984375,
      "KL/rejected_KL_mean": 1.6151580810546875,
      "KL/std": 3.9162797927856445,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9896059036254883,
      "fcm_dpo/q_t": 0.49253350496292114,
      "grad_norm": 16.75493812561035,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": 1.8619482517242432,
      "logits/rejected": 1.8518846035003662,
      "logps/chosen": -267.13067626953125,
      "logps/ref_chosen": -271.7354431152344,
      "logps/ref_rejected": -262.64111328125,
      "logps/rejected": -261.0259704589844,
      "loss": 5.4291,
      "margin_dpo/margin_mean": 2.98960542678833,
      "margin_dpo/margin_std": 4.859260082244873,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 4.920551300048828,
      "KL/mean": 3.7005298137664795,
      "KL/rejected_KL_mean": 2.480520248413086,
      "KL/std": 4.023486137390137,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.4400296211242676,
      "fcm_dpo/q_t": 0.49390551447868347,
      "grad_norm": 14.851919174194336,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": 1.7134696245193481,
      "logits/rejected": 1.5521215200424194,
      "logps/chosen": -289.75006103515625,
      "logps/ref_chosen": -294.6706237792969,
      "logps/ref_rejected": -247.6900177001953,
      "logps/rejected": -245.2095184326172,
      "loss": 5.4509,
      "margin_dpo/margin_mean": 2.4400293827056885,
      "margin_dpo/margin_std": 4.974210739135742,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 5.12578010559082,
      "KL/mean": 3.707528591156006,
      "KL/rejected_KL_mean": 2.289274215698242,
      "KL/std": 3.628737449645996,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.8365039825439453,
      "fcm_dpo/q_t": 0.49291568994522095,
      "grad_norm": 14.641070365905762,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": 1.5785304307937622,
      "logits/rejected": 1.5019304752349854,
      "logps/chosen": -243.02651977539062,
      "logps/ref_chosen": -248.1522979736328,
      "logps/ref_rejected": -209.12808227539062,
      "logps/rejected": -206.8387908935547,
      "loss": 5.4353,
      "margin_dpo/margin_mean": 2.836503505706787,
      "margin_dpo/margin_std": 5.110018730163574,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 5.093940734863281,
      "KL/mean": 3.563575267791748,
      "KL/rejected_KL_mean": 2.033205032348633,
      "KL/std": 4.87030553817749,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0607359409332275,
      "fcm_dpo/q_t": 0.49234941601753235,
      "grad_norm": 16.732343673706055,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": 2.003129720687866,
      "logits/rejected": 2.0792410373687744,
      "logps/chosen": -254.38043212890625,
      "logps/ref_chosen": -259.4743957519531,
      "logps/ref_rejected": -246.9627227783203,
      "logps/rejected": -244.92953491210938,
      "loss": 5.4285,
      "margin_dpo/margin_mean": 3.0607354640960693,
      "margin_dpo/margin_std": 6.407423496246338,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 5.280099868774414,
      "KL/mean": 3.988632917404175,
      "KL/rejected_KL_mean": 2.697164535522461,
      "KL/std": 4.7499775886535645,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.5829391479492188,
      "fcm_dpo/q_t": 0.49354925751686096,
      "grad_norm": 15.852944374084473,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.7865550518035889,
      "logits/rejected": 1.783797264099121,
      "logps/chosen": -280.6464538574219,
      "logps/ref_chosen": -285.926513671875,
      "logps/ref_rejected": -253.77996826171875,
      "logps/rejected": -251.08282470703125,
      "loss": 5.4464,
      "margin_dpo/margin_mean": 2.582939624786377,
      "margin_dpo/margin_std": 5.730757236480713,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 5.985637664794922,
      "KL/mean": 4.573917865753174,
      "KL/rejected_KL_mean": 3.1621932983398438,
      "KL/std": 4.4777302742004395,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.82344651222229,
      "fcm_dpo/q_t": 0.4929467737674713,
      "grad_norm": 14.74967098236084,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 1.7299965620040894,
      "logits/rejected": 1.6685047149658203,
      "logps/chosen": -256.60858154296875,
      "logps/ref_chosen": -262.5942077636719,
      "logps/ref_rejected": -236.687744140625,
      "logps/rejected": -233.52554321289062,
      "loss": 5.4374,
      "margin_dpo/margin_mean": 2.8234472274780273,
      "margin_dpo/margin_std": 6.35840368270874,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 6.382469177246094,
      "KL/mean": 4.828726768493652,
      "KL/rejected_KL_mean": 3.2749710083007812,
      "KL/std": 4.395574569702148,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.1074984073638916,
      "fcm_dpo/q_t": 0.4922400116920471,
      "grad_norm": 17.200931549072266,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": 1.7887407541275024,
      "logits/rejected": 1.7388211488723755,
      "logps/chosen": -264.66070556640625,
      "logps/ref_chosen": -271.04315185546875,
      "logps/ref_rejected": -234.03359985351562,
      "logps/rejected": -230.75860595703125,
      "loss": 5.4255,
      "margin_dpo/margin_mean": 3.107499361038208,
      "margin_dpo/margin_std": 5.578749656677246,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 6.597845077514648,
      "KL/mean": 4.841414451599121,
      "KL/rejected_KL_mean": 3.084989547729492,
      "KL/std": 4.915842056274414,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.5128579139709473,
      "fcm_dpo/q_t": 0.4912325143814087,
      "grad_norm": 15.54843807220459,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": 1.9178483486175537,
      "logits/rejected": 2.006883144378662,
      "logps/chosen": -259.68743896484375,
      "logps/ref_chosen": -266.2852783203125,
      "logps/ref_rejected": -274.32501220703125,
      "logps/rejected": -271.2400207519531,
      "loss": 5.4107,
      "margin_dpo/margin_mean": 3.5128591060638428,
      "margin_dpo/margin_std": 6.553744316101074,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 6.786888122558594,
      "KL/mean": 5.391615390777588,
      "KL/rejected_KL_mean": 3.9963512420654297,
      "KL/std": 5.142050266265869,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.790539264678955,
      "fcm_dpo/q_t": 0.49302875995635986,
      "grad_norm": 15.192692756652832,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": 1.7257513999938965,
      "logits/rejected": 1.708960771560669,
      "logps/chosen": -279.99371337890625,
      "logps/ref_chosen": -286.7806091308594,
      "logps/ref_rejected": -276.1176452636719,
      "logps/rejected": -272.1213073730469,
      "loss": 5.4393,
      "margin_dpo/margin_mean": 2.7905383110046387,
      "margin_dpo/margin_std": 6.708759784698486,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": 6.488300323486328,
      "KL/mean": 5.497576713562012,
      "KL/rejected_KL_mean": 4.506855010986328,
      "KL/std": 4.393231391906738,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9814454317092896,
      "fcm_dpo/q_t": 0.49505019187927246,
      "grad_norm": 13.877166748046875,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": 1.9946916103363037,
      "logits/rejected": 2.0954713821411133,
      "logps/chosen": -228.5415802001953,
      "logps/ref_chosen": -235.02987670898438,
      "logps/ref_rejected": -259.80999755859375,
      "logps/rejected": -255.30316162109375,
      "loss": 5.4696,
      "margin_dpo/margin_mean": 1.981445550918579,
      "margin_dpo/margin_std": 5.376645565032959,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 7.92442512512207,
      "KL/mean": 6.142773151397705,
      "KL/rejected_KL_mean": 4.361135482788086,
      "KL/std": 5.48183536529541,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.5632965564727783,
      "fcm_dpo/q_t": 0.491105318069458,
      "grad_norm": 16.144277572631836,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": 1.8883984088897705,
      "logits/rejected": 1.933871865272522,
      "logps/chosen": -293.3883056640625,
      "logps/ref_chosen": -301.312744140625,
      "logps/ref_rejected": -288.8368225097656,
      "logps/rejected": -284.4756774902344,
      "loss": 5.4097,
      "margin_dpo/margin_mean": 3.563295841217041,
      "margin_dpo/margin_std": 7.410616874694824,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": 8.27791976928711,
      "KL/mean": 6.196424961090088,
      "KL/rejected_KL_mean": 4.114938735961914,
      "KL/std": 5.512639999389648,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.162985801696777,
      "fcm_dpo/q_t": 0.48960787057876587,
      "grad_norm": 15.286627769470215,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": 1.6895256042480469,
      "logits/rejected": 1.6734871864318848,
      "logps/chosen": -257.3880615234375,
      "logps/ref_chosen": -265.6659851074219,
      "logps/ref_rejected": -242.0697479248047,
      "logps/rejected": -237.95480346679688,
      "loss": 5.3849,
      "margin_dpo/margin_mean": 4.162985801696777,
      "margin_dpo/margin_std": 6.44715690612793,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": 7.574924468994141,
      "KL/mean": 5.938198566436768,
      "KL/rejected_KL_mean": 4.301469802856445,
      "KL/std": 5.545859336853027,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.273448944091797,
      "fcm_dpo/q_t": 0.49182775616645813,
      "grad_norm": 14.050113677978516,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 1.7135064601898193,
      "logits/rejected": 1.728864312171936,
      "logps/chosen": -253.1465301513672,
      "logps/ref_chosen": -260.721435546875,
      "logps/ref_rejected": -262.6593322753906,
      "logps/rejected": -258.35784912109375,
      "loss": 5.4208,
      "margin_dpo/margin_mean": 3.2734503746032715,
      "margin_dpo/margin_std": 7.1718974113464355,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": 9.115272521972656,
      "KL/mean": 7.178289890289307,
      "KL/rejected_KL_mean": 5.241308212280273,
      "KL/std": 5.200367450714111,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.873972177505493,
      "fcm_dpo/q_t": 0.49032819271087646,
      "grad_norm": 15.90714168548584,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": 1.795597791671753,
      "logits/rejected": 1.759089469909668,
      "logps/chosen": -247.88755798339844,
      "logps/ref_chosen": -257.0028076171875,
      "logps/ref_rejected": -254.66232299804688,
      "logps/rejected": -249.42103576660156,
      "loss": 5.396,
      "margin_dpo/margin_mean": 3.873971700668335,
      "margin_dpo/margin_std": 6.136815071105957,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": 8.98997688293457,
      "KL/mean": 7.32076358795166,
      "KL/rejected_KL_mean": 5.651544570922852,
      "KL/std": 5.607697486877441,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.338440179824829,
      "fcm_dpo/q_t": 0.49166616797447205,
      "grad_norm": 15.288473129272461,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 1.9387388229370117,
      "logits/rejected": 1.8767954111099243,
      "logps/chosen": -289.4204406738281,
      "logps/ref_chosen": -298.4103698730469,
      "logps/ref_rejected": -262.8761901855469,
      "logps/rejected": -257.2246398925781,
      "loss": 5.418,
      "margin_dpo/margin_mean": 3.3384389877319336,
      "margin_dpo/margin_std": 6.747113227844238,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": 8.887788772583008,
      "KL/mean": 7.194285869598389,
      "KL/rejected_KL_mean": 5.500776290893555,
      "KL/std": 5.466890811920166,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.3870155811309814,
      "fcm_dpo/q_t": 0.49155035614967346,
      "grad_norm": 14.715645790100098,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 1.7785838842391968,
      "logits/rejected": 1.8333340883255005,
      "logps/chosen": -218.63609313964844,
      "logps/ref_chosen": -227.5238800048828,
      "logps/ref_rejected": -227.11428833007812,
      "logps/rejected": -221.61351013183594,
      "loss": 5.4167,
      "margin_dpo/margin_mean": 3.3870153427124023,
      "margin_dpo/margin_std": 7.3266754150390625,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": 8.979122161865234,
      "KL/mean": 7.106125831604004,
      "KL/rejected_KL_mean": 5.233119964599609,
      "KL/std": 6.647099494934082,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.746001958847046,
      "fcm_dpo/q_t": 0.4906472861766815,
      "grad_norm": 15.545022964477539,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": 1.9102656841278076,
      "logits/rejected": 1.8167051076889038,
      "logps/chosen": -266.4031066894531,
      "logps/ref_chosen": -275.3822326660156,
      "logps/ref_rejected": -269.8356018066406,
      "logps/rejected": -264.60247802734375,
      "loss": 5.4032,
      "margin_dpo/margin_mean": 3.746001958847046,
      "margin_dpo/margin_std": 7.727083206176758,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": 9.251794815063477,
      "KL/mean": 7.449893951416016,
      "KL/rejected_KL_mean": 5.647991180419922,
      "KL/std": 5.758368015289307,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6038031578063965,
      "fcm_dpo/q_t": 0.49100762605667114,
      "grad_norm": 14.60382080078125,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": 1.7598062753677368,
      "logits/rejected": 1.7438071966171265,
      "logps/chosen": -218.86459350585938,
      "logps/ref_chosen": -228.1163787841797,
      "logps/ref_rejected": -228.26266479492188,
      "logps/rejected": -222.6146697998047,
      "loss": 5.4079,
      "margin_dpo/margin_mean": 3.6038029193878174,
      "margin_dpo/margin_std": 7.1873779296875,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": 10.07636833190918,
      "KL/mean": 7.996395111083984,
      "KL/rejected_KL_mean": 5.916419982910156,
      "KL/std": 6.6812591552734375,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.159952163696289,
      "fcm_dpo/q_t": 0.4896257221698761,
      "grad_norm": 15.560564994812012,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": 1.780318260192871,
      "logits/rejected": 1.8767807483673096,
      "logps/chosen": -273.95086669921875,
      "logps/ref_chosen": -284.0272521972656,
      "logps/ref_rejected": -282.1720275878906,
      "logps/rejected": -276.255615234375,
      "loss": 5.3887,
      "margin_dpo/margin_mean": 4.159952163696289,
      "margin_dpo/margin_std": 8.65852165222168,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": 9.824239730834961,
      "KL/mean": 7.567661285400391,
      "KL/rejected_KL_mean": 5.311090469360352,
      "KL/std": 6.87279748916626,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.513144016265869,
      "fcm_dpo/q_t": 0.48874539136886597,
      "grad_norm": 15.964912414550781,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": 1.9025814533233643,
      "logits/rejected": 1.8399163484573364,
      "logps/chosen": -293.2255554199219,
      "logps/ref_chosen": -303.0498352050781,
      "logps/ref_rejected": -259.8980712890625,
      "logps/rejected": -254.5869903564453,
      "loss": 5.375,
      "margin_dpo/margin_mean": 4.513142108917236,
      "margin_dpo/margin_std": 8.8762845993042,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": 9.894670486450195,
      "KL/mean": 7.72609281539917,
      "KL/rejected_KL_mean": 5.557516098022461,
      "KL/std": 6.996020793914795,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.337155342102051,
      "fcm_dpo/q_t": 0.48918259143829346,
      "grad_norm": 15.265756607055664,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": 1.8706061840057373,
      "logits/rejected": 1.8334473371505737,
      "logps/chosen": -260.6929931640625,
      "logps/ref_chosen": -270.58770751953125,
      "logps/ref_rejected": -238.90069580078125,
      "logps/rejected": -233.34315490722656,
      "loss": 5.3806,
      "margin_dpo/margin_mean": 4.337155818939209,
      "margin_dpo/margin_std": 7.8739824295043945,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": 9.72779655456543,
      "KL/mean": 7.42535924911499,
      "KL/rejected_KL_mean": 5.122922897338867,
      "KL/std": 6.983025074005127,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.604873180389404,
      "fcm_dpo/q_t": 0.48853224515914917,
      "grad_norm": 15.284632682800293,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": 1.8285350799560547,
      "logits/rejected": 1.71226167678833,
      "logps/chosen": -286.457763671875,
      "logps/ref_chosen": -296.1855773925781,
      "logps/ref_rejected": -260.35333251953125,
      "logps/rejected": -255.23043823242188,
      "loss": 5.3732,
      "margin_dpo/margin_mean": 4.604873180389404,
      "margin_dpo/margin_std": 9.742921829223633,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": 10.049724578857422,
      "KL/mean": 7.636887550354004,
      "KL/rejected_KL_mean": 5.224050521850586,
      "KL/std": 7.415311813354492,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.825664520263672,
      "fcm_dpo/q_t": 0.48797595500946045,
      "grad_norm": 15.730220794677734,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": 1.855591058731079,
      "logits/rejected": 1.8337557315826416,
      "logps/chosen": -267.6656494140625,
      "logps/ref_chosen": -277.7153625488281,
      "logps/ref_rejected": -272.9094543457031,
      "logps/rejected": -267.6853942871094,
      "loss": 5.3653,
      "margin_dpo/margin_mean": 4.8256635665893555,
      "margin_dpo/margin_std": 9.507255554199219,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": 10.241092681884766,
      "KL/mean": 7.69697380065918,
      "KL/rejected_KL_mean": 5.152858734130859,
      "KL/std": 7.618227958679199,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.0882415771484375,
      "fcm_dpo/q_t": 0.4873075485229492,
      "grad_norm": 15.0337553024292,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": 1.7558929920196533,
      "logits/rejected": 1.727840781211853,
      "logps/chosen": -262.2967224121094,
      "logps/ref_chosen": -272.5378112792969,
      "logps/ref_rejected": -244.6329345703125,
      "logps/rejected": -239.4800567626953,
      "loss": 5.3557,
      "margin_dpo/margin_mean": 5.088242530822754,
      "margin_dpo/margin_std": 10.095664978027344,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": 9.302288055419922,
      "KL/mean": 7.0179948806762695,
      "KL/rejected_KL_mean": 4.733715057373047,
      "KL/std": 7.91300106048584,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.568582534790039,
      "fcm_dpo/q_t": 0.4886173605918884,
      "grad_norm": 15.564704895019531,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.6690064668655396,
      "logits/rejected": 1.587247610092163,
      "logps/chosen": -280.51007080078125,
      "logps/ref_chosen": -289.8123474121094,
      "logps/ref_rejected": -274.29071044921875,
      "logps/rejected": -269.5569763183594,
      "loss": 5.3743,
      "margin_dpo/margin_mean": 4.568582534790039,
      "margin_dpo/margin_std": 9.26849365234375,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": 9.943389892578125,
      "KL/mean": 7.645666122436523,
      "KL/rejected_KL_mean": 5.347942352294922,
      "KL/std": 8.10798168182373,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.595434665679932,
      "fcm_dpo/q_t": 0.48854488134384155,
      "grad_norm": 15.391023635864258,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 1.763291835784912,
      "logits/rejected": 1.9354881048202515,
      "logps/chosen": -235.7697296142578,
      "logps/ref_chosen": -245.71311950683594,
      "logps/ref_rejected": -272.08935546875,
      "logps/rejected": -266.7414245605469,
      "loss": 5.3743,
      "margin_dpo/margin_mean": 4.595433712005615,
      "margin_dpo/margin_std": 9.793839454650879,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": 10.273580551147461,
      "KL/mean": 7.057726860046387,
      "KL/rejected_KL_mean": 3.8418731689453125,
      "KL/std": 7.585107803344727,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.4317097663879395,
      "fcm_dpo/q_t": 0.48396462202072144,
      "grad_norm": 15.214155197143555,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": 1.5149221420288086,
      "logits/rejected": 1.6004772186279297,
      "logps/chosen": -199.6688232421875,
      "logps/ref_chosen": -209.94239807128906,
      "logps/ref_rejected": -230.66897583007812,
      "logps/rejected": -226.82708740234375,
      "loss": 5.3012,
      "margin_dpo/margin_mean": 6.4317097663879395,
      "margin_dpo/margin_std": 8.954239845275879,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": 9.342567443847656,
      "KL/mean": 6.908594131469727,
      "KL/rejected_KL_mean": 4.474617004394531,
      "KL/std": 8.4212646484375,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.867946624755859,
      "fcm_dpo/q_t": 0.4878772795200348,
      "grad_norm": 16.631824493408203,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": 1.8337438106536865,
      "logits/rejected": 1.9308557510375977,
      "logps/chosen": -241.9570770263672,
      "logps/ref_chosen": -251.2996368408203,
      "logps/ref_rejected": -277.4847412109375,
      "logps/rejected": -273.0101318359375,
      "loss": 5.3646,
      "margin_dpo/margin_mean": 4.867948532104492,
      "margin_dpo/margin_std": 10.476612091064453,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": 8.246908187866211,
      "KL/mean": 6.065648555755615,
      "KL/rejected_KL_mean": 3.8843860626220703,
      "KL/std": 8.72115707397461,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.362521648406982,
      "fcm_dpo/q_t": 0.48914027214050293,
      "grad_norm": 14.311569213867188,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": 1.6857103109359741,
      "logits/rejected": 1.717315673828125,
      "logps/chosen": -249.53250122070312,
      "logps/ref_chosen": -257.7794189453125,
      "logps/ref_rejected": -227.91546630859375,
      "logps/rejected": -224.0310821533203,
      "loss": 5.3864,
      "margin_dpo/margin_mean": 4.362521648406982,
      "margin_dpo/margin_std": 11.369268417358398,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": 10.248620986938477,
      "KL/mean": 6.143117427825928,
      "KL/rejected_KL_mean": 2.037616729736328,
      "KL/std": 9.7485933303833,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.211008071899414,
      "fcm_dpo/q_t": 0.47954505681991577,
      "grad_norm": 17.840850830078125,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": 1.8722895383834839,
      "logits/rejected": 1.850420355796814,
      "logps/chosen": -301.8454284667969,
      "logps/ref_chosen": -312.09405517578125,
      "logps/ref_rejected": -277.97332763671875,
      "logps/rejected": -275.9356994628906,
      "loss": 5.2363,
      "margin_dpo/margin_mean": 8.211007118225098,
      "margin_dpo/margin_std": 10.749405860900879,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": 6.650535583496094,
      "KL/mean": 3.927499532699585,
      "KL/rejected_KL_mean": 1.2044544219970703,
      "KL/std": 8.97716999053955,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.446088790893555,
      "fcm_dpo/q_t": 0.48644259572029114,
      "grad_norm": 16.503108978271484,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": 1.7998017072677612,
      "logits/rejected": 1.8056597709655762,
      "logps/chosen": -278.7158203125,
      "logps/ref_chosen": -285.3663330078125,
      "logps/ref_rejected": -262.2779541015625,
      "logps/rejected": -261.0734558105469,
      "loss": 5.344,
      "margin_dpo/margin_mean": 5.4460906982421875,
      "margin_dpo/margin_std": 11.151394844055176,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": 9.359670639038086,
      "KL/mean": 5.673666954040527,
      "KL/rejected_KL_mean": 1.987661361694336,
      "KL/std": 9.65538215637207,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.372006416320801,
      "fcm_dpo/q_t": 0.48162493109703064,
      "grad_norm": 16.516834259033203,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": 1.7575511932373047,
      "logits/rejected": 1.8232022523880005,
      "logps/chosen": -286.93389892578125,
      "logps/ref_chosen": -296.2935791015625,
      "logps/ref_rejected": -291.5162353515625,
      "logps/rejected": -289.528564453125,
      "loss": 5.2713,
      "margin_dpo/margin_mean": 7.372007369995117,
      "margin_dpo/margin_std": 11.778236389160156,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": 9.446426391601562,
      "KL/mean": 5.337229251861572,
      "KL/rejected_KL_mean": 1.228036880493164,
      "KL/std": 9.054306030273438,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.218381881713867,
      "fcm_dpo/q_t": 0.4795321822166443,
      "grad_norm": 16.106088638305664,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": 2.054405689239502,
      "logits/rejected": 1.9370282888412476,
      "logps/chosen": -278.67791748046875,
      "logps/ref_chosen": -288.12432861328125,
      "logps/ref_rejected": -232.35720825195312,
      "logps/rejected": -231.12916564941406,
      "loss": 5.2354,
      "margin_dpo/margin_mean": 8.218379974365234,
      "margin_dpo/margin_std": 10.675679206848145,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": 8.082271575927734,
      "KL/mean": 3.9147870540618896,
      "KL/rejected_KL_mean": -0.2526988983154297,
      "KL/std": 10.182801246643066,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.334965705871582,
      "fcm_dpo/q_t": 0.47929707169532776,
      "grad_norm": 17.871719360351562,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": 1.7408560514450073,
      "logits/rejected": 1.6947420835494995,
      "logps/chosen": -309.1192932128906,
      "logps/ref_chosen": -317.2015686035156,
      "logps/ref_rejected": -278.0653381347656,
      "logps/rejected": -278.3180236816406,
      "loss": 5.2373,
      "margin_dpo/margin_mean": 8.334964752197266,
      "margin_dpo/margin_std": 12.99758529663086,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": 8.691797256469727,
      "KL/mean": 4.6519365310668945,
      "KL/rejected_KL_mean": 0.6120891571044922,
      "KL/std": 10.257315635681152,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.079712867736816,
      "fcm_dpo/q_t": 0.4798974394798279,
      "grad_norm": 16.03093147277832,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.5886785984039307,
      "logits/rejected": 1.565246820449829,
      "logps/chosen": -254.24766540527344,
      "logps/ref_chosen": -262.939453125,
      "logps/ref_rejected": -257.82275390625,
      "logps/rejected": -257.2106628417969,
      "loss": 5.247,
      "margin_dpo/margin_mean": 8.0797119140625,
      "margin_dpo/margin_std": 13.254253387451172,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": 9.064720153808594,
      "KL/mean": 4.677756309509277,
      "KL/rejected_KL_mean": 0.29079437255859375,
      "KL/std": 11.386920928955078,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.773923873901367,
      "fcm_dpo/q_t": 0.4782067835330963,
      "grad_norm": 18.45173454284668,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 1.8416138887405396,
      "logits/rejected": 1.7450803518295288,
      "logps/chosen": -310.089111328125,
      "logps/ref_chosen": -319.15380859375,
      "logps/ref_rejected": -281.7733154296875,
      "logps/rejected": -281.4825134277344,
      "loss": 5.2208,
      "margin_dpo/margin_mean": 8.773923873901367,
      "margin_dpo/margin_std": 12.990989685058594,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": 5.35273551940918,
      "KL/mean": 3.171708345413208,
      "KL/rejected_KL_mean": 0.9906806945800781,
      "KL/std": 10.60285758972168,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.362063407897949,
      "fcm_dpo/q_t": 0.4891389012336731,
      "grad_norm": 14.564783096313477,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": 1.6844780445098877,
      "logits/rejected": 1.7813938856124878,
      "logps/chosen": -245.14768981933594,
      "logps/ref_chosen": -250.50042724609375,
      "logps/ref_rejected": -268.32635498046875,
      "logps/rejected": -267.3356628417969,
      "loss": 5.3962,
      "margin_dpo/margin_mean": 4.362063407897949,
      "margin_dpo/margin_std": 14.239696502685547,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": 6.6666412353515625,
      "KL/mean": 1.5873042345046997,
      "KL/rejected_KL_mean": -3.4920272827148438,
      "KL/std": 11.767230987548828,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.158655166625977,
      "fcm_dpo/q_t": 0.47478657960891724,
      "grad_norm": 16.894495010375977,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": 1.626943826675415,
      "logits/rejected": 1.6659835577011108,
      "logps/chosen": -272.7349548339844,
      "logps/ref_chosen": -279.4015808105469,
      "logps/ref_rejected": -266.6392822265625,
      "logps/rejected": -270.1313171386719,
      "loss": 5.1717,
      "margin_dpo/margin_mean": 10.15865707397461,
      "margin_dpo/margin_std": 14.37594223022461,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": 5.041717529296875,
      "KL/mean": 2.479550838470459,
      "KL/rejected_KL_mean": -0.08260726928710938,
      "KL/std": 10.766268730163574,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.124326705932617,
      "fcm_dpo/q_t": 0.48726481199264526,
      "grad_norm": 15.689340591430664,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 1.946668267250061,
      "logits/rejected": 1.9447931051254272,
      "logps/chosen": -249.21878051757812,
      "logps/ref_chosen": -254.26048278808594,
      "logps/ref_rejected": -241.8769989013672,
      "logps/rejected": -241.9595947265625,
      "loss": 5.362,
      "margin_dpo/margin_mean": 5.124327182769775,
      "margin_dpo/margin_std": 13.138858795166016,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": 5.508440017700195,
      "KL/mean": 2.5763766765594482,
      "KL/rejected_KL_mean": -0.3556995391845703,
      "KL/std": 11.982223510742188,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.864138603210449,
      "fcm_dpo/q_t": 0.4854711890220642,
      "grad_norm": 16.179004669189453,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": 1.6016563177108765,
      "logits/rejected": 1.6319979429244995,
      "logps/chosen": -283.29608154296875,
      "logps/ref_chosen": -288.80450439453125,
      "logps/ref_rejected": -266.8573913574219,
      "logps/rejected": -267.21307373046875,
      "loss": 5.3377,
      "margin_dpo/margin_mean": 5.864140510559082,
      "margin_dpo/margin_std": 14.577569961547852,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": 6.813718795776367,
      "KL/mean": 2.7296156883239746,
      "KL/rejected_KL_mean": -1.3544845581054688,
      "KL/std": 11.229461669921875,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.168201446533203,
      "fcm_dpo/q_t": 0.47970157861709595,
      "grad_norm": 16.345657348632812,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": 1.694004774093628,
      "logits/rejected": 1.6720221042633057,
      "logps/chosen": -265.376220703125,
      "logps/ref_chosen": -272.18994140625,
      "logps/ref_rejected": -253.14334106445312,
      "logps/rejected": -254.497802734375,
      "loss": 5.2466,
      "margin_dpo/margin_mean": 8.168201446533203,
      "margin_dpo/margin_std": 14.164774894714355,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": 5.529722213745117,
      "KL/mean": 1.3398051261901855,
      "KL/rejected_KL_mean": -2.8501052856445312,
      "KL/std": 11.12479019165039,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.379829406738281,
      "fcm_dpo/q_t": 0.47917458415031433,
      "grad_norm": 15.854077339172363,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": 1.6752437353134155,
      "logits/rejected": 1.6689536571502686,
      "logps/chosen": -244.87290954589844,
      "logps/ref_chosen": -250.4026336669922,
      "logps/ref_rejected": -229.63853454589844,
      "logps/rejected": -232.48863220214844,
      "loss": 5.2363,
      "margin_dpo/margin_mean": 8.379829406738281,
      "margin_dpo/margin_std": 12.591842651367188,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": 5.506731033325195,
      "KL/mean": 0.9220665693283081,
      "KL/rejected_KL_mean": -3.662607192993164,
      "KL/std": 12.892915725708008,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.169331550598145,
      "fcm_dpo/q_t": 0.47723501920700073,
      "grad_norm": 20.344806671142578,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": 1.8329553604125977,
      "logits/rejected": 1.8867353200912476,
      "logps/chosen": -252.88916015625,
      "logps/ref_chosen": -258.3959045410156,
      "logps/ref_rejected": -268.84228515625,
      "logps/rejected": -272.5049133300781,
      "loss": 5.2103,
      "margin_dpo/margin_mean": 9.169331550598145,
      "margin_dpo/margin_std": 14.731281280517578,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": 6.028781890869141,
      "KL/mean": 2.1645748615264893,
      "KL/rejected_KL_mean": -1.6996345520019531,
      "KL/std": 14.341768264770508,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.7284135818481445,
      "fcm_dpo/q_t": 0.4809097945690155,
      "grad_norm": 15.90015983581543,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": 1.9073671102523804,
      "logits/rejected": 1.8856362104415894,
      "logps/chosen": -311.2738037109375,
      "logps/ref_chosen": -317.30255126953125,
      "logps/ref_rejected": -300.47552490234375,
      "logps/rejected": -302.1751403808594,
      "loss": 5.2758,
      "margin_dpo/margin_mean": 7.728414058685303,
      "margin_dpo/margin_std": 16.971065521240234,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": 6.99530029296875,
      "KL/mean": 2.4653427600860596,
      "KL/rejected_KL_mean": -2.0646133422851562,
      "KL/std": 13.4789457321167,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.059913635253906,
      "fcm_dpo/q_t": 0.47754383087158203,
      "grad_norm": 16.541982650756836,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.78767728805542,
      "logits/rejected": 1.7354193925857544,
      "logps/chosen": -298.2591247558594,
      "logps/ref_chosen": -305.2544250488281,
      "logps/ref_rejected": -259.40716552734375,
      "logps/rejected": -261.4717712402344,
      "loss": 5.2207,
      "margin_dpo/margin_mean": 9.059913635253906,
      "margin_dpo/margin_std": 16.493165969848633,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": 3.5057220458984375,
      "KL/mean": 0.7952462434768677,
      "KL/rejected_KL_mean": -1.9152240753173828,
      "KL/std": 11.908321380615234,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.420945644378662,
      "fcm_dpo/q_t": 0.4865472912788391,
      "grad_norm": 14.622404098510742,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 1.6811366081237793,
      "logits/rejected": 1.7041386365890503,
      "logps/chosen": -213.13499450683594,
      "logps/ref_chosen": -216.64073181152344,
      "logps/ref_rejected": -224.36422729492188,
      "logps/rejected": -226.27944946289062,
      "loss": 5.3547,
      "margin_dpo/margin_mean": 5.420945167541504,
      "margin_dpo/margin_std": 14.458181381225586,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": 5.62921142578125,
      "KL/mean": 1.9514158964157104,
      "KL/rejected_KL_mean": -1.7263832092285156,
      "KL/std": 14.125900268554688,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.355593681335449,
      "fcm_dpo/q_t": 0.4817846715450287,
      "grad_norm": 15.868084907531738,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": 1.729200839996338,
      "logits/rejected": 1.7256333827972412,
      "logps/chosen": -263.1895751953125,
      "logps/ref_chosen": -268.8188171386719,
      "logps/ref_rejected": -272.55853271484375,
      "logps/rejected": -274.284912109375,
      "loss": 5.2904,
      "margin_dpo/margin_mean": 7.355593204498291,
      "margin_dpo/margin_std": 17.690921783447266,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": 8.519977569580078,
      "KL/mean": 3.222280740737915,
      "KL/rejected_KL_mean": -2.075408935546875,
      "KL/std": 15.066630363464355,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.595373153686523,
      "fcm_dpo/q_t": 0.4737876057624817,
      "grad_norm": 18.2539119720459,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": 1.695539951324463,
      "logits/rejected": 1.672548532485962,
      "logps/chosen": -313.1717834472656,
      "logps/ref_chosen": -321.6917419433594,
      "logps/ref_rejected": -279.1517333984375,
      "logps/rejected": -281.22711181640625,
      "loss": 5.1677,
      "margin_dpo/margin_mean": 10.59537410736084,
      "margin_dpo/margin_std": 18.353940963745117,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": 6.146183013916016,
      "KL/mean": 1.3429876565933228,
      "KL/rejected_KL_mean": -3.4602155685424805,
      "KL/std": 14.509088516235352,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.606401443481445,
      "fcm_dpo/q_t": 0.47617751359939575,
      "grad_norm": 16.64905548095703,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": 1.7271466255187988,
      "logits/rejected": 1.6182743310928345,
      "logps/chosen": -265.5290222167969,
      "logps/ref_chosen": -271.6752014160156,
      "logps/ref_rejected": -221.85577392578125,
      "logps/rejected": -225.31597900390625,
      "loss": 5.2,
      "margin_dpo/margin_mean": 9.606401443481445,
      "margin_dpo/margin_std": 15.955260276794434,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": 5.487314224243164,
      "KL/mean": 0.7873013019561768,
      "KL/rejected_KL_mean": -3.9127120971679688,
      "KL/std": 14.160686492919922,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.400023460388184,
      "fcm_dpo/q_t": 0.47676992416381836,
      "grad_norm": 16.482099533081055,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": 1.6574158668518066,
      "logits/rejected": 1.6650267839431763,
      "logps/chosen": -264.9117431640625,
      "logps/ref_chosen": -270.3990478515625,
      "logps/ref_rejected": -264.16925048828125,
      "logps/rejected": -268.08197021484375,
      "loss": 5.214,
      "margin_dpo/margin_mean": 9.400022506713867,
      "margin_dpo/margin_std": 18.513341903686523,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": 6.892814636230469,
      "KL/mean": 1.560383677482605,
      "KL/rejected_KL_mean": -3.7720470428466797,
      "KL/std": 15.73596477508545,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.664872169494629,
      "fcm_dpo/q_t": 0.4735832214355469,
      "grad_norm": 16.68074607849121,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": 1.622692346572876,
      "logits/rejected": 1.5840619802474976,
      "logps/chosen": -306.35845947265625,
      "logps/ref_chosen": -313.25128173828125,
      "logps/ref_rejected": -283.0433349609375,
      "logps/rejected": -286.81536865234375,
      "loss": 5.1672,
      "margin_dpo/margin_mean": 10.664872169494629,
      "margin_dpo/margin_std": 18.452678680419922,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": 7.969184875488281,
      "KL/mean": 1.7778995037078857,
      "KL/rejected_KL_mean": -4.413379669189453,
      "KL/std": 16.15068817138672,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.382560729980469,
      "fcm_dpo/q_t": 0.46937376260757446,
      "grad_norm": 16.80250358581543,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 1.6250829696655273,
      "logits/rejected": 1.547389268875122,
      "logps/chosen": -290.5948181152344,
      "logps/ref_chosen": -298.5639953613281,
      "logps/ref_rejected": -274.021728515625,
      "logps/rejected": -278.43511962890625,
      "loss": 5.1026,
      "margin_dpo/margin_mean": 12.382559776306152,
      "margin_dpo/margin_std": 18.3065242767334,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": 8.862152099609375,
      "KL/mean": 1.9788604974746704,
      "KL/rejected_KL_mean": -4.904428482055664,
      "KL/std": 17.061735153198242,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.766579627990723,
      "fcm_dpo/q_t": 0.4660224914550781,
      "grad_norm": 16.199844360351562,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": 1.630589246749878,
      "logits/rejected": 1.662994146347046,
      "logps/chosen": -248.2687225341797,
      "logps/ref_chosen": -257.130859375,
      "logps/ref_rejected": -247.71636962890625,
      "logps/rejected": -252.6207733154297,
      "loss": 5.0569,
      "margin_dpo/margin_mean": 13.766580581665039,
      "margin_dpo/margin_std": 20.150243759155273,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": 8.922870635986328,
      "KL/mean": 2.6507699489593506,
      "KL/rejected_KL_mean": -3.6213226318359375,
      "KL/std": 17.113662719726562,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.544188499450684,
      "fcm_dpo/q_t": 0.46907299757003784,
      "grad_norm": 16.407958984375,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": 1.6225829124450684,
      "logits/rejected": 1.6599586009979248,
      "logps/chosen": -286.03765869140625,
      "logps/ref_chosen": -294.9605712890625,
      "logps/ref_rejected": -272.6249694824219,
      "logps/rejected": -276.24627685546875,
      "loss": 5.1057,
      "margin_dpo/margin_mean": 12.544188499450684,
      "margin_dpo/margin_std": 20.357309341430664,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": 7.5218658447265625,
      "KL/mean": 1.676572561264038,
      "KL/rejected_KL_mean": -4.168737411499023,
      "KL/std": 15.609966278076172,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.690610885620117,
      "fcm_dpo/q_t": 0.4710570275783539,
      "grad_norm": 21.08173942565918,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 1.5897964239120483,
      "logits/rejected": 1.6563528776168823,
      "logps/chosen": -268.92034912109375,
      "logps/ref_chosen": -276.4422302246094,
      "logps/ref_rejected": -290.241455078125,
      "logps/rejected": -294.4101867675781,
      "loss": 5.1298,
      "margin_dpo/margin_mean": 11.6906099319458,
      "margin_dpo/margin_std": 19.31304359436035,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": 4.93031120300293,
      "KL/mean": 0.014614507555961609,
      "KL/rejected_KL_mean": -4.901090621948242,
      "KL/std": 16.61847496032715,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.831413269042969,
      "fcm_dpo/q_t": 0.47579699754714966,
      "grad_norm": 17.539602279663086,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 1.6064653396606445,
      "logits/rejected": 1.6063125133514404,
      "logps/chosen": -299.8944091796875,
      "logps/ref_chosen": -304.82470703125,
      "logps/ref_rejected": -289.9726867675781,
      "logps/rejected": -294.873779296875,
      "loss": 5.2106,
      "margin_dpo/margin_mean": 9.831412315368652,
      "margin_dpo/margin_std": 21.35895347595215,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": 7.034294128417969,
      "KL/mean": 0.4843648076057434,
      "KL/rejected_KL_mean": -6.06556510925293,
      "KL/std": 18.657806396484375,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.099861145019531,
      "fcm_dpo/q_t": 0.4679507911205292,
      "grad_norm": 17.841094970703125,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": 1.5017870664596558,
      "logits/rejected": 1.6120854616165161,
      "logps/chosen": -253.28045654296875,
      "logps/ref_chosen": -260.31475830078125,
      "logps/ref_rejected": -280.93756103515625,
      "logps/rejected": -287.00311279296875,
      "loss": 5.0996,
      "margin_dpo/margin_mean": 13.099862098693848,
      "margin_dpo/margin_std": 23.527923583984375,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": 7.510984420776367,
      "KL/mean": 0.7360672354698181,
      "KL/rejected_KL_mean": -6.038843154907227,
      "KL/std": 17.369871139526367,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.549842834472656,
      "fcm_dpo/q_t": 0.46649181842803955,
      "grad_norm": 16.18189811706543,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": 1.555115818977356,
      "logits/rejected": 1.5179142951965332,
      "logps/chosen": -271.609619140625,
      "logps/ref_chosen": -279.12060546875,
      "logps/ref_rejected": -247.22982788085938,
      "logps/rejected": -253.2686767578125,
      "loss": 5.0718,
      "margin_dpo/margin_mean": 13.54984188079834,
      "margin_dpo/margin_std": 21.237762451171875,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": 1.2811546325683594,
      "KL/mean": -3.465763568878174,
      "KL/rejected_KL_mean": -8.212678909301758,
      "KL/std": 17.69608497619629,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.49384880065918,
      "fcm_dpo/q_t": 0.47661879658699036,
      "grad_norm": 26.338592529296875,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": 1.7695540189743042,
      "logits/rejected": 1.8025752305984497,
      "logps/chosen": -314.32293701171875,
      "logps/ref_chosen": -315.6040954589844,
      "logps/ref_rejected": -291.45635986328125,
      "logps/rejected": -299.6690368652344,
      "loss": 5.2191,
      "margin_dpo/margin_mean": 9.493849754333496,
      "margin_dpo/margin_std": 20.25198745727539,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": 4.928071975708008,
      "KL/mean": -1.7125107049942017,
      "KL/rejected_KL_mean": -8.35308837890625,
      "KL/std": 18.227964401245117,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.281157493591309,
      "fcm_dpo/q_t": 0.4672252833843231,
      "grad_norm": 17.002321243286133,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": 1.472666621208191,
      "logits/rejected": 1.4604074954986572,
      "logps/chosen": -277.9449157714844,
      "logps/ref_chosen": -282.87298583984375,
      "logps/ref_rejected": -265.0682373046875,
      "logps/rejected": -273.4212951660156,
      "loss": 5.0758,
      "margin_dpo/margin_mean": 13.281157493591309,
      "margin_dpo/margin_std": 19.581642150878906,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": 4.083703994750977,
      "KL/mean": -2.286695957183838,
      "KL/rejected_KL_mean": -8.657093048095703,
      "KL/std": 17.814237594604492,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.740805625915527,
      "fcm_dpo/q_t": 0.46860769391059875,
      "grad_norm": 19.865968704223633,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": 1.4025537967681885,
      "logits/rejected": 1.5649460554122925,
      "logps/chosen": -259.35821533203125,
      "logps/ref_chosen": -263.4419250488281,
      "logps/ref_rejected": -251.43150329589844,
      "logps/rejected": -260.0885925292969,
      "loss": 5.1047,
      "margin_dpo/margin_mean": 12.740806579589844,
      "margin_dpo/margin_std": 21.957069396972656,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": 2.0643863677978516,
      "KL/mean": -3.5345051288604736,
      "KL/rejected_KL_mean": -9.133394241333008,
      "KL/std": 19.36092185974121,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.197775840759277,
      "fcm_dpo/q_t": 0.4723700284957886,
      "grad_norm": 19.680261611938477,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": 1.5411317348480225,
      "logits/rejected": 1.533325433731079,
      "logps/chosen": -293.20635986328125,
      "logps/ref_chosen": -295.2707214355469,
      "logps/ref_rejected": -283.9851989746094,
      "logps/rejected": -293.1186218261719,
      "loss": 5.1587,
      "margin_dpo/margin_mean": 11.197776794433594,
      "margin_dpo/margin_std": 20.97260284423828,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": 0.6056690216064453,
      "KL/mean": -4.960485458374023,
      "KL/rejected_KL_mean": -10.52664566040039,
      "KL/std": 18.549562454223633,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.13231372833252,
      "fcm_dpo/q_t": 0.4727070927619934,
      "grad_norm": 16.97205352783203,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": 1.3775867223739624,
      "logits/rejected": 1.365391492843628,
      "logps/chosen": -281.95220947265625,
      "logps/ref_chosen": -282.5578918457031,
      "logps/ref_rejected": -262.4772644042969,
      "logps/rejected": -273.00390625,
      "loss": 5.1728,
      "margin_dpo/margin_mean": 11.13231372833252,
      "margin_dpo/margin_std": 23.69282341003418,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": 1.811544418334961,
      "KL/mean": -2.2636001110076904,
      "KL/rejected_KL_mean": -6.338748931884766,
      "KL/std": 19.353012084960938,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.150288581848145,
      "fcm_dpo/q_t": 0.4799652695655823,
      "grad_norm": 17.075469970703125,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": 1.6507481336593628,
      "logits/rejected": 1.7246859073638916,
      "logps/chosen": -261.8392639160156,
      "logps/ref_chosen": -263.65081787109375,
      "logps/ref_rejected": -266.2420959472656,
      "logps/rejected": -272.5808410644531,
      "loss": 5.2845,
      "margin_dpo/margin_mean": 8.150290489196777,
      "margin_dpo/margin_std": 23.839759826660156,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": 5.529518127441406,
      "KL/mean": -2.0035970211029053,
      "KL/rejected_KL_mean": -9.536701202392578,
      "KL/std": 19.25857925415039,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.066213607788086,
      "fcm_dpo/q_t": 0.4629426896572113,
      "grad_norm": 16.375015258789062,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 1.7526944875717163,
      "logits/rejected": 1.720436692237854,
      "logps/chosen": -266.8529357910156,
      "logps/ref_chosen": -272.3824462890625,
      "logps/ref_rejected": -248.96444702148438,
      "logps/rejected": -258.50115966796875,
      "loss": 5.0165,
      "margin_dpo/margin_mean": 15.066213607788086,
      "margin_dpo/margin_std": 21.559059143066406,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": 2.9159793853759766,
      "KL/mean": -4.298311710357666,
      "KL/rejected_KL_mean": -11.512588500976562,
      "KL/std": 20.350566864013672,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.428566932678223,
      "fcm_dpo/q_t": 0.4645114243030548,
      "grad_norm": 22.543052673339844,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 1.5954140424728394,
      "logits/rejected": 1.644343376159668,
      "logps/chosen": -285.15509033203125,
      "logps/ref_chosen": -288.071044921875,
      "logps/ref_rejected": -274.83477783203125,
      "logps/rejected": -286.3473205566406,
      "loss": 5.0527,
      "margin_dpo/margin_mean": 14.428566932678223,
      "margin_dpo/margin_std": 24.45475196838379,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": 6.097406387329102,
      "KL/mean": -1.012816071510315,
      "KL/rejected_KL_mean": -8.123037338256836,
      "KL/std": 20.251998901367188,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.220438957214355,
      "fcm_dpo/q_t": 0.4651198983192444,
      "grad_norm": 17.25643539428711,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": 1.5306440591812134,
      "logits/rejected": 1.4179447889328003,
      "logps/chosen": -282.8944396972656,
      "logps/ref_chosen": -288.9918518066406,
      "logps/ref_rejected": -228.26071166992188,
      "logps/rejected": -236.38372802734375,
      "loss": 5.0619,
      "margin_dpo/margin_mean": 14.220440864562988,
      "margin_dpo/margin_std": 24.457334518432617,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": 5.720268249511719,
      "KL/mean": -2.425501585006714,
      "KL/rejected_KL_mean": -10.571264266967773,
      "KL/std": 21.313739776611328,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.291526794433594,
      "fcm_dpo/q_t": 0.4601180851459503,
      "grad_norm": 18.92660140991211,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": 1.489121913909912,
      "logits/rejected": 1.5202325582504272,
      "logps/chosen": -283.75421142578125,
      "logps/ref_chosen": -289.4744873046875,
      "logps/ref_rejected": -265.130615234375,
      "logps/rejected": -275.701904296875,
      "loss": 4.996,
      "margin_dpo/margin_mean": 16.291526794433594,
      "margin_dpo/margin_std": 26.781078338623047,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": 7.1892852783203125,
      "KL/mean": 0.40949952602386475,
      "KL/rejected_KL_mean": -6.370277404785156,
      "KL/std": 20.614206314086914,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.55956745147705,
      "fcm_dpo/q_t": 0.4667853116989136,
      "grad_norm": 19.44406509399414,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": 1.6926894187927246,
      "logits/rejected": 1.6756823062896729,
      "logps/chosen": -283.3970947265625,
      "logps/ref_chosen": -290.5863342285156,
      "logps/ref_rejected": -263.41046142578125,
      "logps/rejected": -269.7807312011719,
      "loss": 5.0852,
      "margin_dpo/margin_mean": 13.559566497802734,
      "margin_dpo/margin_std": 24.296241760253906,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": 7.28510856628418,
      "KL/mean": -1.4730184078216553,
      "KL/rejected_KL_mean": -10.23114013671875,
      "KL/std": 19.682371139526367,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.51624870300293,
      "fcm_dpo/q_t": 0.457155704498291,
      "grad_norm": 17.066204071044922,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": 1.6921424865722656,
      "logits/rejected": 1.6142408847808838,
      "logps/chosen": -257.2551574707031,
      "logps/ref_chosen": -264.5402526855469,
      "logps/ref_rejected": -214.7958221435547,
      "logps/rejected": -225.02696228027344,
      "loss": 4.9451,
      "margin_dpo/margin_mean": 17.51624870300293,
      "margin_dpo/margin_std": 24.685867309570312,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": 3.1016483306884766,
      "KL/mean": -3.8503642082214355,
      "KL/rejected_KL_mean": -10.8023681640625,
      "KL/std": 19.700557708740234,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.904016494750977,
      "fcm_dpo/q_t": 0.4658479392528534,
      "grad_norm": 26.559860229492188,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": 1.6372588872909546,
      "logits/rejected": 1.7681186199188232,
      "logps/chosen": -267.078125,
      "logps/ref_chosen": -270.17974853515625,
      "logps/ref_rejected": -261.2323913574219,
      "logps/rejected": -272.0347595214844,
      "loss": 5.0716,
      "margin_dpo/margin_mean": 13.904016494750977,
      "margin_dpo/margin_std": 24.00600814819336,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": 1.6271686553955078,
      "KL/mean": -4.333284854888916,
      "KL/rejected_KL_mean": -10.293737411499023,
      "KL/std": 22.16680335998535,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.920907020568848,
      "fcm_dpo/q_t": 0.4708361327648163,
      "grad_norm": 17.387216567993164,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": 1.5833531618118286,
      "logits/rejected": 1.56257963180542,
      "logps/chosen": -281.1534118652344,
      "logps/ref_chosen": -282.7806091308594,
      "logps/ref_rejected": -249.23399353027344,
      "logps/rejected": -259.5277099609375,
      "loss": 5.1642,
      "margin_dpo/margin_mean": 11.920906066894531,
      "margin_dpo/margin_std": 26.6894588470459,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": 2.243375778198242,
      "KL/mean": -4.042187690734863,
      "KL/rejected_KL_mean": -10.327751159667969,
      "KL/std": 20.842845916748047,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.571127891540527,
      "fcm_dpo/q_t": 0.4692600667476654,
      "grad_norm": 16.492469787597656,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": 1.6099210977554321,
      "logits/rejected": 1.627861738204956,
      "logps/chosen": -226.67391967773438,
      "logps/ref_chosen": -228.9173126220703,
      "logps/ref_rejected": -226.81222534179688,
      "logps/rejected": -237.13998413085938,
      "loss": 5.1303,
      "margin_dpo/margin_mean": 12.571126937866211,
      "margin_dpo/margin_std": 25.929073333740234,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": 3.637248992919922,
      "KL/mean": -4.202162742614746,
      "KL/rejected_KL_mean": -12.041574478149414,
      "KL/std": 21.598217010498047,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.678825378417969,
      "fcm_dpo/q_t": 0.4616299271583557,
      "grad_norm": 18.618160247802734,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": 1.5622092485427856,
      "logits/rejected": 1.528815507888794,
      "logps/chosen": -307.44866943359375,
      "logps/ref_chosen": -311.0859375,
      "logps/ref_rejected": -303.225341796875,
      "logps/rejected": -315.26690673828125,
      "loss": 5.0122,
      "margin_dpo/margin_mean": 15.678825378417969,
      "margin_dpo/margin_std": 25.788564682006836,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": 1.1149120330810547,
      "KL/mean": -6.284457206726074,
      "KL/rejected_KL_mean": -13.683834075927734,
      "KL/std": 20.9503173828125,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.798748970031738,
      "fcm_dpo/q_t": 0.4636927843093872,
      "grad_norm": 17.919178009033203,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 1.406896710395813,
      "logits/rejected": 1.4150539636611938,
      "logps/chosen": -269.3685302734375,
      "logps/ref_chosen": -270.48345947265625,
      "logps/ref_rejected": -269.0491638183594,
      "logps/rejected": -282.73297119140625,
      "loss": 5.0397,
      "margin_dpo/margin_mean": 14.798749923706055,
      "margin_dpo/margin_std": 24.814159393310547,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": 4.418113708496094,
      "KL/mean": -6.873030662536621,
      "KL/rejected_KL_mean": -18.1641845703125,
      "KL/std": 22.918441772460938,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.010055048391222954,
      "fcm_dpo/delta": 0.013687034137547016,
      "fcm_dpo/margin": 22.582286834716797,
      "fcm_dpo/q_t": 0.44467097520828247,
      "grad_norm": 19.57944107055664,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.4989944696426392,
      "logits/rejected": 1.515008807182312,
      "logps/chosen": -283.7475891113281,
      "logps/ref_chosen": -288.16571044921875,
      "logps/ref_rejected": -264.7412414550781,
      "logps/rejected": -282.9054260253906,
      "loss": 4.7729,
      "margin_dpo/margin_mean": 22.582286834716797,
      "margin_dpo/margin_std": 28.405309677124023,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": 3.3562088012695312,
      "KL/mean": -5.1849799156188965,
      "KL/rejected_KL_mean": -13.726175308227539,
      "KL/std": 23.107046127319336,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.0101100979372859,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.08238983154297,
      "fcm_dpo/q_t": 0.45782357454299927,
      "grad_norm": 20.39621925354004,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": 1.2527517080307007,
      "logits/rejected": 1.321351408958435,
      "logps/chosen": -256.58233642578125,
      "logps/ref_chosen": -259.9385070800781,
      "logps/ref_rejected": -266.2453308105469,
      "logps/rejected": -279.9715576171875,
      "loss": 4.9695,
      "margin_dpo/margin_mean": 17.08238983154297,
      "margin_dpo/margin_std": 27.618545532226562,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": 1.953378677368164,
      "KL/mean": -10.318442344665527,
      "KL/rejected_KL_mean": -22.590261459350586,
      "KL/std": 24.62543487548828,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.010321839712560177,
      "fcm_dpo/delta": 0.034631531685590744,
      "fcm_dpo/margin": 24.543630599975586,
      "fcm_dpo/q_t": 0.43848657608032227,
      "grad_norm": 22.164987564086914,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 1.6414861679077148,
      "logits/rejected": 1.7091753482818604,
      "logps/chosen": -263.6236572265625,
      "logps/ref_chosen": -265.5769958496094,
      "logps/ref_rejected": -277.6246032714844,
      "logps/rejected": -300.21484375,
      "loss": 4.6767,
      "margin_dpo/margin_mean": 24.543630599975586,
      "margin_dpo/margin_std": 27.022510528564453,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -0.15570449829101562,
      "KL/mean": -9.099052429199219,
      "KL/rejected_KL_mean": -18.04239273071289,
      "KL/std": 25.2545223236084,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.010394117794930935,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.886686325073242,
      "fcm_dpo/q_t": 0.4546920359134674,
      "grad_norm": 20.077585220336914,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": 1.3967751264572144,
      "logits/rejected": 1.3666248321533203,
      "logps/chosen": -282.4291076660156,
      "logps/ref_chosen": -282.2734069824219,
      "logps/ref_rejected": -261.5135498046875,
      "logps/rejected": -279.5559387207031,
      "loss": 4.9495,
      "margin_dpo/margin_mean": 17.88668441772461,
      "margin_dpo/margin_std": 31.39680290222168,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -1.3937911987304688,
      "KL/mean": -8.020185470581055,
      "KL/rejected_KL_mean": -14.646583557128906,
      "KL/std": 23.012699127197266,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.010394117794930935,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.252791404724121,
      "fcm_dpo/q_t": 0.4664270579814911,
      "grad_norm": 19.56340789794922,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": 1.595047116279602,
      "logits/rejected": 1.6882283687591553,
      "logps/chosen": -280.1053771972656,
      "logps/ref_chosen": -278.7115783691406,
      "logps/ref_rejected": -298.0945739746094,
      "logps/rejected": -312.74114990234375,
      "loss": 5.1078,
      "margin_dpo/margin_mean": 13.252791404724121,
      "margin_dpo/margin_std": 28.350543975830078,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -0.12450027465820312,
      "KL/mean": -11.341375350952148,
      "KL/rejected_KL_mean": -22.558246612548828,
      "KL/std": 25.48090362548828,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.010489447042346,
      "fcm_dpo/delta": 0.026143595576286316,
      "fcm_dpo/margin": 22.433757781982422,
      "fcm_dpo/q_t": 0.4431641399860382,
      "grad_norm": 19.25153923034668,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": 1.653823971748352,
      "logits/rejected": 1.6861575841903687,
      "logps/chosen": -268.47564697265625,
      "logps/ref_chosen": -268.35113525390625,
      "logps/ref_rejected": -241.9660186767578,
      "logps/rejected": -264.5242919921875,
      "loss": 4.7837,
      "margin_dpo/margin_mean": 22.433757781982422,
      "margin_dpo/margin_std": 33.53373336791992,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -1.5132007598876953,
      "KL/mean": -9.272302627563477,
      "KL/rejected_KL_mean": -17.031391143798828,
      "KL/std": 24.873733520507812,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.010691219009459019,
      "fcm_dpo/delta": 0.009085104800760746,
      "fcm_dpo/margin": 15.518184661865234,
      "fcm_dpo/q_t": 0.45982784032821655,
      "grad_norm": 26.817617416381836,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": 1.4206135272979736,
      "logits/rejected": 1.4804435968399048,
      "logps/chosen": -266.9168395996094,
      "logps/ref_chosen": -265.40362548828125,
      "logps/ref_rejected": -257.82440185546875,
      "logps/rejected": -274.8558044433594,
      "loss": 5.0146,
      "margin_dpo/margin_mean": 15.518184661865234,
      "margin_dpo/margin_std": 28.87850570678711,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -0.7544136047363281,
      "KL/mean": -9.26556396484375,
      "KL/rejected_KL_mean": -17.776708602905273,
      "KL/std": 24.63243293762207,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.010704242624342442,
      "fcm_dpo/delta": 0.012122713029384613,
      "fcm_dpo/margin": 17.02230453491211,
      "fcm_dpo/q_t": 0.4556833803653717,
      "grad_norm": 21.16738510131836,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": 1.442050814628601,
      "logits/rejected": 1.485360860824585,
      "logps/chosen": -281.91668701171875,
      "logps/ref_chosen": -281.1622619628906,
      "logps/ref_rejected": -268.2899475097656,
      "logps/rejected": -286.066650390625,
      "loss": 4.9558,
      "margin_dpo/margin_mean": 17.022306442260742,
      "margin_dpo/margin_std": 29.565460205078125,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -1.5836906433105469,
      "KL/mean": -8.403104782104492,
      "KL/rejected_KL_mean": -15.222509384155273,
      "KL/std": 25.65123748779297,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.010795408859848976,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.638814926147461,
      "fcm_dpo/q_t": 0.46464693546295166,
      "grad_norm": 25.587305068969727,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": 1.6808595657348633,
      "logits/rejected": 1.5864752531051636,
      "logps/chosen": -312.90869140625,
      "logps/ref_chosen": -311.32501220703125,
      "logps/ref_rejected": -248.85350036621094,
      "logps/rejected": -264.07598876953125,
      "loss": 5.1256,
      "margin_dpo/margin_mean": 13.638816833496094,
      "margin_dpo/margin_std": 33.95243835449219,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": 2.2486438751220703,
      "KL/mean": -8.23265552520752,
      "KL/rejected_KL_mean": -18.713945388793945,
      "KL/std": 25.501998901367188,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.010872100479900837,
      "fcm_dpo/delta": 0.016497626900672913,
      "fcm_dpo/margin": 20.96258544921875,
      "fcm_dpo/q_t": 0.4452162981033325,
      "grad_norm": 21.568923950195312,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": 1.5142244100570679,
      "logits/rejected": 1.5229939222335815,
      "logps/chosen": -264.8603210449219,
      "logps/ref_chosen": -267.10894775390625,
      "logps/ref_rejected": -258.66162109375,
      "logps/rejected": -277.3755798339844,
      "loss": 4.7988,
      "margin_dpo/margin_mean": 20.962583541870117,
      "margin_dpo/margin_std": 29.371719360351562,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": 1.7124061584472656,
      "KL/mean": -9.285618782043457,
      "KL/rejected_KL_mean": -20.283641815185547,
      "KL/std": 26.60118865966797,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.010967358946800232,
      "fcm_dpo/delta": 0.008662651292979717,
      "fcm_dpo/margin": 21.996055603027344,
      "fcm_dpo/q_t": 0.44194597005844116,
      "grad_norm": 28.744068145751953,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 1.7401057481765747,
      "logits/rejected": 1.6398823261260986,
      "logps/chosen": -300.17999267578125,
      "logps/ref_chosen": -301.89239501953125,
      "logps/ref_rejected": -247.645263671875,
      "logps/rejected": -267.92889404296875,
      "loss": 4.7768,
      "margin_dpo/margin_mean": 21.996055603027344,
      "margin_dpo/margin_std": 33.60957717895508,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": 2.9218883514404297,
      "KL/mean": -7.01794958114624,
      "KL/rejected_KL_mean": -16.957782745361328,
      "KL/std": 25.88132667541504,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.010991818271577358,
      "fcm_dpo/delta": -0.0026226334739476442,
      "fcm_dpo/margin": 19.879669189453125,
      "fcm_dpo/q_t": 0.4470987319946289,
      "grad_norm": 27.99663543701172,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": 1.3844854831695557,
      "logits/rejected": 1.3556245565414429,
      "logps/chosen": -285.3111877441406,
      "logps/ref_chosen": -288.2330627441406,
      "logps/ref_rejected": -252.5148162841797,
      "logps/rejected": -269.47259521484375,
      "loss": 4.8494,
      "margin_dpo/margin_mean": 19.879669189453125,
      "margin_dpo/margin_std": 31.44927406311035,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": 0.735565185546875,
      "KL/mean": -10.822467803955078,
      "KL/rejected_KL_mean": -22.380508422851562,
      "KL/std": 25.03566551208496,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.01105956919491291,
      "fcm_dpo/delta": 0.023430872708559036,
      "fcm_dpo/margin": 23.116077423095703,
      "fcm_dpo/q_t": 0.438064843416214,
      "grad_norm": 33.8231201171875,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": 1.4928569793701172,
      "logits/rejected": 1.4581284523010254,
      "logps/chosen": -286.02825927734375,
      "logps/ref_chosen": -286.7638244628906,
      "logps/ref_rejected": -262.8302917480469,
      "logps/rejected": -285.21075439453125,
      "loss": 4.6888,
      "margin_dpo/margin_mean": 23.11607551574707,
      "margin_dpo/margin_std": 28.119739532470703,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -6.791299819946289,
      "KL/mean": -16.716739654541016,
      "KL/rejected_KL_mean": -26.642183303833008,
      "KL/std": 26.729385375976562,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.011204415932297707,
      "fcm_dpo/delta": 0.004114303272217512,
      "fcm_dpo/margin": 19.850872039794922,
      "fcm_dpo/q_t": 0.44632551074028015,
      "grad_norm": 28.238964080810547,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": 1.6854768991470337,
      "logits/rejected": 1.6470931768417358,
      "logps/chosen": -267.0697937011719,
      "logps/ref_chosen": -260.2784729003906,
      "logps/ref_rejected": -217.10556030273438,
      "logps/rejected": -243.74774169921875,
      "loss": 4.8291,
      "margin_dpo/margin_mean": 19.850872039794922,
      "margin_dpo/margin_std": 30.556453704833984,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -1.7575531005859375,
      "KL/mean": -10.245406150817871,
      "KL/rejected_KL_mean": -18.733251571655273,
      "KL/std": 24.99262809753418,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.011288748122751713,
      "fcm_dpo/delta": 0.011521327309310436,
      "fcm_dpo/margin": 16.97568130493164,
      "fcm_dpo/q_t": 0.4537385106086731,
      "grad_norm": 31.228561401367188,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": 1.5499904155731201,
      "logits/rejected": 1.5970572233200073,
      "logps/chosen": -272.31182861328125,
      "logps/ref_chosen": -270.5543212890625,
      "logps/ref_rejected": -259.80218505859375,
      "logps/rejected": -278.535400390625,
      "loss": 4.9301,
      "margin_dpo/margin_mean": 16.975677490234375,
      "margin_dpo/margin_std": 29.00372314453125,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -3.4060707092285156,
      "KL/mean": -11.133772850036621,
      "KL/rejected_KL_mean": -18.861465454101562,
      "KL/std": 27.443342208862305,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.011408022604882717,
      "fcm_dpo/delta": 0.007390481885522604,
      "fcm_dpo/margin": 15.455399513244629,
      "fcm_dpo/q_t": 0.4580705761909485,
      "grad_norm": 23.064071655273438,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 1.6677390336990356,
      "logits/rejected": 1.661797046661377,
      "logps/chosen": -273.70050048828125,
      "logps/ref_chosen": -270.29443359375,
      "logps/ref_rejected": -230.40151977539062,
      "logps/rejected": -249.26296997070312,
      "loss": 5.0407,
      "margin_dpo/margin_mean": 15.455398559570312,
      "margin_dpo/margin_std": 35.31707763671875,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -3.3137989044189453,
      "KL/mean": -12.561918258666992,
      "KL/rejected_KL_mean": -21.81003761291504,
      "KL/std": 26.816286087036133,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.011413662694394588,
      "fcm_dpo/delta": 0.004934557713568211,
      "fcm_dpo/margin": 18.49625015258789,
      "fcm_dpo/q_t": 0.4491766095161438,
      "grad_norm": 22.43897247314453,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": 1.6711032390594482,
      "logits/rejected": 1.6316914558410645,
      "logps/chosen": -276.7743835449219,
      "logps/ref_chosen": -273.4605712890625,
      "logps/ref_rejected": -248.86997985839844,
      "logps/rejected": -270.6800231933594,
      "loss": 4.8852,
      "margin_dpo/margin_mean": 18.496248245239258,
      "margin_dpo/margin_std": 32.05202102661133,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": 0.7094688415527344,
      "KL/mean": -11.080504417419434,
      "KL/rejected_KL_mean": -22.870481491088867,
      "KL/std": 28.838674545288086,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.011518949642777443,
      "fcm_dpo/delta": 0.011982799507677555,
      "fcm_dpo/margin": 23.579954147338867,
      "fcm_dpo/q_t": 0.4347684681415558,
      "grad_norm": 28.802583694458008,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": 1.6713992357254028,
      "logits/rejected": 1.6802916526794434,
      "logps/chosen": -305.01519775390625,
      "logps/ref_chosen": -305.7247009277344,
      "logps/ref_rejected": -290.16650390625,
      "logps/rejected": -313.0369873046875,
      "loss": 4.675,
      "margin_dpo/margin_mean": 23.579954147338867,
      "margin_dpo/margin_std": 31.07076644897461,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": 0.3470611572265625,
      "KL/mean": -13.401091575622559,
      "KL/rejected_KL_mean": -27.149242401123047,
      "KL/std": 28.684965133666992,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.011510763317346573,
      "fcm_dpo/delta": -0.01709855906665325,
      "fcm_dpo/margin": 27.496299743652344,
      "fcm_dpo/q_t": 0.42438074946403503,
      "grad_norm": 20.1732234954834,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": 1.5505868196487427,
      "logits/rejected": 1.6319119930267334,
      "logps/chosen": -251.82321166992188,
      "logps/ref_chosen": -252.1702880859375,
      "logps/ref_rejected": -233.34750366210938,
      "logps/rejected": -260.4967346191406,
      "loss": 4.5314,
      "margin_dpo/margin_mean": 27.496299743652344,
      "margin_dpo/margin_std": 32.141719818115234,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": 1.6806411743164062,
      "KL/mean": -10.368851661682129,
      "KL/rejected_KL_mean": -22.418344497680664,
      "KL/std": 27.264205932617188,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.011751345358788967,
      "fcm_dpo/delta": 0.05769862234592438,
      "fcm_dpo/margin": 24.098997116088867,
      "fcm_dpo/q_t": 0.4327648878097534,
      "grad_norm": 28.241954803466797,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": 1.4562406539916992,
      "logits/rejected": 1.4976229667663574,
      "logps/chosen": -300.7977294921875,
      "logps/ref_chosen": -302.4783935546875,
      "logps/ref_rejected": -259.40777587890625,
      "logps/rejected": -281.82611083984375,
      "loss": 4.6355,
      "margin_dpo/margin_mean": 24.098995208740234,
      "margin_dpo/margin_std": 32.278564453125,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": 0.7701129913330078,
      "KL/mean": -11.42619514465332,
      "KL/rejected_KL_mean": -23.622509002685547,
      "KL/std": 28.058786392211914,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.01193427387624979,
      "fcm_dpo/delta": 0.0025230085011571646,
      "fcm_dpo/margin": 24.392614364624023,
      "fcm_dpo/q_t": 0.4306865930557251,
      "grad_norm": 26.48332977294922,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 1.3849910497665405,
      "logits/rejected": 1.496009349822998,
      "logps/chosen": -249.87660217285156,
      "logps/ref_chosen": -250.646728515625,
      "logps/ref_rejected": -259.1659851074219,
      "logps/rejected": -282.78851318359375,
      "loss": 4.619,
      "margin_dpo/margin_mean": 24.39261245727539,
      "margin_dpo/margin_std": 31.400779724121094,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": 1.2688407897949219,
      "KL/mean": -6.934935569763184,
      "KL/rejected_KL_mean": -15.138717651367188,
      "KL/std": 28.8588924407959,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.01200177427381277,
      "fcm_dpo/delta": 0.0067544481717050076,
      "fcm_dpo/margin": 16.407550811767578,
      "fcm_dpo/q_t": 0.45247969031333923,
      "grad_norm": 25.667985916137695,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": 1.637864112854004,
      "logits/rejected": 1.6452244520187378,
      "logps/chosen": -327.7560729980469,
      "logps/ref_chosen": -329.02490234375,
      "logps/ref_rejected": -293.0215759277344,
      "logps/rejected": -308.1603088378906,
      "loss": 4.9688,
      "margin_dpo/margin_mean": 16.40755271911621,
      "margin_dpo/margin_std": 33.70692443847656,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -0.29935455322265625,
      "KL/mean": -10.511642456054688,
      "KL/rejected_KL_mean": -20.723934173583984,
      "KL/std": 28.731508255004883,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.012040091678500175,
      "fcm_dpo/delta": 0.0011034263297915459,
      "fcm_dpo/margin": 20.424579620361328,
      "fcm_dpo/q_t": 0.44135037064552307,
      "grad_norm": 24.6225528717041,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": 1.489302635192871,
      "logits/rejected": 1.4574329853057861,
      "logps/chosen": -290.44000244140625,
      "logps/ref_chosen": -290.140625,
      "logps/ref_rejected": -271.474853515625,
      "logps/rejected": -292.1987609863281,
      "loss": 4.7817,
      "margin_dpo/margin_mean": 20.424577713012695,
      "margin_dpo/margin_std": 31.998905181884766,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -0.5919399261474609,
      "KL/mean": -13.4032621383667,
      "KL/rejected_KL_mean": -26.214590072631836,
      "KL/std": 28.453168869018555,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.012006629258394241,
      "fcm_dpo/delta": -0.00716695562005043,
      "fcm_dpo/margin": 25.622657775878906,
      "fcm_dpo/q_t": 0.4268648028373718,
      "grad_norm": 27.664628982543945,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": 1.5641463994979858,
      "logits/rejected": 1.4867647886276245,
      "logps/chosen": -329.8992004394531,
      "logps/ref_chosen": -329.3072509765625,
      "logps/ref_rejected": -256.6839599609375,
      "logps/rejected": -282.8985595703125,
      "loss": 4.5653,
      "margin_dpo/margin_mean": 25.622661590576172,
      "margin_dpo/margin_std": 31.98184585571289,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -3.694713592529297,
      "KL/mean": -13.806648254394531,
      "KL/rejected_KL_mean": -23.91858673095703,
      "KL/std": 29.392501831054688,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.012022260576486588,
      "fcm_dpo/delta": 0.007601415738463402,
      "fcm_dpo/margin": 20.223865509033203,
      "fcm_dpo/q_t": 0.44179317355155945,
      "grad_norm": 38.27320861816406,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": 1.5666911602020264,
      "logits/rejected": 1.5582891702651978,
      "logps/chosen": -279.59698486328125,
      "logps/ref_chosen": -275.9023132324219,
      "logps/ref_rejected": -281.4163513183594,
      "logps/rejected": -305.3349304199219,
      "loss": 4.8202,
      "margin_dpo/margin_mean": 20.223865509033203,
      "margin_dpo/margin_std": 35.62342071533203,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -6.122219085693359,
      "KL/mean": -14.216109275817871,
      "KL/rejected_KL_mean": -22.30999755859375,
      "KL/std": 29.391691207885742,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.012014658190310001,
      "fcm_dpo/delta": -0.006528819445520639,
      "fcm_dpo/margin": 16.187788009643555,
      "fcm_dpo/q_t": 0.4533299505710602,
      "grad_norm": 28.004331588745117,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": 1.4710972309112549,
      "logits/rejected": 1.539896011352539,
      "logps/chosen": -249.68643188476562,
      "logps/ref_chosen": -243.56423950195312,
      "logps/ref_rejected": -239.12905883789062,
      "logps/rejected": -261.43902587890625,
      "loss": 5.0117,
      "margin_dpo/margin_mean": 16.187786102294922,
      "margin_dpo/margin_std": 35.992008209228516,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -7.832380294799805,
      "KL/mean": -15.139298439025879,
      "KL/rejected_KL_mean": -22.44622230529785,
      "KL/std": 30.188575744628906,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.011990725062787533,
      "fcm_dpo/delta": 0.0012450310168787837,
      "fcm_dpo/margin": 14.613851547241211,
      "fcm_dpo/q_t": 0.4581787586212158,
      "grad_norm": 24.036779403686523,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": 1.6910889148712158,
      "logits/rejected": 1.6194746494293213,
      "logps/chosen": -301.13604736328125,
      "logps/ref_chosen": -293.30364990234375,
      "logps/ref_rejected": -292.293701171875,
      "logps/rejected": -314.73992919921875,
      "loss": 5.1212,
      "margin_dpo/margin_mean": 14.613849639892578,
      "margin_dpo/margin_std": 39.674659729003906,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -3.1419734954833984,
      "KL/mean": -15.711864471435547,
      "KL/rejected_KL_mean": -28.281755447387695,
      "KL/std": 32.00127029418945,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.011991831474006176,
      "fcm_dpo/delta": 0.004078309051692486,
      "fcm_dpo/margin": 25.139781951904297,
      "fcm_dpo/q_t": 0.42914286255836487,
      "grad_norm": 25.652549743652344,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": 1.5067684650421143,
      "logits/rejected": 1.5234694480895996,
      "logps/chosen": -279.2682189941406,
      "logps/ref_chosen": -276.126220703125,
      "logps/ref_rejected": -240.31179809570312,
      "logps/rejected": -268.59356689453125,
      "loss": 4.6432,
      "margin_dpo/margin_mean": 25.139781951904297,
      "margin_dpo/margin_std": 38.09182357788086,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -3.7499008178710938,
      "KL/mean": -13.65191650390625,
      "KL/rejected_KL_mean": -23.553939819335938,
      "KL/std": 30.675369262695312,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.012122081592679024,
      "fcm_dpo/delta": 0.015231732279062271,
      "fcm_dpo/margin": 19.80405044555664,
      "fcm_dpo/q_t": 0.4431571364402771,
      "grad_norm": 34.862857818603516,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 1.3151205778121948,
      "logits/rejected": 1.4281214475631714,
      "logps/chosen": -253.61233520507812,
      "logps/ref_chosen": -249.8624267578125,
      "logps/ref_rejected": -271.18603515625,
      "logps/rejected": -294.739990234375,
      "loss": 4.8431,
      "margin_dpo/margin_mean": 19.80405044555664,
      "margin_dpo/margin_std": 35.81821060180664,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -4.045038223266602,
      "KL/mean": -16.410900115966797,
      "KL/rejected_KL_mean": -28.77677345275879,
      "KL/std": 30.051267623901367,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.012371964752674103,
      "fcm_dpo/delta": 0.024217654019594193,
      "fcm_dpo/margin": 24.73173713684082,
      "fcm_dpo/q_t": 0.42816925048828125,
      "grad_norm": 39.40521240234375,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": 1.4741824865341187,
      "logits/rejected": 1.5057199001312256,
      "logps/chosen": -264.71417236328125,
      "logps/ref_chosen": -260.66912841796875,
      "logps/ref_rejected": -229.25491333007812,
      "logps/rejected": -258.03167724609375,
      "loss": 4.6074,
      "margin_dpo/margin_mean": 24.731735229492188,
      "margin_dpo/margin_std": 34.881534576416016,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -4.550603866577148,
      "KL/mean": -16.125930786132812,
      "KL/rejected_KL_mean": -27.701255798339844,
      "KL/std": 31.507169723510742,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.012354785576462746,
      "fcm_dpo/delta": -0.006280969362705946,
      "fcm_dpo/margin": 23.150653839111328,
      "fcm_dpo/q_t": 0.43251562118530273,
      "grad_norm": 29.51620101928711,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.6002473831176758,
      "logits/rejected": 1.5772075653076172,
      "logps/chosen": -295.439208984375,
      "logps/ref_chosen": -290.8885803222656,
      "logps/ref_rejected": -275.1922302246094,
      "logps/rejected": -302.8934631347656,
      "loss": 4.7291,
      "margin_dpo/margin_mean": 23.150653839111328,
      "margin_dpo/margin_std": 38.71656799316406,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -7.448886871337891,
      "KL/mean": -21.986312866210938,
      "KL/rejected_KL_mean": -36.52374267578125,
      "KL/std": 31.393590927124023,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.01226753182709217,
      "fcm_dpo/delta": -0.006223441567271948,
      "fcm_dpo/margin": 29.07485580444336,
      "fcm_dpo/q_t": 0.4150747060775757,
      "grad_norm": 29.116426467895508,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": 1.4838136434555054,
      "logits/rejected": 1.5051133632659912,
      "logps/chosen": -278.0157165527344,
      "logps/ref_chosen": -270.56683349609375,
      "logps/ref_rejected": -260.3321533203125,
      "logps/rejected": -296.85589599609375,
      "loss": 4.4188,
      "margin_dpo/margin_mean": 29.07485580444336,
      "margin_dpo/margin_std": 33.31055450439453,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -7.519248962402344,
      "KL/mean": -19.583415985107422,
      "KL/rejected_KL_mean": -31.647600173950195,
      "KL/std": 32.045372009277344,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.012288594618439674,
      "fcm_dpo/delta": 0.0009415128733962774,
      "fcm_dpo/margin": 24.128345489501953,
      "fcm_dpo/q_t": 0.430392324924469,
      "grad_norm": 39.20041275024414,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": 1.7407642602920532,
      "logits/rejected": 1.797974705696106,
      "logps/chosen": -268.5472106933594,
      "logps/ref_chosen": -261.0279846191406,
      "logps/ref_rejected": -240.72361755371094,
      "logps/rejected": -272.3712463378906,
      "loss": 4.6941,
      "margin_dpo/margin_mean": 24.128347396850586,
      "margin_dpo/margin_std": 40.53470230102539,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -2.994281768798828,
      "KL/mean": -21.23874855041504,
      "KL/rejected_KL_mean": -39.48322677612305,
      "KL/std": 35.68898391723633,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.011903500184416771,
      "fcm_dpo/delta": -0.053442347794771194,
      "fcm_dpo/margin": 36.48895263671875,
      "fcm_dpo/q_t": 0.3980030119419098,
      "grad_norm": 28.29414939880371,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": 1.6047555208206177,
      "logits/rejected": 1.5928974151611328,
      "logps/chosen": -312.078125,
      "logps/ref_chosen": -309.0838317871094,
      "logps/ref_rejected": -263.6435241699219,
      "logps/rejected": -303.12677001953125,
      "loss": 4.2616,
      "margin_dpo/margin_mean": 36.488948822021484,
      "margin_dpo/margin_std": 43.0412712097168,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -10.358444213867188,
      "KL/mean": -21.835647583007812,
      "KL/rejected_KL_mean": -33.31285858154297,
      "KL/std": 33.04166793823242,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.011681336909532547,
      "fcm_dpo/delta": -0.011190216988325119,
      "fcm_dpo/margin": 22.95440673828125,
      "fcm_dpo/q_t": 0.4366164207458496,
      "grad_norm": 27.781278610229492,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": 1.551453948020935,
      "logits/rejected": 1.6309767961502075,
      "logps/chosen": -276.0601501464844,
      "logps/ref_chosen": -265.70172119140625,
      "logps/ref_rejected": -257.5074157714844,
      "logps/rejected": -290.82025146484375,
      "loss": 4.7563,
      "margin_dpo/margin_mean": 22.954404830932617,
      "margin_dpo/margin_std": 38.021060943603516,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -12.55882453918457,
      "KL/mean": -23.864208221435547,
      "KL/rejected_KL_mean": -35.169586181640625,
      "KL/std": 31.694580078125,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.011727683246135712,
      "fcm_dpo/delta": 0.001280205324292183,
      "fcm_dpo/margin": 22.610763549804688,
      "fcm_dpo/q_t": 0.4372168481349945,
      "grad_norm": 35.04063034057617,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": 1.6383016109466553,
      "logits/rejected": 1.6784754991531372,
      "logps/chosen": -304.2838134765625,
      "logps/ref_chosen": -291.7249755859375,
      "logps/ref_rejected": -273.54010009765625,
      "logps/rejected": -308.7096862792969,
      "loss": 4.7448,
      "margin_dpo/margin_mean": 22.610763549804688,
      "margin_dpo/margin_std": 36.146488189697266,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -11.539850234985352,
      "KL/mean": -22.910442352294922,
      "KL/rejected_KL_mean": -34.281028747558594,
      "KL/std": 33.96072769165039,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.011726637370884418,
      "fcm_dpo/delta": -0.0006737913936376572,
      "fcm_dpo/margin": 22.741168975830078,
      "fcm_dpo/q_t": 0.43786221742630005,
      "grad_norm": 28.08049201965332,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": 1.6483415365219116,
      "logits/rejected": 1.6911652088165283,
      "logps/chosen": -333.2417297363281,
      "logps/ref_chosen": -321.70184326171875,
      "logps/ref_rejected": -261.8386535644531,
      "logps/rejected": -296.1197204589844,
      "loss": 4.8191,
      "margin_dpo/margin_mean": 22.74117088317871,
      "margin_dpo/margin_std": 44.27610778808594,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -13.493354797363281,
      "KL/mean": -25.251081466674805,
      "KL/rejected_KL_mean": -37.008811950683594,
      "KL/std": 33.3421630859375,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.011816182173788548,
      "fcm_dpo/delta": 0.00955005083233118,
      "fcm_dpo/margin": 23.515460968017578,
      "fcm_dpo/q_t": 0.43458616733551025,
      "grad_norm": 41.97201156616211,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": 1.3766467571258545,
      "logits/rejected": 1.3482625484466553,
      "logps/chosen": -269.4087829589844,
      "logps/ref_chosen": -255.91542053222656,
      "logps/ref_rejected": -245.1119842529297,
      "logps/rejected": -282.12078857421875,
      "loss": 4.7771,
      "margin_dpo/margin_mean": 23.515459060668945,
      "margin_dpo/margin_std": 42.94520568847656,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -4.988622665405273,
      "KL/mean": -19.48944664001465,
      "KL/rejected_KL_mean": -33.990272521972656,
      "KL/std": 33.57756805419922,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.011782538145780563,
      "fcm_dpo/delta": -0.010233163833618164,
      "fcm_dpo/margin": 29.001649856567383,
      "fcm_dpo/q_t": 0.41916242241859436,
      "grad_norm": 35.62556076049805,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": 1.4420344829559326,
      "logits/rejected": 1.472320556640625,
      "logps/chosen": -271.0071716308594,
      "logps/ref_chosen": -266.0185546875,
      "logps/ref_rejected": -272.862548828125,
      "logps/rejected": -306.85284423828125,
      "loss": 4.4807,
      "margin_dpo/margin_mean": 29.001649856567383,
      "margin_dpo/margin_std": 35.6319694519043,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -6.430919647216797,
      "KL/mean": -20.365341186523438,
      "KL/rejected_KL_mean": -34.29975128173828,
      "KL/std": 33.77674102783203,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.011748439632356167,
      "fcm_dpo/delta": 0.016459988430142403,
      "fcm_dpo/margin": 27.868839263916016,
      "fcm_dpo/q_t": 0.4228605031967163,
      "grad_norm": 29.87738800048828,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 1.359779953956604,
      "logits/rejected": 1.292455792427063,
      "logps/chosen": -256.5819091796875,
      "logps/ref_chosen": -250.15098571777344,
      "logps/ref_rejected": -223.34393310546875,
      "logps/rejected": -257.6436767578125,
      "loss": 4.5769,
      "margin_dpo/margin_mean": 27.868839263916016,
      "margin_dpo/margin_std": 41.007022857666016,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -3.902681350708008,
      "KL/mean": -17.21375274658203,
      "KL/rejected_KL_mean": -30.524818420410156,
      "KL/std": 34.98585510253906,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.011760472320020199,
      "fcm_dpo/delta": -0.0010248874314129353,
      "fcm_dpo/margin": 26.622127532958984,
      "fcm_dpo/q_t": 0.42690107226371765,
      "grad_norm": 27.49928855895996,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 1.2241400480270386,
      "logits/rejected": 1.2827937602996826,
      "logps/chosen": -242.2944793701172,
      "logps/ref_chosen": -238.3917999267578,
      "logps/ref_rejected": -243.70272827148438,
      "logps/rejected": -274.2275695800781,
      "loss": 4.6582,
      "margin_dpo/margin_mean": 26.622127532958984,
      "margin_dpo/margin_std": 43.611629486083984,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -10.263917922973633,
      "KL/mean": -20.438392639160156,
      "KL/rejected_KL_mean": -30.612865447998047,
      "KL/std": 34.654441833496094,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.01176578737795353,
      "fcm_dpo/delta": -0.002903435379266739,
      "fcm_dpo/margin": 20.34895133972168,
      "fcm_dpo/q_t": 0.44343268871307373,
      "grad_norm": 30.37371253967285,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.4824174642562866,
      "logits/rejected": 1.5572298765182495,
      "logps/chosen": -266.9009094238281,
      "logps/ref_chosen": -256.6369934082031,
      "logps/ref_rejected": -256.8894348144531,
      "logps/rejected": -287.5022888183594,
      "loss": 4.9345,
      "margin_dpo/margin_mean": 20.34895133972168,
      "margin_dpo/margin_std": 43.88816833496094,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -11.934091567993164,
      "KL/mean": -23.364789962768555,
      "KL/rejected_KL_mean": -34.79547882080078,
      "KL/std": 36.13188552856445,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.011913150548934937,
      "fcm_dpo/delta": 0.013051112182438374,
      "fcm_dpo/margin": 22.861391067504883,
      "fcm_dpo/q_t": 0.4363504648208618,
      "grad_norm": 31.75264549255371,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": 1.694105625152588,
      "logits/rejected": 1.793583869934082,
      "logps/chosen": -284.7337646484375,
      "logps/ref_chosen": -272.7996826171875,
      "logps/ref_rejected": -289.2391052246094,
      "logps/rejected": -324.0345458984375,
      "loss": 4.7814,
      "margin_dpo/margin_mean": 22.861392974853516,
      "margin_dpo/margin_std": 42.15224075317383,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -10.382034301757812,
      "KL/mean": -19.38272476196289,
      "KL/rejected_KL_mean": -28.383407592773438,
      "KL/std": 36.093116760253906,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.011896221898496151,
      "fcm_dpo/delta": -0.002824758645147085,
      "fcm_dpo/margin": 18.001379013061523,
      "fcm_dpo/q_t": 0.4503095746040344,
      "grad_norm": 66.98899841308594,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": 1.5177068710327148,
      "logits/rejected": 1.574666142463684,
      "logps/chosen": -291.69110107421875,
      "logps/ref_chosen": -281.30902099609375,
      "logps/ref_rejected": -246.3243408203125,
      "logps/rejected": -274.7077941894531,
      "loss": 4.9821,
      "margin_dpo/margin_mean": 18.001379013061523,
      "margin_dpo/margin_std": 40.663963317871094,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -11.31611442565918,
      "KL/mean": -20.089797973632812,
      "KL/rejected_KL_mean": -28.86347198486328,
      "KL/std": 35.59619140625,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.011999674141407013,
      "fcm_dpo/delta": 0.02612481452524662,
      "fcm_dpo/margin": 17.54735565185547,
      "fcm_dpo/q_t": 0.4508902430534363,
      "grad_norm": 30.195571899414062,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": 1.6706976890563965,
      "logits/rejected": 1.7804713249206543,
      "logps/chosen": -278.1742248535156,
      "logps/ref_chosen": -266.8581237792969,
      "logps/ref_rejected": -276.0855712890625,
      "logps/rejected": -304.9490051269531,
      "loss": 5.0219,
      "margin_dpo/margin_mean": 17.5473575592041,
      "margin_dpo/margin_std": 43.244049072265625,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -1.975687026977539,
      "KL/mean": -16.805191040039062,
      "KL/rejected_KL_mean": -31.634700775146484,
      "KL/std": 34.10515213012695,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.012013810686767101,
      "fcm_dpo/delta": -0.029074227437376976,
      "fcm_dpo/margin": 29.659008026123047,
      "fcm_dpo/q_t": 0.41625940799713135,
      "grad_norm": 27.682371139526367,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": 1.3193668127059937,
      "logits/rejected": 1.4246602058410645,
      "logps/chosen": -268.97064208984375,
      "logps/ref_chosen": -266.99493408203125,
      "logps/ref_rejected": -248.60513305664062,
      "logps/rejected": -280.2398681640625,
      "loss": 4.4588,
      "margin_dpo/margin_mean": 29.659008026123047,
      "margin_dpo/margin_std": 37.662452697753906,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -0.9976558685302734,
      "KL/mean": -19.66681480407715,
      "KL/rejected_KL_mean": -38.335968017578125,
      "KL/std": 37.64072799682617,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.011545807123184204,
      "fcm_dpo/delta": -0.08687985688447952,
      "fcm_dpo/margin": 37.33831024169922,
      "fcm_dpo/q_t": 0.39866381883621216,
      "grad_norm": 31.51327896118164,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": 1.4887365102767944,
      "logits/rejected": 1.4710103273391724,
      "logps/chosen": -303.9874572753906,
      "logps/ref_chosen": -302.98980712890625,
      "logps/ref_rejected": -262.8002624511719,
      "logps/rejected": -301.1361999511719,
      "loss": 4.2618,
      "margin_dpo/margin_mean": 37.33830261230469,
      "margin_dpo/margin_std": 41.389320373535156,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -2.462800979614258,
      "KL/mean": -16.808237075805664,
      "KL/rejected_KL_mean": -31.15367317199707,
      "KL/std": 37.65160369873047,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.011047590523958206,
      "fcm_dpo/delta": 0.008029544726014137,
      "fcm_dpo/margin": 28.69086265563965,
      "fcm_dpo/q_t": 0.42580804228782654,
      "grad_norm": 38.539817810058594,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": 1.3882038593292236,
      "logits/rejected": 1.4700844287872314,
      "logps/chosen": -298.6745910644531,
      "logps/ref_chosen": -296.2117614746094,
      "logps/ref_rejected": -289.8357238769531,
      "logps/rejected": -320.9893798828125,
      "loss": 4.6573,
      "margin_dpo/margin_mean": 28.690860748291016,
      "margin_dpo/margin_std": 47.33806228637695,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -3.9134578704833984,
      "KL/mean": -17.417110443115234,
      "KL/rejected_KL_mean": -30.92074966430664,
      "KL/std": 35.111961364746094,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.011411065235733986,
      "fcm_dpo/delta": 0.03827835991978645,
      "fcm_dpo/margin": 27.007282257080078,
      "fcm_dpo/q_t": 0.42802032828330994,
      "grad_norm": 33.73345184326172,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": 1.3910315036773682,
      "logits/rejected": 1.3346062898635864,
      "logps/chosen": -259.2060852050781,
      "logps/ref_chosen": -255.29263305664062,
      "logps/ref_rejected": -212.69761657714844,
      "logps/rejected": -243.6183624267578,
      "loss": 4.6616,
      "margin_dpo/margin_mean": 27.007282257080078,
      "margin_dpo/margin_std": 43.80791473388672,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -2.7024612426757812,
      "KL/mean": -14.631046295166016,
      "KL/rejected_KL_mean": -26.55961799621582,
      "KL/std": 36.54867172241211,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.011494940146803856,
      "fcm_dpo/delta": -0.011354226619005203,
      "fcm_dpo/margin": 23.857166290283203,
      "fcm_dpo/q_t": 0.43678539991378784,
      "grad_norm": 36.07775115966797,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 1.6387319564819336,
      "logits/rejected": 1.63539457321167,
      "logps/chosen": -316.2388610839844,
      "logps/ref_chosen": -313.5364074707031,
      "logps/ref_rejected": -274.349609375,
      "logps/rejected": -300.9092102050781,
      "loss": 4.8732,
      "margin_dpo/margin_mean": 23.857166290283203,
      "margin_dpo/margin_std": 51.12529373168945,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_KL/chosen_KL_mean": -7.609809398651123,
      "eval_KL/mean": -20.614116668701172,
      "eval_KL/rejected_KL_mean": -33.61842346191406,
      "eval_KL/std": 36.49480438232422,
      "eval_fcm_dpo/beta": 0.01139148510992527,
      "eval_logits/chosen": 1.570547342300415,
      "eval_logits/rejected": 1.6149033308029175,
      "eval_logps/chosen": -288.0264892578125,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -298.2639465332031,
      "eval_loss": 0.5899988412857056,
      "eval_margin_dpo/margin_mean": 26.008609771728516,
      "eval_margin_dpo/margin_std": 42.729888916015625,
      "eval_runtime": 88.313,
      "eval_samples_per_second": 22.647,
      "eval_steps_per_second": 2.831,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -6.287517547607422,
      "KL/mean": -20.21714210510254,
      "KL/rejected_KL_mean": -34.14675521850586,
      "KL/std": 37.49905014038086,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.010093741118907928,
      "fcm_dpo/delta": 0.011817049235105515,
      "fcm_dpo/margin": 27.85923194885254,
      "fcm_dpo/q_t": 0.4329674541950226,
      "grad_norm": 27.89253807067871,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 1.299346923828125,
      "logits/rejected": 1.3348581790924072,
      "logps/chosen": -268.9947204589844,
      "logps/ref_chosen": -262.7071838378906,
      "logps/ref_rejected": -243.85879516601562,
      "logps/rejected": -278.00555419921875,
      "loss": 4.6903,
      "margin_dpo/margin_mean": 27.85923194885254,
      "margin_dpo/margin_std": 42.519535064697266,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -9.550416946411133,
      "KL/mean": -23.203289031982422,
      "KL/rejected_KL_mean": -36.85615539550781,
      "KL/std": 37.85678482055664,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.01010669581592083,
      "fcm_dpo/delta": 0.0020130672492086887,
      "fcm_dpo/margin": 27.30573272705078,
      "fcm_dpo/q_t": 0.4349828362464905,
      "grad_norm": 36.384613037109375,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": 1.4864568710327148,
      "logits/rejected": 1.4612295627593994,
      "logps/chosen": -273.837158203125,
      "logps/ref_chosen": -264.2867431640625,
      "logps/ref_rejected": -231.6894989013672,
      "logps/rejected": -268.545654296875,
      "loss": 4.7733,
      "margin_dpo/margin_mean": 27.30573272705078,
      "margin_dpo/margin_std": 48.64140319824219,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -7.88517951965332,
      "KL/mean": -18.981632232666016,
      "KL/rejected_KL_mean": -30.078083038330078,
      "KL/std": 36.3074951171875,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.010154437273740768,
      "fcm_dpo/delta": 0.01545935869216919,
      "fcm_dpo/margin": 22.192907333374023,
      "fcm_dpo/q_t": 0.44609302282333374,
      "grad_norm": 30.40738296508789,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": 1.518162488937378,
      "logits/rejected": 1.4964696168899536,
      "logps/chosen": -279.16937255859375,
      "logps/ref_chosen": -271.2842102050781,
      "logps/ref_rejected": -247.95462036132812,
      "logps/rejected": -278.03271484375,
      "loss": 4.8466,
      "margin_dpo/margin_mean": 22.192909240722656,
      "margin_dpo/margin_std": 37.68293762207031,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -5.86326789855957,
      "KL/mean": -21.984228134155273,
      "KL/rejected_KL_mean": -38.10517120361328,
      "KL/std": 37.07770538330078,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.010455988347530365,
      "fcm_dpo/delta": 0.04793383553624153,
      "fcm_dpo/margin": 32.241905212402344,
      "fcm_dpo/q_t": 0.42106541991233826,
      "grad_norm": 31.462316513061523,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": 1.5231130123138428,
      "logits/rejected": 1.4950542449951172,
      "logps/chosen": -315.99371337890625,
      "logps/ref_chosen": -310.13043212890625,
      "logps/ref_rejected": -267.3905334472656,
      "logps/rejected": -305.49566650390625,
      "loss": 4.5267,
      "margin_dpo/margin_mean": 32.241905212402344,
      "margin_dpo/margin_std": 44.5057258605957,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -4.685155868530273,
      "KL/mean": -19.366838455200195,
      "KL/rejected_KL_mean": -34.04852294921875,
      "KL/std": 37.388946533203125,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.010767693631350994,
      "fcm_dpo/delta": 0.01077722292393446,
      "fcm_dpo/margin": 29.363372802734375,
      "fcm_dpo/q_t": 0.42571812868118286,
      "grad_norm": 32.622413635253906,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 1.4032535552978516,
      "logits/rejected": 1.4036904573440552,
      "logps/chosen": -281.8458251953125,
      "logps/ref_chosen": -277.16064453125,
      "logps/ref_rejected": -292.91815185546875,
      "logps/rejected": -326.9667053222656,
      "loss": 4.6497,
      "margin_dpo/margin_mean": 29.363372802734375,
      "margin_dpo/margin_std": 46.61855697631836,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -3.531536102294922,
      "KL/mean": -17.947568893432617,
      "KL/rejected_KL_mean": -32.363609313964844,
      "KL/std": 37.58164978027344,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.010654217563569546,
      "fcm_dpo/delta": -0.005246859043836594,
      "fcm_dpo/margin": 28.832061767578125,
      "fcm_dpo/q_t": 0.4274274706840515,
      "grad_norm": 35.82722854614258,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": 1.5636481046676636,
      "logits/rejected": 1.6762700080871582,
      "logps/chosen": -290.3141174316406,
      "logps/ref_chosen": -286.7825927734375,
      "logps/ref_rejected": -289.08270263671875,
      "logps/rejected": -321.4462890625,
      "loss": 4.6506,
      "margin_dpo/margin_mean": 28.832061767578125,
      "margin_dpo/margin_std": 45.128360748291016,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -6.805263519287109,
      "KL/mean": -19.03407859802246,
      "KL/rejected_KL_mean": -31.262893676757812,
      "KL/std": 39.583213806152344,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.010667615570127964,
      "fcm_dpo/delta": 0.0013164784759283066,
      "fcm_dpo/margin": 24.45763397216797,
      "fcm_dpo/q_t": 0.43955904245376587,
      "grad_norm": 31.435632705688477,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": 1.3648594617843628,
      "logits/rejected": 1.4722827672958374,
      "logps/chosen": -273.3375244140625,
      "logps/ref_chosen": -266.5322570800781,
      "logps/ref_rejected": -275.20672607421875,
      "logps/rejected": -306.4696044921875,
      "loss": 4.875,
      "margin_dpo/margin_mean": 24.457632064819336,
      "margin_dpo/margin_std": 51.05473327636719,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -4.7914581298828125,
      "KL/mean": -18.980504989624023,
      "KL/rejected_KL_mean": -33.1695671081543,
      "KL/std": 38.462303161621094,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.01064885314553976,
      "fcm_dpo/delta": -0.0029236297123134136,
      "fcm_dpo/margin": 28.37810516357422,
      "fcm_dpo/q_t": 0.42898520827293396,
      "grad_norm": 66.4640121459961,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": 1.7078354358673096,
      "logits/rejected": 1.7187843322753906,
      "logps/chosen": -277.5928039550781,
      "logps/ref_chosen": -272.80133056640625,
      "logps/ref_rejected": -288.4167785644531,
      "logps/rejected": -321.5863037109375,
      "loss": 4.6562,
      "margin_dpo/margin_mean": 28.37810516357422,
      "margin_dpo/margin_std": 44.100303649902344,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -10.320882797241211,
      "KL/mean": -21.551101684570312,
      "KL/rejected_KL_mean": -32.781333923339844,
      "KL/std": 38.39013671875,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.010696676559746265,
      "fcm_dpo/delta": 0.020279550924897194,
      "fcm_dpo/margin": 22.460460662841797,
      "fcm_dpo/q_t": 0.44404229521751404,
      "grad_norm": 31.64073944091797,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": 1.5672868490219116,
      "logits/rejected": 1.4325628280639648,
      "logps/chosen": -321.00030517578125,
      "logps/ref_chosen": -310.679443359375,
      "logps/ref_rejected": -259.3792419433594,
      "logps/rejected": -292.1605529785156,
      "loss": 4.8885,
      "margin_dpo/margin_mean": 22.460460662841797,
      "margin_dpo/margin_std": 44.377403259277344,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -4.320219039916992,
      "KL/mean": -18.66399574279785,
      "KL/rejected_KL_mean": -33.00777053833008,
      "KL/std": 39.27182388305664,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.010705474764108658,
      "fcm_dpo/delta": -0.014603743329644203,
      "fcm_dpo/margin": 28.68754005432129,
      "fcm_dpo/q_t": 0.42766663432121277,
      "grad_norm": 30.856002807617188,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 1.3609716892242432,
      "logits/rejected": 1.404768705368042,
      "logps/chosen": -279.2883605957031,
      "logps/ref_chosen": -274.9681701660156,
      "logps/ref_rejected": -283.346923828125,
      "logps/rejected": -316.3547058105469,
      "loss": 4.6582,
      "margin_dpo/margin_mean": 28.687538146972656,
      "margin_dpo/margin_std": 45.193241119384766,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": 0.6788368225097656,
      "KL/mean": -18.199113845825195,
      "KL/rejected_KL_mean": -37.077064514160156,
      "KL/std": 38.437313079833984,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.010514242574572563,
      "fcm_dpo/delta": -0.04467291757464409,
      "fcm_dpo/margin": 37.75589370727539,
      "fcm_dpo/q_t": 0.4062609374523163,
      "grad_norm": 29.043231964111328,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.4237481355667114,
      "logits/rejected": 1.4189656972885132,
      "logps/chosen": -265.60076904296875,
      "logps/ref_chosen": -266.2795715332031,
      "logps/ref_rejected": -249.7120819091797,
      "logps/rejected": -286.7891845703125,
      "loss": 4.3151,
      "margin_dpo/margin_mean": 37.75589370727539,
      "margin_dpo/margin_std": 41.18382263183594,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -4.289358139038086,
      "KL/mean": -17.93153953552246,
      "KL/rejected_KL_mean": -31.573707580566406,
      "KL/std": 38.363670349121094,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.01049887202680111,
      "fcm_dpo/delta": 0.03350698947906494,
      "fcm_dpo/margin": 27.28434181213379,
      "fcm_dpo/q_t": 0.4323800504207611,
      "grad_norm": 54.8814697265625,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": 1.3459088802337646,
      "logits/rejected": 1.3788813352584839,
      "logps/chosen": -269.0422058105469,
      "logps/ref_chosen": -264.7528076171875,
      "logps/ref_rejected": -241.02842712402344,
      "logps/rejected": -272.6021423339844,
      "loss": 4.7219,
      "margin_dpo/margin_mean": 27.284339904785156,
      "margin_dpo/margin_std": 45.9613151550293,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -7.899026870727539,
      "KL/mean": -21.52096939086914,
      "KL/rejected_KL_mean": -35.142913818359375,
      "KL/std": 39.09897232055664,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.010641629807651043,
      "fcm_dpo/delta": 0.005819341167807579,
      "fcm_dpo/margin": 27.243886947631836,
      "fcm_dpo/q_t": 0.433135986328125,
      "grad_norm": 36.383636474609375,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": 1.6178821325302124,
      "logits/rejected": 1.7444372177124023,
      "logps/chosen": -302.0376281738281,
      "logps/ref_chosen": -294.13861083984375,
      "logps/ref_rejected": -303.11669921875,
      "logps/rejected": -338.2596435546875,
      "loss": 4.7104,
      "margin_dpo/margin_mean": 27.2438907623291,
      "margin_dpo/margin_std": 45.517356872558594,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -17.10884666442871,
      "KL/mean": -29.23906707763672,
      "KL/rejected_KL_mean": -41.36928939819336,
      "KL/std": 37.78112030029297,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.01067170687019825,
      "fcm_dpo/delta": -0.01007808931171894,
      "fcm_dpo/margin": 24.260435104370117,
      "fcm_dpo/q_t": 0.4394315779209137,
      "grad_norm": 35.85784149169922,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": 1.4514799118041992,
      "logits/rejected": 1.6016725301742554,
      "logps/chosen": -283.68560791015625,
      "logps/ref_chosen": -266.5767517089844,
      "logps/ref_rejected": -291.2264709472656,
      "logps/rejected": -332.59576416015625,
      "loss": 4.8192,
      "margin_dpo/margin_mean": 24.260435104370117,
      "margin_dpo/margin_std": 45.26163864135742,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -12.411754608154297,
      "KL/mean": -25.089853286743164,
      "KL/rejected_KL_mean": -37.76795959472656,
      "KL/std": 41.92486572265625,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.010495892725884914,
      "fcm_dpo/delta": -0.019128231331706047,
      "fcm_dpo/margin": 25.3562068939209,
      "fcm_dpo/q_t": 0.4382995665073395,
      "grad_norm": 32.44038772583008,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": 1.5334930419921875,
      "logits/rejected": 1.5429962873458862,
      "logps/chosen": -297.4525146484375,
      "logps/ref_chosen": -285.040771484375,
      "logps/ref_rejected": -258.0254821777344,
      "logps/rejected": -295.79345703125,
      "loss": 4.824,
      "margin_dpo/margin_mean": 25.35620880126953,
      "margin_dpo/margin_std": 48.848716735839844,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -13.413309097290039,
      "KL/mean": -27.722599029541016,
      "KL/rejected_KL_mean": -42.03188705444336,
      "KL/std": 42.903316497802734,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.010467009618878365,
      "fcm_dpo/delta": 0.0074461232870817184,
      "fcm_dpo/margin": 28.61859130859375,
      "fcm_dpo/q_t": 0.42952966690063477,
      "grad_norm": 43.95420455932617,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": 1.4845691919326782,
      "logits/rejected": 1.4927157163619995,
      "logps/chosen": -346.1647033691406,
      "logps/ref_chosen": -332.7513732910156,
      "logps/ref_rejected": -302.9167785644531,
      "logps/rejected": -344.9486083984375,
      "loss": 4.7458,
      "margin_dpo/margin_mean": 28.61859130859375,
      "margin_dpo/margin_std": 51.66400146484375,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -14.372535705566406,
      "KL/mean": -33.50685501098633,
      "KL/rejected_KL_mean": -52.641170501708984,
      "KL/std": 42.18860626220703,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.010430972091853619,
      "fcm_dpo/delta": -0.011273941956460476,
      "fcm_dpo/margin": 38.26862716674805,
      "fcm_dpo/q_t": 0.4081307053565979,
      "grad_norm": 26.254362106323242,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": 1.5874731540679932,
      "logits/rejected": 1.5897008180618286,
      "logps/chosen": -300.8404846191406,
      "logps/ref_chosen": -286.46795654296875,
      "logps/ref_rejected": -253.99298095703125,
      "logps/rejected": -306.6341552734375,
      "loss": 4.3901,
      "margin_dpo/margin_mean": 38.26862716674805,
      "margin_dpo/margin_std": 51.76972198486328,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -10.903087615966797,
      "KL/mean": -26.180707931518555,
      "KL/rejected_KL_mean": -41.45832061767578,
      "KL/std": 42.05992126464844,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.010443596169352531,
      "fcm_dpo/delta": 0.011297769844532013,
      "fcm_dpo/margin": 30.555225372314453,
      "fcm_dpo/q_t": 0.42514780163764954,
      "grad_norm": 49.15101623535156,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 1.5880109071731567,
      "logits/rejected": 1.6512058973312378,
      "logps/chosen": -322.2816162109375,
      "logps/ref_chosen": -311.3785400390625,
      "logps/ref_rejected": -290.3092956542969,
      "logps/rejected": -331.7676086425781,
      "loss": 4.61,
      "margin_dpo/margin_mean": 30.55522918701172,
      "margin_dpo/margin_std": 46.25550079345703,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -10.934755325317383,
      "KL/mean": -26.61871337890625,
      "KL/rejected_KL_mean": -42.30267333984375,
      "KL/std": 40.274574279785156,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.010494871065020561,
      "fcm_dpo/delta": 0.007473754696547985,
      "fcm_dpo/margin": 31.36792755126953,
      "fcm_dpo/q_t": 0.42262107133865356,
      "grad_norm": 34.781063079833984,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": 1.323378324508667,
      "logits/rejected": 1.3844687938690186,
      "logps/chosen": -270.2676696777344,
      "logps/ref_chosen": -259.33294677734375,
      "logps/ref_rejected": -261.7661437988281,
      "logps/rejected": -304.06878662109375,
      "loss": 4.5971,
      "margin_dpo/margin_mean": 31.36792755126953,
      "margin_dpo/margin_std": 48.2261962890625,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -12.78665542602539,
      "KL/mean": -29.74393081665039,
      "KL/rejected_KL_mean": -46.701202392578125,
      "KL/std": 42.528141021728516,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.010664639994502068,
      "fcm_dpo/delta": 0.026440581306815147,
      "fcm_dpo/margin": 33.91455841064453,
      "fcm_dpo/q_t": 0.41586917638778687,
      "grad_norm": 29.507923126220703,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 1.6367168426513672,
      "logits/rejected": 1.5805543661117554,
      "logps/chosen": -318.99102783203125,
      "logps/ref_chosen": -306.2043762207031,
      "logps/ref_rejected": -256.7926330566406,
      "logps/rejected": -303.4938659667969,
      "loss": 4.5171,
      "margin_dpo/margin_mean": 33.9145622253418,
      "margin_dpo/margin_std": 50.79574966430664,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -15.630537033081055,
      "KL/mean": -28.332603454589844,
      "KL/rejected_KL_mean": -41.03468322753906,
      "KL/std": 42.38873291015625,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.01087187509983778,
      "fcm_dpo/delta": 0.000987144187092781,
      "fcm_dpo/margin": 25.404138565063477,
      "fcm_dpo/q_t": 0.43524104356765747,
      "grad_norm": 33.557979583740234,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 1.19321608543396,
      "logits/rejected": 1.309771180152893,
      "logps/chosen": -308.96246337890625,
      "logps/ref_chosen": -293.3319396972656,
      "logps/ref_rejected": -282.7599182128906,
      "logps/rejected": -323.79461669921875,
      "loss": 4.8104,
      "margin_dpo/margin_mean": 25.404138565063477,
      "margin_dpo/margin_std": 48.518733978271484,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -12.58218765258789,
      "KL/mean": -29.292984008789062,
      "KL/rejected_KL_mean": -46.0037841796875,
      "KL/std": 40.75558090209961,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.010592925362288952,
      "fcm_dpo/delta": -0.03296639025211334,
      "fcm_dpo/margin": 33.42159652709961,
      "fcm_dpo/q_t": 0.4167301058769226,
      "grad_norm": 42.25018310546875,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": 1.514768362045288,
      "logits/rejected": 1.4268888235092163,
      "logps/chosen": -299.58831787109375,
      "logps/ref_chosen": -287.0061340332031,
      "logps/ref_rejected": -233.23130798339844,
      "logps/rejected": -279.2350769042969,
      "loss": 4.4973,
      "margin_dpo/margin_mean": 33.42159652709961,
      "margin_dpo/margin_std": 46.07832717895508,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -14.424854278564453,
      "KL/mean": -32.933990478515625,
      "KL/rejected_KL_mean": -51.443119049072266,
      "KL/std": 44.27592468261719,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.010267972014844418,
      "fcm_dpo/delta": -0.04338767006993294,
      "fcm_dpo/margin": 37.01826858520508,
      "fcm_dpo/q_t": 0.41123396158218384,
      "grad_norm": 30.563940048217773,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": 1.439980149269104,
      "logits/rejected": 1.3428581953048706,
      "logps/chosen": -290.2401428222656,
      "logps/ref_chosen": -275.8153076171875,
      "logps/ref_rejected": -238.54286193847656,
      "logps/rejected": -289.9859619140625,
      "loss": 4.4495,
      "margin_dpo/margin_mean": 37.01826477050781,
      "margin_dpo/margin_std": 51.33940887451172,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -13.980792999267578,
      "KL/mean": -32.915870666503906,
      "KL/rejected_KL_mean": -51.8509521484375,
      "KL/std": 42.56742858886719,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.010069815441966057,
      "fcm_dpo/delta": -0.0377618670463562,
      "fcm_dpo/margin": 37.87016296386719,
      "fcm_dpo/q_t": 0.4102567434310913,
      "grad_norm": 33.127132415771484,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": 1.363016963005066,
      "logits/rejected": 1.4268131256103516,
      "logps/chosen": -253.0811767578125,
      "logps/ref_chosen": -239.1003875732422,
      "logps/ref_rejected": -230.31900024414062,
      "logps/rejected": -282.1699523925781,
      "loss": 4.3921,
      "margin_dpo/margin_mean": 37.87016296386719,
      "margin_dpo/margin_std": 46.051780700683594,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -24.11736488342285,
      "KL/mean": -37.37209701538086,
      "KL/rejected_KL_mean": -50.626827239990234,
      "KL/std": 43.44748306274414,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.009877342730760574,
      "fcm_dpo/delta": 0.009139633737504482,
      "fcm_dpo/margin": 26.50945472717285,
      "fcm_dpo/q_t": 0.4397391974925995,
      "grad_norm": 33.3519172668457,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": 1.5566563606262207,
      "logits/rejected": 1.5831375122070312,
      "logps/chosen": -324.44671630859375,
      "logps/ref_chosen": -300.3293762207031,
      "logps/ref_rejected": -332.132568359375,
      "logps/rejected": -382.7593688964844,
      "loss": 4.8845,
      "margin_dpo/margin_mean": 26.50945472717285,
      "margin_dpo/margin_std": 56.77092742919922,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -19.329978942871094,
      "KL/mean": -31.918411254882812,
      "KL/rejected_KL_mean": -44.506839752197266,
      "KL/std": 41.845584869384766,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.010124340653419495,
      "fcm_dpo/delta": 0.03310726583003998,
      "fcm_dpo/margin": 25.17685890197754,
      "fcm_dpo/q_t": 0.44075173139572144,
      "grad_norm": 48.522830963134766,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": 1.5398565530776978,
      "logits/rejected": 1.6094462871551514,
      "logps/chosen": -264.74212646484375,
      "logps/ref_chosen": -245.41213989257812,
      "logps/ref_rejected": -251.72999572753906,
      "logps/rejected": -296.23681640625,
      "loss": 4.8378,
      "margin_dpo/margin_mean": 25.17685890197754,
      "margin_dpo/margin_std": 48.57210922241211,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -17.239002227783203,
      "KL/mean": -36.159061431884766,
      "KL/rejected_KL_mean": -55.07910919189453,
      "KL/std": 44.369693756103516,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.010069957002997398,
      "fcm_dpo/delta": -0.007162913680076599,
      "fcm_dpo/margin": 37.84011459350586,
      "fcm_dpo/q_t": 0.41100072860717773,
      "grad_norm": 34.61928176879883,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": 1.1706323623657227,
      "logits/rejected": 1.205276370048523,
      "logps/chosen": -294.11480712890625,
      "logps/ref_chosen": -276.87579345703125,
      "logps/ref_rejected": -259.0455017089844,
      "logps/rejected": -314.1246032714844,
      "loss": 4.4351,
      "margin_dpo/margin_mean": 37.84011459350586,
      "margin_dpo/margin_std": 51.40252685546875,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -15.96080207824707,
      "KL/mean": -34.57950210571289,
      "KL/rejected_KL_mean": -53.19819641113281,
      "KL/std": 43.23419189453125,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.010071149095892906,
      "fcm_dpo/delta": -0.030604416504502296,
      "fcm_dpo/margin": 37.237396240234375,
      "fcm_dpo/q_t": 0.4125247299671173,
      "grad_norm": 46.97827911376953,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": 1.5880306959152222,
      "logits/rejected": 1.5945672988891602,
      "logps/chosen": -319.9318542480469,
      "logps/ref_chosen": -303.9710693359375,
      "logps/ref_rejected": -255.62033081054688,
      "logps/rejected": -308.8185119628906,
      "loss": 4.4397,
      "margin_dpo/margin_mean": 37.23739242553711,
      "margin_dpo/margin_std": 49.701683044433594,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -17.193899154663086,
      "KL/mean": -40.445411682128906,
      "KL/rejected_KL_mean": -63.696922302246094,
      "KL/std": 45.4671516418457,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.009639523923397064,
      "fcm_dpo/delta": -0.05153639614582062,
      "fcm_dpo/margin": 46.503013610839844,
      "fcm_dpo/q_t": 0.395062118768692,
      "grad_norm": 42.260986328125,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": 1.4046367406845093,
      "logits/rejected": 1.399046540260315,
      "logps/chosen": -298.88983154296875,
      "logps/ref_chosen": -281.6959533691406,
      "logps/ref_rejected": -258.93353271484375,
      "logps/rejected": -322.6304931640625,
      "loss": 4.2127,
      "margin_dpo/margin_mean": 46.503013610839844,
      "margin_dpo/margin_std": 51.82863235473633,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -18.450824737548828,
      "KL/mean": -35.77487564086914,
      "KL/rejected_KL_mean": -53.09892272949219,
      "KL/std": 44.0614013671875,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.009551331400871277,
      "fcm_dpo/delta": 0.002714039757847786,
      "fcm_dpo/margin": 34.64807891845703,
      "fcm_dpo/q_t": 0.42234984040260315,
      "grad_norm": 43.14898681640625,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 1.1032363176345825,
      "logits/rejected": 1.189947485923767,
      "logps/chosen": -242.40736389160156,
      "logps/ref_chosen": -223.95655822753906,
      "logps/ref_rejected": -238.95187377929688,
      "logps/rejected": -292.05078125,
      "loss": 4.5914,
      "margin_dpo/margin_mean": 34.64807891845703,
      "margin_dpo/margin_std": 52.82122802734375,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -25.962800979614258,
      "KL/mean": -40.51888656616211,
      "KL/rejected_KL_mean": -55.07497024536133,
      "KL/std": 43.152706146240234,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.009528500959277153,
      "fcm_dpo/delta": 0.014889342710375786,
      "fcm_dpo/margin": 29.112167358398438,
      "fcm_dpo/q_t": 0.43444520235061646,
      "grad_norm": 40.90285873413086,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 1.2657538652420044,
      "logits/rejected": 1.3211772441864014,
      "logps/chosen": -340.8874206542969,
      "logps/ref_chosen": -314.92462158203125,
      "logps/ref_rejected": -279.6347351074219,
      "logps/rejected": -334.7096862792969,
      "loss": 4.7256,
      "margin_dpo/margin_mean": 29.112167358398438,
      "margin_dpo/margin_std": 47.383827209472656,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -15.167354583740234,
      "KL/mean": -32.18161392211914,
      "KL/rejected_KL_mean": -49.19587326049805,
      "KL/std": 46.40901184082031,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.009547957219183445,
      "fcm_dpo/delta": -0.009836734272539616,
      "fcm_dpo/margin": 34.02851867675781,
      "fcm_dpo/q_t": 0.4243396818637848,
      "grad_norm": 28.843780517578125,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": 1.3943054676055908,
      "logits/rejected": 1.4602539539337158,
      "logps/chosen": -278.4869689941406,
      "logps/ref_chosen": -263.3196105957031,
      "logps/ref_rejected": -247.13612365722656,
      "logps/rejected": -296.3320007324219,
      "loss": 4.6009,
      "margin_dpo/margin_mean": 34.02851867675781,
      "margin_dpo/margin_std": 51.41886520385742,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -16.410741806030273,
      "KL/mean": -33.92689514160156,
      "KL/rejected_KL_mean": -51.443058013916016,
      "KL/std": 44.932857513427734,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.009540662169456482,
      "fcm_dpo/delta": -0.006916055455803871,
      "fcm_dpo/margin": 35.03230667114258,
      "fcm_dpo/q_t": 0.42210057377815247,
      "grad_norm": 63.45244598388672,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": 1.2906028032302856,
      "logits/rejected": 1.3554030656814575,
      "logps/chosen": -295.77294921875,
      "logps/ref_chosen": -279.36224365234375,
      "logps/ref_rejected": -264.46112060546875,
      "logps/rejected": -315.9041748046875,
      "loss": 4.5694,
      "margin_dpo/margin_mean": 35.032310485839844,
      "margin_dpo/margin_std": 52.100738525390625,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -12.549001693725586,
      "KL/mean": -31.37158203125,
      "KL/rejected_KL_mean": -50.19416809082031,
      "KL/std": 47.65837860107422,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.009445380419492722,
      "fcm_dpo/delta": -0.002116305287927389,
      "fcm_dpo/margin": 37.645164489746094,
      "fcm_dpo/q_t": 0.4176445007324219,
      "grad_norm": 33.066978454589844,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": 1.385382056236267,
      "logits/rejected": 1.4666144847869873,
      "logps/chosen": -298.99810791015625,
      "logps/ref_chosen": -286.4491271972656,
      "logps/ref_rejected": -294.5469665527344,
      "logps/rejected": -344.7411193847656,
      "loss": 4.5743,
      "margin_dpo/margin_mean": 37.64516830444336,
      "margin_dpo/margin_std": 59.1423225402832,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -14.277935028076172,
      "KL/mean": -31.777904510498047,
      "KL/rejected_KL_mean": -49.27787399291992,
      "KL/std": 48.331668853759766,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.009579543955624104,
      "fcm_dpo/delta": 0.003551023080945015,
      "fcm_dpo/margin": 34.99993133544922,
      "fcm_dpo/q_t": 0.42172202467918396,
      "grad_norm": 59.75294494628906,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": 1.517030119895935,
      "logits/rejected": 1.5324795246124268,
      "logps/chosen": -296.6380310058594,
      "logps/ref_chosen": -282.36004638671875,
      "logps/ref_rejected": -243.79052734375,
      "logps/rejected": -293.0683898925781,
      "loss": 4.6276,
      "margin_dpo/margin_mean": 34.99992752075195,
      "margin_dpo/margin_std": 57.96569061279297,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -7.833158493041992,
      "KL/mean": -27.046016693115234,
      "KL/rejected_KL_mean": -46.258880615234375,
      "KL/std": 45.41553497314453,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.009383566677570343,
      "fcm_dpo/delta": -0.03877762705087662,
      "fcm_dpo/margin": 38.42571258544922,
      "fcm_dpo/q_t": 0.41506338119506836,
      "grad_norm": 26.381277084350586,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": 1.308509111404419,
      "logits/rejected": 1.3174598217010498,
      "logps/chosen": -286.9725646972656,
      "logps/ref_chosen": -279.139404296875,
      "logps/ref_rejected": -240.5850067138672,
      "logps/rejected": -286.8439025878906,
      "loss": 4.4828,
      "margin_dpo/margin_mean": 38.42571258544922,
      "margin_dpo/margin_std": 52.305118560791016,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -12.442237854003906,
      "KL/mean": -28.722257614135742,
      "KL/rejected_KL_mean": -45.00227355957031,
      "KL/std": 43.17566680908203,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.009205186739563942,
      "fcm_dpo/delta": 0.0015201438218355179,
      "fcm_dpo/margin": 32.56005096435547,
      "fcm_dpo/q_t": 0.4301590621471405,
      "grad_norm": 41.93576431274414,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": 1.251318097114563,
      "logits/rejected": 1.3218382596969604,
      "logps/chosen": -271.2537536621094,
      "logps/ref_chosen": -258.8114929199219,
      "logps/ref_rejected": -266.3898010253906,
      "logps/rejected": -311.39208984375,
      "loss": 4.6933,
      "margin_dpo/margin_mean": 32.5600471496582,
      "margin_dpo/margin_std": 53.999778747558594,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -11.953329086303711,
      "KL/mean": -31.2573299407959,
      "KL/rejected_KL_mean": -50.56132888793945,
      "KL/std": 47.75309371948242,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.009165092371404171,
      "fcm_dpo/delta": -0.00427668634802103,
      "fcm_dpo/margin": 38.608009338378906,
      "fcm_dpo/q_t": 0.41809749603271484,
      "grad_norm": 33.0722770690918,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": 1.5598325729370117,
      "logits/rejected": 1.508669376373291,
      "logps/chosen": -314.0195007324219,
      "logps/ref_chosen": -302.0661926269531,
      "logps/ref_rejected": -265.5518798828125,
      "logps/rejected": -316.1131896972656,
      "loss": 4.5232,
      "margin_dpo/margin_mean": 38.60800552368164,
      "margin_dpo/margin_std": 55.525794982910156,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -7.751003265380859,
      "KL/mean": -23.701391220092773,
      "KL/rejected_KL_mean": -39.65178298950195,
      "KL/std": 46.29627227783203,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.009365163743495941,
      "fcm_dpo/delta": 0.03429359942674637,
      "fcm_dpo/margin": 31.900775909423828,
      "fcm_dpo/q_t": 0.43113911151885986,
      "grad_norm": 28.250051498413086,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": 1.5521318912506104,
      "logits/rejected": 1.6586846113204956,
      "logps/chosen": -266.74932861328125,
      "logps/ref_chosen": -258.99835205078125,
      "logps/ref_rejected": -273.7834777832031,
      "logps/rejected": -313.4352722167969,
      "loss": 4.7364,
      "margin_dpo/margin_mean": 31.900775909423828,
      "margin_dpo/margin_std": 58.54168701171875,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -5.052398681640625,
      "KL/mean": -25.780982971191406,
      "KL/rejected_KL_mean": -46.509578704833984,
      "KL/std": 45.723548889160156,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.0093209408223629,
      "fcm_dpo/delta": -0.020243065431714058,
      "fcm_dpo/margin": 41.45718765258789,
      "fcm_dpo/q_t": 0.4093928337097168,
      "grad_norm": 48.13176345825195,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 1.380242943763733,
      "logits/rejected": 1.4731944799423218,
      "logps/chosen": -280.575439453125,
      "logps/ref_chosen": -275.5230407714844,
      "logps/ref_rejected": -255.50784301757812,
      "logps/rejected": -302.0174255371094,
      "loss": 4.39,
      "margin_dpo/margin_mean": 41.45718765258789,
      "margin_dpo/margin_std": 52.281673431396484,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -11.749269485473633,
      "KL/mean": -27.01513671875,
      "KL/rejected_KL_mean": -42.28101348876953,
      "KL/std": 46.513031005859375,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.009344187565147877,
      "fcm_dpo/delta": 0.017225971445441246,
      "fcm_dpo/margin": 30.53174591064453,
      "fcm_dpo/q_t": 0.43416643142700195,
      "grad_norm": 44.12722396850586,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.551112413406372,
      "logits/rejected": 1.6020417213439941,
      "logps/chosen": -265.11273193359375,
      "logps/ref_chosen": -253.36349487304688,
      "logps/ref_rejected": -247.1916961669922,
      "logps/rejected": -289.4726867675781,
      "loss": 4.7759,
      "margin_dpo/margin_mean": 30.531747817993164,
      "margin_dpo/margin_std": 57.105255126953125,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -7.701848983764648,
      "KL/mean": -26.719501495361328,
      "KL/rejected_KL_mean": -45.737144470214844,
      "KL/std": 45.892547607421875,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.009242896921932697,
      "fcm_dpo/delta": -0.02676640823483467,
      "fcm_dpo/margin": 38.03529357910156,
      "fcm_dpo/q_t": 0.41691428422927856,
      "grad_norm": 44.619140625,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": 1.4444420337677002,
      "logits/rejected": 1.3861929178237915,
      "logps/chosen": -272.68994140625,
      "logps/ref_chosen": -264.9880676269531,
      "logps/ref_rejected": -243.07838439941406,
      "logps/rejected": -288.8155212402344,
      "loss": 4.4959,
      "margin_dpo/margin_mean": 38.03529739379883,
      "margin_dpo/margin_std": 51.80820846557617,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -3.742572784423828,
      "KL/mean": -25.653615951538086,
      "KL/rejected_KL_mean": -47.564659118652344,
      "KL/std": 45.69383239746094,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.009033995680510998,
      "fcm_dpo/delta": -0.07475632429122925,
      "fcm_dpo/margin": 43.82208251953125,
      "fcm_dpo/q_t": 0.40706589818000793,
      "grad_norm": 37.47987365722656,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": 1.3462307453155518,
      "logits/rejected": 1.399829626083374,
      "logps/chosen": -319.6132507324219,
      "logps/ref_chosen": -315.8706970214844,
      "logps/ref_rejected": -304.3729248046875,
      "logps/rejected": -351.93756103515625,
      "loss": 4.3518,
      "margin_dpo/margin_mean": 43.822078704833984,
      "margin_dpo/margin_std": 49.976715087890625,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -6.3336181640625,
      "KL/mean": -28.682540893554688,
      "KL/rejected_KL_mean": -51.0314826965332,
      "KL/std": 47.801910400390625,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.008536333218216896,
      "fcm_dpo/delta": -0.04059959203004837,
      "fcm_dpo/margin": 44.69786071777344,
      "fcm_dpo/q_t": 0.41009336709976196,
      "grad_norm": 31.837512969970703,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 1.452755331993103,
      "logits/rejected": 1.4202756881713867,
      "logps/chosen": -317.4307861328125,
      "logps/ref_chosen": -311.09716796875,
      "logps/ref_rejected": -266.01959228515625,
      "logps/rejected": -317.0510559082031,
      "loss": 4.3972,
      "margin_dpo/margin_mean": 44.69786071777344,
      "margin_dpo/margin_std": 55.09814453125,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -5.7591705322265625,
      "KL/mean": -28.31631851196289,
      "KL/rejected_KL_mean": -50.87346649169922,
      "KL/std": 47.076480865478516,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.008309687487781048,
      "fcm_dpo/delta": -0.03877272084355354,
      "fcm_dpo/margin": 45.11430740356445,
      "fcm_dpo/q_t": 0.4111943244934082,
      "grad_norm": 37.424591064453125,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": 1.5774998664855957,
      "logits/rejected": 1.479419231414795,
      "logps/chosen": -290.4996337890625,
      "logps/ref_chosen": -284.740478515625,
      "logps/ref_rejected": -237.64117431640625,
      "logps/rejected": -288.5146789550781,
      "loss": 4.4038,
      "margin_dpo/margin_mean": 45.11430740356445,
      "margin_dpo/margin_std": 53.434776306152344,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -17.321117401123047,
      "KL/mean": -31.070613861083984,
      "KL/rejected_KL_mean": -44.82012176513672,
      "KL/std": 46.40237808227539,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.008177634328603745,
      "fcm_dpo/delta": 0.0015263324603438377,
      "fcm_dpo/margin": 27.49900245666504,
      "fcm_dpo/q_t": 0.4475019574165344,
      "grad_norm": 23.568252563476562,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": 1.5368808507919312,
      "logits/rejected": 1.5696860551834106,
      "logps/chosen": -289.93402099609375,
      "logps/ref_chosen": -272.6128845214844,
      "logps/ref_rejected": -249.99725341796875,
      "logps/rejected": -294.8173828125,
      "loss": 4.9398,
      "margin_dpo/margin_mean": 27.49900245666504,
      "margin_dpo/margin_std": 57.713592529296875,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -8.57467269897461,
      "KL/mean": -27.495817184448242,
      "KL/rejected_KL_mean": -46.416961669921875,
      "KL/std": 47.079429626464844,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.0082346610724926,
      "fcm_dpo/delta": 0.00613771378993988,
      "fcm_dpo/margin": 37.8422966003418,
      "fcm_dpo/q_t": 0.42654314637184143,
      "grad_norm": 39.769134521484375,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": 1.3886809349060059,
      "logits/rejected": 1.3900736570358276,
      "logps/chosen": -283.0104675292969,
      "logps/ref_chosen": -274.435791015625,
      "logps/ref_rejected": -257.8125305175781,
      "logps/rejected": -304.2294921875,
      "loss": 4.6025,
      "margin_dpo/margin_mean": 37.8422966003418,
      "margin_dpo/margin_std": 54.31791687011719,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -13.15407943725586,
      "KL/mean": -27.621341705322266,
      "KL/rejected_KL_mean": -42.08860778808594,
      "KL/std": 45.52972412109375,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.008216112852096558,
      "fcm_dpo/delta": -0.004140892066061497,
      "fcm_dpo/margin": 28.93453598022461,
      "fcm_dpo/q_t": 0.4438374638557434,
      "grad_norm": 51.65553665161133,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": 1.348482608795166,
      "logits/rejected": 1.3929824829101562,
      "logps/chosen": -274.46539306640625,
      "logps/ref_chosen": -261.3113098144531,
      "logps/ref_rejected": -231.152587890625,
      "logps/rejected": -273.2411804199219,
      "loss": 4.8624,
      "margin_dpo/margin_mean": 28.934534072875977,
      "margin_dpo/margin_std": 54.82958984375,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -15.060667037963867,
      "KL/mean": -32.957820892333984,
      "KL/rejected_KL_mean": -50.854976654052734,
      "KL/std": 49.33285140991211,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.008207570761442184,
      "fcm_dpo/delta": 0.02291705831885338,
      "fcm_dpo/margin": 35.7943115234375,
      "fcm_dpo/q_t": 0.4308225214481354,
      "grad_norm": 37.96660232543945,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": 1.505326271057129,
      "logits/rejected": 1.4708384275436401,
      "logps/chosen": -283.52789306640625,
      "logps/ref_chosen": -268.46722412109375,
      "logps/ref_rejected": -240.27597045898438,
      "logps/rejected": -291.1309509277344,
      "loss": 4.6828,
      "margin_dpo/margin_mean": 35.7943115234375,
      "margin_dpo/margin_std": 56.7739372253418,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -17.306367874145508,
      "KL/mean": -36.21662521362305,
      "KL/rejected_KL_mean": -55.12687301635742,
      "KL/std": 51.424251556396484,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.008283278904855251,
      "fcm_dpo/delta": -0.006828071549534798,
      "fcm_dpo/margin": 37.82052230834961,
      "fcm_dpo/q_t": 0.4264187514781952,
      "grad_norm": 23.653406143188477,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 1.4709653854370117,
      "logits/rejected": 1.587652325630188,
      "logps/chosen": -302.41156005859375,
      "logps/ref_chosen": -285.1051940917969,
      "logps/ref_rejected": -281.2331848144531,
      "logps/rejected": -336.3600769042969,
      "loss": 4.6547,
      "margin_dpo/margin_mean": 37.82052230834961,
      "margin_dpo/margin_std": 61.2096061706543,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -14.253381729125977,
      "KL/mean": -32.611568450927734,
      "KL/rejected_KL_mean": -50.969757080078125,
      "KL/std": 48.73473358154297,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.008269348181784153,
      "fcm_dpo/delta": -0.011054194532334805,
      "fcm_dpo/margin": 36.71636962890625,
      "fcm_dpo/q_t": 0.42908698320388794,
      "grad_norm": 35.240299224853516,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 1.320767879486084,
      "logits/rejected": 1.2816089391708374,
      "logps/chosen": -288.3659973144531,
      "logps/ref_chosen": -274.1126403808594,
      "logps/ref_rejected": -256.7730407714844,
      "logps/rejected": -307.7427978515625,
      "loss": 4.6494,
      "margin_dpo/margin_mean": 36.71636962890625,
      "margin_dpo/margin_std": 55.78811264038086,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -24.1849365234375,
      "KL/mean": -43.23986053466797,
      "KL/rejected_KL_mean": -62.29478073120117,
      "KL/std": 51.25864028930664,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.008017309010028839,
      "fcm_dpo/delta": -0.016017895191907883,
      "fcm_dpo/margin": 38.10984802246094,
      "fcm_dpo/q_t": 0.42850780487060547,
      "grad_norm": 31.450891494750977,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": 1.6005125045776367,
      "logits/rejected": 1.576390266418457,
      "logps/chosen": -313.75067138671875,
      "logps/ref_chosen": -289.56573486328125,
      "logps/ref_rejected": -277.9891662597656,
      "logps/rejected": -340.283935546875,
      "loss": 4.6942,
      "margin_dpo/margin_mean": 38.10984802246094,
      "margin_dpo/margin_std": 63.907535552978516,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -19.24492073059082,
      "KL/mean": -35.059410095214844,
      "KL/rejected_KL_mean": -50.8739013671875,
      "KL/std": 50.78015899658203,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.008131861686706543,
      "fcm_dpo/delta": 0.011123756878077984,
      "fcm_dpo/margin": 31.628984451293945,
      "fcm_dpo/q_t": 0.43972745537757874,
      "grad_norm": 41.78746795654297,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": 1.56095290184021,
      "logits/rejected": 1.5611060857772827,
      "logps/chosen": -289.4288635253906,
      "logps/ref_chosen": -270.1839294433594,
      "logps/ref_rejected": -245.86949157714844,
      "logps/rejected": -296.7433776855469,
      "loss": 4.8194,
      "margin_dpo/margin_mean": 31.62898063659668,
      "margin_dpo/margin_std": 58.8775634765625,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -17.32006072998047,
      "KL/mean": -36.75452423095703,
      "KL/rejected_KL_mean": -56.188987731933594,
      "KL/std": 50.993865966796875,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.008216392248868942,
      "fcm_dpo/delta": -0.017387766391038895,
      "fcm_dpo/margin": 38.868927001953125,
      "fcm_dpo/q_t": 0.42446374893188477,
      "grad_norm": 31.08074951171875,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": 1.3635997772216797,
      "logits/rejected": 1.3543472290039062,
      "logps/chosen": -278.0065002441406,
      "logps/ref_chosen": -260.6864318847656,
      "logps/ref_rejected": -245.99508666992188,
      "logps/rejected": -302.18408203125,
      "loss": 4.5779,
      "margin_dpo/margin_mean": 38.868927001953125,
      "margin_dpo/margin_std": 51.846187591552734,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -15.681177139282227,
      "KL/mean": -37.65758514404297,
      "KL/rejected_KL_mean": -59.63398742675781,
      "KL/std": 51.497169494628906,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.007992291823029518,
      "fcm_dpo/delta": -0.02315811812877655,
      "fcm_dpo/margin": 43.95281219482422,
      "fcm_dpo/q_t": 0.41785573959350586,
      "grad_norm": 35.47465896606445,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": 1.6138161420822144,
      "logits/rejected": 1.5338842868804932,
      "logps/chosen": -313.59503173828125,
      "logps/ref_chosen": -297.9138488769531,
      "logps/ref_rejected": -253.39952087402344,
      "logps/rejected": -313.0334777832031,
      "loss": 4.5321,
      "margin_dpo/margin_mean": 43.95281219482422,
      "margin_dpo/margin_std": 64.55175018310547,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -14.726995468139648,
      "KL/mean": -36.61515808105469,
      "KL/rejected_KL_mean": -58.503318786621094,
      "KL/std": 51.75815200805664,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.007810108829289675,
      "fcm_dpo/delta": -0.04600879177451134,
      "fcm_dpo/margin": 43.77632141113281,
      "fcm_dpo/q_t": 0.4193764925003052,
      "grad_norm": 36.6204833984375,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": 1.4714971780776978,
      "logits/rejected": 1.4366422891616821,
      "logps/chosen": -324.65338134765625,
      "logps/ref_chosen": -309.9263916015625,
      "logps/ref_rejected": -253.452392578125,
      "logps/rejected": -311.9557189941406,
      "loss": 4.5792,
      "margin_dpo/margin_mean": 43.77632141113281,
      "margin_dpo/margin_std": 65.2590103149414,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -19.411840438842773,
      "KL/mean": -39.80348587036133,
      "KL/rejected_KL_mean": -60.19514083862305,
      "KL/std": 53.04247283935547,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.0076012397184967995,
      "fcm_dpo/delta": -0.0026081581600010395,
      "fcm_dpo/margin": 40.78329849243164,
      "fcm_dpo/q_t": 0.4275486469268799,
      "grad_norm": 34.90549850463867,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 1.2005559206008911,
      "logits/rejected": 1.2336088418960571,
      "logps/chosen": -279.2254638671875,
      "logps/ref_chosen": -259.8136291503906,
      "logps/ref_rejected": -245.84947204589844,
      "logps/rejected": -306.04461669921875,
      "loss": 4.6256,
      "margin_dpo/margin_mean": 40.78329849243164,
      "margin_dpo/margin_std": 61.779483795166016,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -17.512104034423828,
      "KL/mean": -40.235015869140625,
      "KL/rejected_KL_mean": -62.95793533325195,
      "KL/std": 53.05385971069336,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.007620013318955898,
      "fcm_dpo/delta": 0.015766555443406105,
      "fcm_dpo/margin": 45.445823669433594,
      "fcm_dpo/q_t": 0.41774579882621765,
      "grad_norm": 31.96754264831543,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": 1.4439107179641724,
      "logits/rejected": 1.5247019529342651,
      "logps/chosen": -282.86578369140625,
      "logps/ref_chosen": -265.35369873046875,
      "logps/ref_rejected": -247.9174346923828,
      "logps/rejected": -310.8753662109375,
      "loss": 4.4684,
      "margin_dpo/margin_mean": 45.445831298828125,
      "margin_dpo/margin_std": 56.09292984008789,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -17.48977279663086,
      "KL/mean": -38.04568862915039,
      "KL/rejected_KL_mean": -58.60159683227539,
      "KL/std": 52.513526916503906,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.007811093702912331,
      "fcm_dpo/delta": 0.02592150866985321,
      "fcm_dpo/margin": 41.1118278503418,
      "fcm_dpo/q_t": 0.42412111163139343,
      "grad_norm": 42.4964485168457,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": 1.4517053365707397,
      "logits/rejected": 1.4565212726593018,
      "logps/chosen": -305.7130126953125,
      "logps/ref_chosen": -288.2232360839844,
      "logps/ref_rejected": -270.7300109863281,
      "logps/rejected": -329.33160400390625,
      "loss": 4.591,
      "margin_dpo/margin_mean": 41.1118278503418,
      "margin_dpo/margin_std": 61.90205383300781,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -23.217283248901367,
      "KL/mean": -31.556880950927734,
      "KL/rejected_KL_mean": -39.896480560302734,
      "KL/std": 50.41972351074219,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.007871251553297043,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.679189682006836,
      "fcm_dpo/q_t": 0.46903055906295776,
      "grad_norm": 41.47114562988281,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 1.3227968215942383,
      "logits/rejected": 1.3607016801834106,
      "logps/chosen": -304.83697509765625,
      "logps/ref_chosen": -281.61968994140625,
      "logps/ref_rejected": -271.03973388671875,
      "logps/rejected": -310.93621826171875,
      "loss": 5.2728,
      "margin_dpo/margin_mean": 16.67919158935547,
      "margin_dpo/margin_std": 60.548675537109375,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -25.87925910949707,
      "KL/mean": -43.77243423461914,
      "KL/rejected_KL_mean": -61.66560745239258,
      "KL/std": 51.95642852783203,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.007981901057064533,
      "fcm_dpo/delta": 0.019334139302372932,
      "fcm_dpo/margin": 35.786338806152344,
      "fcm_dpo/q_t": 0.43325603008270264,
      "grad_norm": 33.35435485839844,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 1.185638189315796,
      "logits/rejected": 1.3351001739501953,
      "logps/chosen": -268.3424072265625,
      "logps/ref_chosen": -242.4631805419922,
      "logps/ref_rejected": -244.0020751953125,
      "logps/rejected": -305.6676940917969,
      "loss": 4.7541,
      "margin_dpo/margin_mean": 35.786338806152344,
      "margin_dpo/margin_std": 65.08773803710938,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -17.703340530395508,
      "KL/mean": -39.496910095214844,
      "KL/rejected_KL_mean": -61.29048156738281,
      "KL/std": 53.23859405517578,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.00795234739780426,
      "fcm_dpo/delta": -0.0017024129629135132,
      "fcm_dpo/margin": 43.58715057373047,
      "fcm_dpo/q_t": 0.41861289739608765,
      "grad_norm": 38.74953842163086,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": 1.385488748550415,
      "logits/rejected": 1.4084782600402832,
      "logps/chosen": -309.69195556640625,
      "logps/ref_chosen": -291.98858642578125,
      "logps/ref_rejected": -273.35955810546875,
      "logps/rejected": -334.6500549316406,
      "loss": 4.5066,
      "margin_dpo/margin_mean": 43.587154388427734,
      "margin_dpo/margin_std": 59.84076690673828,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -25.03870964050293,
      "KL/mean": -43.31680679321289,
      "KL/rejected_KL_mean": -61.59490966796875,
      "KL/std": 54.164371490478516,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.008037635125219822,
      "fcm_dpo/delta": 0.012096789665520191,
      "fcm_dpo/margin": 36.55619812011719,
      "fcm_dpo/q_t": 0.4317954182624817,
      "grad_norm": 40.56562805175781,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": 1.3228437900543213,
      "logits/rejected": 1.304059386253357,
      "logps/chosen": -300.02716064453125,
      "logps/ref_chosen": -274.98846435546875,
      "logps/ref_rejected": -254.75782775878906,
      "logps/rejected": -316.35272216796875,
      "loss": 4.7174,
      "margin_dpo/margin_mean": 36.55619812011719,
      "margin_dpo/margin_std": 62.566261291503906,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -14.977754592895508,
      "KL/mean": -37.8474006652832,
      "KL/rejected_KL_mean": -60.7170524597168,
      "KL/std": 53.008026123046875,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.008018470369279385,
      "fcm_dpo/delta": -0.001144391018897295,
      "fcm_dpo/margin": 45.73929977416992,
      "fcm_dpo/q_t": 0.41437965631484985,
      "grad_norm": 30.4667911529541,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": 1.1412526369094849,
      "logits/rejected": 1.20687735080719,
      "logps/chosen": -266.6039123535156,
      "logps/ref_chosen": -251.62615966796875,
      "logps/ref_rejected": -251.9549560546875,
      "logps/rejected": -312.6719665527344,
      "loss": 4.4324,
      "margin_dpo/margin_mean": 45.73929977416992,
      "margin_dpo/margin_std": 58.40346908569336,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -22.662710189819336,
      "KL/mean": -44.04167938232422,
      "KL/rejected_KL_mean": -65.42064666748047,
      "KL/std": 55.73441696166992,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.007967759855091572,
      "fcm_dpo/delta": -0.02265811525285244,
      "fcm_dpo/margin": 42.7579460144043,
      "fcm_dpo/q_t": 0.4204194247722626,
      "grad_norm": 34.849002838134766,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": 1.3276201486587524,
      "logits/rejected": 1.3779035806655884,
      "logps/chosen": -300.5993347167969,
      "logps/ref_chosen": -277.9366149902344,
      "logps/ref_rejected": -262.689453125,
      "logps/rejected": -328.110107421875,
      "loss": 4.5568,
      "margin_dpo/margin_mean": 42.7579460144043,
      "margin_dpo/margin_std": 63.13814926147461,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -11.108684539794922,
      "KL/mean": -33.52579116821289,
      "KL/rejected_KL_mean": -55.94289016723633,
      "KL/std": 54.05701446533203,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.00787594448775053,
      "fcm_dpo/delta": -0.027414295822381973,
      "fcm_dpo/margin": 44.83421325683594,
      "fcm_dpo/q_t": 0.4164822995662689,
      "grad_norm": 30.900665283203125,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": 1.6636230945587158,
      "logits/rejected": 1.6647976636886597,
      "logps/chosen": -281.5535888671875,
      "logps/ref_chosen": -270.4449157714844,
      "logps/ref_rejected": -259.5942077636719,
      "logps/rejected": -315.537109375,
      "loss": 4.4829,
      "margin_dpo/margin_mean": 44.83421325683594,
      "margin_dpo/margin_std": 57.65927505493164,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -20.635791778564453,
      "KL/mean": -39.626441955566406,
      "KL/rejected_KL_mean": -58.61709976196289,
      "KL/std": 55.14639663696289,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.00778606254607439,
      "fcm_dpo/delta": 0.010608670301735401,
      "fcm_dpo/margin": 37.98130416870117,
      "fcm_dpo/q_t": 0.4307641088962555,
      "grad_norm": 38.43115997314453,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": 1.2288126945495605,
      "logits/rejected": 1.2933337688446045,
      "logps/chosen": -290.4756774902344,
      "logps/ref_chosen": -269.83990478515625,
      "logps/ref_rejected": -246.33746337890625,
      "logps/rejected": -304.9545593261719,
      "loss": 4.7201,
      "margin_dpo/margin_mean": 37.98130416870117,
      "margin_dpo/margin_std": 66.50846099853516,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -15.36760139465332,
      "KL/mean": -35.499603271484375,
      "KL/rejected_KL_mean": -55.63159942626953,
      "KL/std": 54.25305938720703,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.00793545600026846,
      "fcm_dpo/delta": 0.028612473979592323,
      "fcm_dpo/margin": 40.26398468017578,
      "fcm_dpo/q_t": 0.42559781670570374,
      "grad_norm": 27.9097843170166,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": 1.397802710533142,
      "logits/rejected": 1.3829798698425293,
      "logps/chosen": -312.90155029296875,
      "logps/ref_chosen": -297.533935546875,
      "logps/ref_rejected": -270.3407897949219,
      "logps/rejected": -325.9723815917969,
      "loss": 4.6012,
      "margin_dpo/margin_mean": 40.26398468017578,
      "margin_dpo/margin_std": 60.653358459472656,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -18.63843536376953,
      "KL/mean": -38.046878814697266,
      "KL/rejected_KL_mean": -57.455322265625,
      "KL/std": 54.79336929321289,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.008066361770033836,
      "fcm_dpo/delta": 0.02476578578352928,
      "fcm_dpo/margin": 38.81688690185547,
      "fcm_dpo/q_t": 0.4265764057636261,
      "grad_norm": 36.19751739501953,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": 1.2523844242095947,
      "logits/rejected": 1.3014583587646484,
      "logps/chosen": -258.9772644042969,
      "logps/ref_chosen": -240.33883666992188,
      "logps/ref_rejected": -220.71717834472656,
      "logps/rejected": -278.1724853515625,
      "loss": 4.6768,
      "margin_dpo/margin_mean": 38.81688690185547,
      "margin_dpo/margin_std": 66.3595199584961,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -16.66028594970703,
      "KL/mean": -39.79643249511719,
      "KL/rejected_KL_mean": -62.93256378173828,
      "KL/std": 57.64405822753906,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.008110888302326202,
      "fcm_dpo/delta": -0.006299967877566814,
      "fcm_dpo/margin": 46.27227020263672,
      "fcm_dpo/q_t": 0.41256463527679443,
      "grad_norm": 49.34062957763672,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 1.7138807773590088,
      "logits/rejected": 1.7365413904190063,
      "logps/chosen": -325.55340576171875,
      "logps/ref_chosen": -308.8930969238281,
      "logps/ref_rejected": -278.47869873046875,
      "logps/rejected": -341.4112548828125,
      "loss": 4.5137,
      "margin_dpo/margin_mean": 46.27227020263672,
      "margin_dpo/margin_std": 70.6196060180664,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -17.031749725341797,
      "KL/mean": -38.669464111328125,
      "KL/rejected_KL_mean": -60.307167053222656,
      "KL/std": 54.181495666503906,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.008062233217060566,
      "fcm_dpo/delta": -0.019395561888813972,
      "fcm_dpo/margin": 43.275421142578125,
      "fcm_dpo/q_t": 0.4187086522579193,
      "grad_norm": 48.84977340698242,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.302065134048462,
      "logits/rejected": 1.3138329982757568,
      "logps/chosen": -292.4599914550781,
      "logps/ref_chosen": -275.42822265625,
      "logps/ref_rejected": -251.50906372070312,
      "logps/rejected": -311.81622314453125,
      "loss": 4.5127,
      "margin_dpo/margin_mean": 43.275421142578125,
      "margin_dpo/margin_std": 60.45093536376953,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -14.979257583618164,
      "KL/mean": -35.29031753540039,
      "KL/rejected_KL_mean": -55.601375579833984,
      "KL/std": 54.39499282836914,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.007889529690146446,
      "fcm_dpo/delta": -0.009940781630575657,
      "fcm_dpo/margin": 40.62211608886719,
      "fcm_dpo/q_t": 0.4250425696372986,
      "grad_norm": 47.24869918823242,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": 1.5054211616516113,
      "logits/rejected": 1.5296497344970703,
      "logps/chosen": -304.36199951171875,
      "logps/ref_chosen": -289.38275146484375,
      "logps/ref_rejected": -274.895751953125,
      "logps/rejected": -330.4971618652344,
      "loss": 4.6138,
      "margin_dpo/margin_mean": 40.62211990356445,
      "margin_dpo/margin_std": 61.261051177978516,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -15.733943939208984,
      "KL/mean": -37.962135314941406,
      "KL/rejected_KL_mean": -60.19032669067383,
      "KL/std": 55.257476806640625,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.007862160913646221,
      "fcm_dpo/delta": 0.0012959819287061691,
      "fcm_dpo/margin": 44.45639419555664,
      "fcm_dpo/q_t": 0.41862159967422485,
      "grad_norm": 45.3828239440918,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": 1.4397683143615723,
      "logits/rejected": 1.5586602687835693,
      "logps/chosen": -300.722412109375,
      "logps/ref_chosen": -284.98846435546875,
      "logps/ref_rejected": -273.5077819824219,
      "logps/rejected": -333.6981201171875,
      "loss": 4.5447,
      "margin_dpo/margin_mean": 44.45639419555664,
      "margin_dpo/margin_std": 66.31834411621094,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -17.497085571289062,
      "KL/mean": -41.57096862792969,
      "KL/rejected_KL_mean": -65.64485931396484,
      "KL/std": 58.25270080566406,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.0077584958635270596,
      "fcm_dpo/delta": -0.036438554525375366,
      "fcm_dpo/margin": 48.14777374267578,
      "fcm_dpo/q_t": 0.4122614860534668,
      "grad_norm": 42.21443557739258,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": 1.3613735437393188,
      "logits/rejected": 1.4232763051986694,
      "logps/chosen": -280.17669677734375,
      "logps/ref_chosen": -262.6795959472656,
      "logps/ref_rejected": -236.13792419433594,
      "logps/rejected": -301.7828063964844,
      "loss": 4.4456,
      "margin_dpo/margin_mean": 48.14777374267578,
      "margin_dpo/margin_std": 64.33990478515625,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -12.135440826416016,
      "KL/mean": -39.64494323730469,
      "KL/rejected_KL_mean": -67.15446472167969,
      "KL/std": 55.995853424072266,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.007588455453515053,
      "fcm_dpo/delta": -0.031086822971701622,
      "fcm_dpo/margin": 55.019020080566406,
      "fcm_dpo/q_t": 0.40210220217704773,
      "grad_norm": 32.32468795776367,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": 1.3194061517715454,
      "logits/rejected": 1.4309662580490112,
      "logps/chosen": -258.5349426269531,
      "logps/ref_chosen": -246.39950561523438,
      "logps/ref_rejected": -247.4836883544922,
      "logps/rejected": -314.6381530761719,
      "loss": 4.2789,
      "margin_dpo/margin_mean": 55.019020080566406,
      "margin_dpo/margin_std": 63.548709869384766,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -10.927236557006836,
      "KL/mean": -36.29861831665039,
      "KL/rejected_KL_mean": -61.670005798339844,
      "KL/std": 56.746673583984375,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.007361192721873522,
      "fcm_dpo/delta": -0.0337311327457428,
      "fcm_dpo/margin": 50.742767333984375,
      "fcm_dpo/q_t": 0.41251063346862793,
      "grad_norm": 23.98717498779297,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": 1.5092639923095703,
      "logits/rejected": 1.6273524761199951,
      "logps/chosen": -299.8472900390625,
      "logps/ref_chosen": -288.9200439453125,
      "logps/ref_rejected": -291.69305419921875,
      "logps/rejected": -353.363037109375,
      "loss": 4.425,
      "margin_dpo/margin_mean": 50.742767333984375,
      "margin_dpo/margin_std": 65.73783111572266,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -23.999221801757812,
      "KL/mean": -40.72590637207031,
      "KL/rejected_KL_mean": -57.452598571777344,
      "KL/std": 55.27214431762695,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.007376409135758877,
      "fcm_dpo/delta": 0.014115702360868454,
      "fcm_dpo/margin": 33.45337677001953,
      "fcm_dpo/q_t": 0.44178885221481323,
      "grad_norm": 46.14187240600586,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": 1.042759656906128,
      "logits/rejected": 1.0752969980239868,
      "logps/chosen": -281.0841064453125,
      "logps/ref_chosen": -257.08489990234375,
      "logps/ref_rejected": -251.61532592773438,
      "logps/rejected": -309.06793212890625,
      "loss": 4.905,
      "margin_dpo/margin_mean": 33.453372955322266,
      "margin_dpo/margin_std": 72.41047668457031,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -28.679895401000977,
      "KL/mean": -44.49220657348633,
      "KL/rejected_KL_mean": -60.30452346801758,
      "KL/std": 57.695594787597656,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.007452310528606176,
      "fcm_dpo/delta": 0.012337332591414452,
      "fcm_dpo/margin": 31.6246337890625,
      "fcm_dpo/q_t": 0.44495028257369995,
      "grad_norm": 39.0546760559082,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": 1.3537324666976929,
      "logits/rejected": 1.3153518438339233,
      "logps/chosen": -289.4212646484375,
      "logps/ref_chosen": -260.7413635253906,
      "logps/ref_rejected": -227.4325408935547,
      "logps/rejected": -287.7370910644531,
      "loss": 4.9562,
      "margin_dpo/margin_mean": 31.624637603759766,
      "margin_dpo/margin_std": 70.71369171142578,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -18.025968551635742,
      "KL/mean": -44.137908935546875,
      "KL/rejected_KL_mean": -70.24986267089844,
      "KL/std": 59.700199127197266,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.00731236208230257,
      "fcm_dpo/delta": -0.029781954362988472,
      "fcm_dpo/margin": 52.22388458251953,
      "fcm_dpo/q_t": 0.41036343574523926,
      "grad_norm": 37.281829833984375,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": 1.0856341123580933,
      "logits/rejected": 1.1741048097610474,
      "logps/chosen": -320.9734802246094,
      "logps/ref_chosen": -302.947509765625,
      "logps/ref_rejected": -294.1771545410156,
      "logps/rejected": -364.427001953125,
      "loss": 4.3922,
      "margin_dpo/margin_mean": 52.22389221191406,
      "margin_dpo/margin_std": 64.42584991455078,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -20.20978546142578,
      "KL/mean": -43.02177047729492,
      "KL/rejected_KL_mean": -65.83375549316406,
      "KL/std": 56.50487518310547,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.007370505016297102,
      "fcm_dpo/delta": 0.019991518929600716,
      "fcm_dpo/margin": 45.62397766113281,
      "fcm_dpo/q_t": 0.4216546416282654,
      "grad_norm": 35.781864166259766,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 1.3362503051757812,
      "logits/rejected": 1.414484977722168,
      "logps/chosen": -278.1078796386719,
      "logps/ref_chosen": -257.8981018066406,
      "logps/ref_rejected": -244.9084014892578,
      "logps/rejected": -310.7421569824219,
      "loss": 4.5837,
      "margin_dpo/margin_mean": 45.62397766113281,
      "margin_dpo/margin_std": 70.78388214111328,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -29.355697631835938,
      "KL/mean": -52.84184646606445,
      "KL/rejected_KL_mean": -76.32797241210938,
      "KL/std": 59.14398956298828,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.007361803203821182,
      "fcm_dpo/delta": -0.012710859067738056,
      "fcm_dpo/margin": 46.9722785949707,
      "fcm_dpo/q_t": 0.41911548376083374,
      "grad_norm": 30.374950408935547,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 1.4000204801559448,
      "logits/rejected": 1.4679090976715088,
      "logps/chosen": -319.55035400390625,
      "logps/ref_chosen": -290.1946716308594,
      "logps/ref_rejected": -290.98883056640625,
      "logps/rejected": -367.3167724609375,
      "loss": 4.506,
      "margin_dpo/margin_mean": 46.972286224365234,
      "margin_dpo/margin_std": 62.56603240966797,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -28.42122459411621,
      "KL/mean": -49.12397384643555,
      "KL/rejected_KL_mean": -69.82673645019531,
      "KL/std": 60.08218002319336,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.007360382005572319,
      "fcm_dpo/delta": 0.014726975932717323,
      "fcm_dpo/margin": 41.40552520751953,
      "fcm_dpo/q_t": 0.4296199083328247,
      "grad_norm": 31.55447006225586,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": 1.5416221618652344,
      "logits/rejected": 1.6033389568328857,
      "logps/chosen": -337.338623046875,
      "logps/ref_chosen": -308.91741943359375,
      "logps/ref_rejected": -272.2177734375,
      "logps/rejected": -342.04449462890625,
      "loss": 4.713,
      "margin_dpo/margin_mean": 41.40552520751953,
      "margin_dpo/margin_std": 73.53365325927734,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -31.312150955200195,
      "KL/mean": -53.95912170410156,
      "KL/rejected_KL_mean": -76.60609436035156,
      "KL/std": 56.46576690673828,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.0073984600603580475,
      "fcm_dpo/delta": -0.01071212813258171,
      "fcm_dpo/margin": 45.29396057128906,
      "fcm_dpo/q_t": 0.4217148423194885,
      "grad_norm": 37.25709533691406,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 1.7171378135681152,
      "logits/rejected": 1.7997050285339355,
      "logps/chosen": -279.33416748046875,
      "logps/ref_chosen": -248.02200317382812,
      "logps/ref_rejected": -234.78541564941406,
      "logps/rejected": -311.39154052734375,
      "loss": 4.5741,
      "margin_dpo/margin_mean": 45.29396057128906,
      "margin_dpo/margin_std": 66.89875030517578,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -32.70777893066406,
      "KL/mean": -55.46038055419922,
      "KL/rejected_KL_mean": -78.21296691894531,
      "KL/std": 61.6889762878418,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.007268224842846394,
      "fcm_dpo/delta": -0.015567854046821594,
      "fcm_dpo/margin": 45.50518035888672,
      "fcm_dpo/q_t": 0.4237023591995239,
      "grad_norm": 43.46617889404297,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": 1.2220379114151,
      "logits/rejected": 1.2207473516464233,
      "logps/chosen": -297.197021484375,
      "logps/ref_chosen": -264.4892578125,
      "logps/ref_rejected": -255.43092346191406,
      "logps/rejected": -333.6438903808594,
      "loss": 4.6622,
      "margin_dpo/margin_mean": 45.50518035888672,
      "margin_dpo/margin_std": 77.41083526611328,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -32.53535461425781,
      "KL/mean": -49.788673400878906,
      "KL/rejected_KL_mean": -67.04198455810547,
      "KL/std": 57.929561614990234,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.007238807622343302,
      "fcm_dpo/delta": -0.014740175567567348,
      "fcm_dpo/margin": 34.50663375854492,
      "fcm_dpo/q_t": 0.4412015378475189,
      "grad_norm": 30.687665939331055,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": 1.5593175888061523,
      "logits/rejected": 1.5832394361495972,
      "logps/chosen": -334.0040283203125,
      "logps/ref_chosen": -301.46868896484375,
      "logps/ref_rejected": -274.2380676269531,
      "logps/rejected": -341.28009033203125,
      "loss": 4.8637,
      "margin_dpo/margin_mean": 34.50663375854492,
      "margin_dpo/margin_std": 66.4465560913086,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -36.32493591308594,
      "KL/mean": -55.5125617980957,
      "KL/rejected_KL_mean": -74.70018005371094,
      "KL/std": 57.79391098022461,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.0072789923287928104,
      "fcm_dpo/delta": 0.021744607016444206,
      "fcm_dpo/margin": 38.375240325927734,
      "fcm_dpo/q_t": 0.4350081980228424,
      "grad_norm": 46.40941619873047,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": 1.4179409742355347,
      "logits/rejected": 1.4944617748260498,
      "logps/chosen": -302.88397216796875,
      "logps/ref_chosen": -266.5590515136719,
      "logps/ref_rejected": -280.12066650390625,
      "logps/rejected": -354.82086181640625,
      "loss": 4.7876,
      "margin_dpo/margin_mean": 38.375240325927734,
      "margin_dpo/margin_std": 72.64382934570312,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -34.650169372558594,
      "KL/mean": -55.65296936035156,
      "KL/rejected_KL_mean": -76.6557846069336,
      "KL/std": 54.42596435546875,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.0072936019860208035,
      "fcm_dpo/delta": 0.012156397104263306,
      "fcm_dpo/margin": 42.005619049072266,
      "fcm_dpo/q_t": 0.42769256234169006,
      "grad_norm": 48.871009826660156,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": 1.2404550313949585,
      "logits/rejected": 1.3001395463943481,
      "logps/chosen": -304.5865173339844,
      "logps/ref_chosen": -269.93634033203125,
      "logps/ref_rejected": -232.32330322265625,
      "logps/rejected": -308.9790954589844,
      "loss": 4.6672,
      "margin_dpo/margin_mean": 42.00562286376953,
      "margin_dpo/margin_std": 67.87881469726562,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -35.42621994018555,
      "KL/mean": -55.892181396484375,
      "KL/rejected_KL_mean": -76.3581314086914,
      "KL/std": 58.33458709716797,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.00744377076625824,
      "fcm_dpo/delta": -0.004768785089254379,
      "fcm_dpo/margin": 40.931922912597656,
      "fcm_dpo/q_t": 0.42933687567710876,
      "grad_norm": 35.106781005859375,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": 1.4524424076080322,
      "logits/rejected": 1.418065071105957,
      "logps/chosen": -319.4717102050781,
      "logps/ref_chosen": -284.04547119140625,
      "logps/ref_rejected": -261.19378662109375,
      "logps/rejected": -337.55194091796875,
      "loss": 4.7075,
      "margin_dpo/margin_mean": 40.93191909790039,
      "margin_dpo/margin_std": 71.47019958496094,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -41.606964111328125,
      "KL/mean": -56.00468063354492,
      "KL/rejected_KL_mean": -70.40238952636719,
      "KL/std": 59.85064697265625,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.007384240161627531,
      "fcm_dpo/delta": 0.019193090498447418,
      "fcm_dpo/margin": 28.795425415039062,
      "fcm_dpo/q_t": 0.4503389298915863,
      "grad_norm": 57.082000732421875,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": 1.2914851903915405,
      "logits/rejected": 1.2555955648422241,
      "logps/chosen": -364.8882141113281,
      "logps/ref_chosen": -323.28125,
      "logps/ref_rejected": -274.2093200683594,
      "logps/rejected": -344.6117248535156,
      "loss": 5.0176,
      "margin_dpo/margin_mean": 28.795425415039062,
      "margin_dpo/margin_std": 70.73609924316406,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -36.821231842041016,
      "KL/mean": -54.28681564331055,
      "KL/rejected_KL_mean": -71.75239562988281,
      "KL/std": 59.258785247802734,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.007498524151742458,
      "fcm_dpo/delta": 0.008330443874001503,
      "fcm_dpo/margin": 34.93117141723633,
      "fcm_dpo/q_t": 0.43897923827171326,
      "grad_norm": 46.92943572998047,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 1.2196189165115356,
      "logits/rejected": 1.17889404296875,
      "logps/chosen": -354.2566223144531,
      "logps/ref_chosen": -317.4353942871094,
      "logps/ref_rejected": -309.32403564453125,
      "logps/rejected": -381.0764465332031,
      "loss": 4.8383,
      "margin_dpo/margin_mean": 34.93117141723633,
      "margin_dpo/margin_std": 68.7139663696289,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -30.858919143676758,
      "KL/mean": -56.99393844604492,
      "KL/rejected_KL_mean": -83.12895965576172,
      "KL/std": 62.7364501953125,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.007459241431206465,
      "fcm_dpo/delta": -0.018715938553214073,
      "fcm_dpo/margin": 52.27004623413086,
      "fcm_dpo/q_t": 0.4092599153518677,
      "grad_norm": 59.25837707519531,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 1.1268885135650635,
      "logits/rejected": 1.2850844860076904,
      "logps/chosen": -296.36712646484375,
      "logps/ref_chosen": -265.50823974609375,
      "logps/ref_rejected": -275.1380310058594,
      "logps/rejected": -358.2669677734375,
      "loss": 4.434,
      "margin_dpo/margin_mean": 52.27004623413086,
      "margin_dpo/margin_std": 72.56016540527344,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -27.357873916625977,
      "KL/mean": -49.00636672973633,
      "KL/rejected_KL_mean": -70.65485382080078,
      "KL/std": 59.546241760253906,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.00728296535089612,
      "fcm_dpo/delta": -0.013749905861914158,
      "fcm_dpo/margin": 43.29698181152344,
      "fcm_dpo/q_t": 0.4256154000759125,
      "grad_norm": 46.36683654785156,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": 1.3024742603302002,
      "logits/rejected": 1.4269940853118896,
      "logps/chosen": -277.2591857910156,
      "logps/ref_chosen": -249.9013214111328,
      "logps/ref_rejected": -274.58245849609375,
      "logps/rejected": -345.2373046875,
      "loss": 4.6263,
      "margin_dpo/margin_mean": 43.2969856262207,
      "margin_dpo/margin_std": 65.71458435058594,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -34.607765197753906,
      "KL/mean": -57.68313217163086,
      "KL/rejected_KL_mean": -80.75849914550781,
      "KL/std": 62.589012145996094,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.007285939995199442,
      "fcm_dpo/delta": -0.007581109181046486,
      "fcm_dpo/margin": 46.15072250366211,
      "fcm_dpo/q_t": 0.4216371774673462,
      "grad_norm": 49.17685317993164,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": 1.4194259643554688,
      "logits/rejected": 1.3920180797576904,
      "logps/chosen": -330.9137268066406,
      "logps/ref_chosen": -296.30596923828125,
      "logps/ref_rejected": -268.0640869140625,
      "logps/rejected": -348.82257080078125,
      "loss": 4.5708,
      "margin_dpo/margin_mean": 46.150718688964844,
      "margin_dpo/margin_std": 68.9015884399414,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -29.459735870361328,
      "KL/mean": -49.133724212646484,
      "KL/rejected_KL_mean": -68.80770111083984,
      "KL/std": 60.64479064941406,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.007349840831011534,
      "fcm_dpo/delta": 0.01948559656739235,
      "fcm_dpo/margin": 39.347965240478516,
      "fcm_dpo/q_t": 0.43245962262153625,
      "grad_norm": 25.18802261352539,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": 1.2515159845352173,
      "logits/rejected": 1.2902932167053223,
      "logps/chosen": -263.34271240234375,
      "logps/ref_chosen": -233.8829803466797,
      "logps/ref_rejected": -224.14334106445312,
      "logps/rejected": -292.9510803222656,
      "loss": 4.7482,
      "margin_dpo/margin_mean": 39.347965240478516,
      "margin_dpo/margin_std": 70.32968139648438,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -31.23444366455078,
      "KL/mean": -52.998592376708984,
      "KL/rejected_KL_mean": -74.76274108886719,
      "KL/std": 59.80821228027344,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.007421678863465786,
      "fcm_dpo/delta": 0.005569446831941605,
      "fcm_dpo/margin": 43.52828598022461,
      "fcm_dpo/q_t": 0.4254318177700043,
      "grad_norm": 37.800106048583984,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": 1.3891477584838867,
      "logits/rejected": 1.4062851667404175,
      "logps/chosen": -275.34576416015625,
      "logps/ref_chosen": -244.111328125,
      "logps/ref_rejected": -240.18373107910156,
      "logps/rejected": -314.9464416503906,
      "loss": 4.6569,
      "margin_dpo/margin_mean": 43.52828598022461,
      "margin_dpo/margin_std": 73.74298095703125,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -25.67761993408203,
      "KL/mean": -55.585208892822266,
      "KL/rejected_KL_mean": -85.49279022216797,
      "KL/std": 62.34554672241211,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.007142005953937769,
      "fcm_dpo/delta": -0.07291434705257416,
      "fcm_dpo/margin": 59.815189361572266,
      "fcm_dpo/q_t": 0.3994922637939453,
      "grad_norm": 32.8429069519043,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 1.2506002187728882,
      "logits/rejected": 1.4153300523757935,
      "logps/chosen": -310.3581848144531,
      "logps/ref_chosen": -284.6805725097656,
      "logps/ref_rejected": -282.1923828125,
      "logps/rejected": -367.68511962890625,
      "loss": 4.294,
      "margin_dpo/margin_mean": 59.81519317626953,
      "margin_dpo/margin_std": 73.50148010253906,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -26.859424591064453,
      "KL/mean": -50.714683532714844,
      "KL/rejected_KL_mean": -74.5699462890625,
      "KL/std": 60.48611068725586,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.006949334405362606,
      "fcm_dpo/delta": -0.003227552864700556,
      "fcm_dpo/margin": 47.71051788330078,
      "fcm_dpo/q_t": 0.42253953218460083,
      "grad_norm": 29.133304595947266,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": 1.6264894008636475,
      "logits/rejected": 1.6383724212646484,
      "logps/chosen": -307.08807373046875,
      "logps/ref_chosen": -280.2286376953125,
      "logps/ref_rejected": -280.8172912597656,
      "logps/rejected": -355.3872375488281,
      "loss": 4.574,
      "margin_dpo/margin_mean": 47.71051788330078,
      "margin_dpo/margin_std": 70.38153839111328,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -19.549274444580078,
      "KL/mean": -45.9188346862793,
      "KL/rejected_KL_mean": -72.28837585449219,
      "KL/std": 63.80850601196289,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.006848739925771952,
      "fcm_dpo/delta": -0.04469466581940651,
      "fcm_dpo/margin": 52.739112854003906,
      "fcm_dpo/q_t": 0.41649389266967773,
      "grad_norm": 22.89809226989746,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": 1.4003918170928955,
      "logits/rejected": 1.4116530418395996,
      "logps/chosen": -316.54962158203125,
      "logps/ref_chosen": -297.0003356933594,
      "logps/ref_rejected": -279.1561279296875,
      "logps/rejected": -351.4444885253906,
      "loss": 4.5163,
      "margin_dpo/margin_mean": 52.739112854003906,
      "margin_dpo/margin_std": 76.94337463378906,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -30.181594848632812,
      "KL/mean": -51.5609245300293,
      "KL/rejected_KL_mean": -72.94024658203125,
      "KL/std": 61.50654602050781,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.006731259170919657,
      "fcm_dpo/delta": -0.004076994024217129,
      "fcm_dpo/margin": 42.758663177490234,
      "fcm_dpo/q_t": 0.4317622184753418,
      "grad_norm": 38.264625549316406,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": 1.3998732566833496,
      "logits/rejected": 1.5011959075927734,
      "logps/chosen": -301.6530456542969,
      "logps/ref_chosen": -271.471435546875,
      "logps/ref_rejected": -268.0870056152344,
      "logps/rejected": -341.0272521972656,
      "loss": 4.6664,
      "margin_dpo/margin_mean": 42.75865936279297,
      "margin_dpo/margin_std": 62.11533737182617,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -31.79745101928711,
      "KL/mean": -53.67586135864258,
      "KL/rejected_KL_mean": -75.55427551269531,
      "KL/std": 61.23541259765625,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.006700856611132622,
      "fcm_dpo/delta": -0.00044274493120610714,
      "fcm_dpo/margin": 43.75682830810547,
      "fcm_dpo/q_t": 0.4314420223236084,
      "grad_norm": 29.4553279876709,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 1.5585321187973022,
      "logits/rejected": 1.568390130996704,
      "logps/chosen": -338.15533447265625,
      "logps/ref_chosen": -306.35784912109375,
      "logps/ref_rejected": -268.64776611328125,
      "logps/rejected": -344.2019958496094,
      "loss": 4.7117,
      "margin_dpo/margin_mean": 43.75682830810547,
      "margin_dpo/margin_std": 74.73361206054688,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -30.120189666748047,
      "KL/mean": -51.42067337036133,
      "KL/rejected_KL_mean": -72.72116088867188,
      "KL/std": 61.64916229248047,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.006819958798587322,
      "fcm_dpo/delta": 0.023979216814041138,
      "fcm_dpo/margin": 42.600975036621094,
      "fcm_dpo/q_t": 0.4320065975189209,
      "grad_norm": 42.54679489135742,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.4290990829467773,
      "logits/rejected": 1.5087430477142334,
      "logps/chosen": -300.1126708984375,
      "logps/ref_chosen": -269.9924621582031,
      "logps/ref_rejected": -271.3135681152344,
      "logps/rejected": -344.03472900390625,
      "loss": 4.7071,
      "margin_dpo/margin_mean": 42.600975036621094,
      "margin_dpo/margin_std": 71.96638488769531,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -27.226675033569336,
      "KL/mean": -52.800628662109375,
      "KL/rejected_KL_mean": -78.37458801269531,
      "KL/std": 61.552223205566406,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.006792484782636166,
      "fcm_dpo/delta": 0.007237130776047707,
      "fcm_dpo/margin": 51.14790344238281,
      "fcm_dpo/q_t": 0.4183288514614105,
      "grad_norm": 34.03398132324219,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": 1.3176977634429932,
      "logits/rejected": 1.3952348232269287,
      "logps/chosen": -287.6873779296875,
      "logps/ref_chosen": -260.460693359375,
      "logps/ref_rejected": -254.2005615234375,
      "logps/rejected": -332.57513427734375,
      "loss": 4.5358,
      "margin_dpo/margin_mean": 51.14790344238281,
      "margin_dpo/margin_std": 74.04083251953125,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -26.942230224609375,
      "KL/mean": -49.983097076416016,
      "KL/rejected_KL_mean": -73.02395629882812,
      "KL/std": 61.49386978149414,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.006864907220005989,
      "fcm_dpo/delta": -0.004785989411175251,
      "fcm_dpo/margin": 46.08172607421875,
      "fcm_dpo/q_t": 0.4253062307834625,
      "grad_norm": 50.873653411865234,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": 1.1216137409210205,
      "logits/rejected": 1.1592780351638794,
      "logps/chosen": -323.03851318359375,
      "logps/ref_chosen": -296.09625244140625,
      "logps/ref_rejected": -264.7806701660156,
      "logps/rejected": -337.80462646484375,
      "loss": 4.6563,
      "margin_dpo/margin_mean": 46.081729888916016,
      "margin_dpo/margin_std": 74.13328552246094,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -37.93655776977539,
      "KL/mean": -58.30284118652344,
      "KL/rejected_KL_mean": -78.66912078857422,
      "KL/std": 64.7446060180664,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.006854083389043808,
      "fcm_dpo/delta": -0.0007908522384241223,
      "fcm_dpo/margin": 40.7325553894043,
      "fcm_dpo/q_t": 0.4342608153820038,
      "grad_norm": 34.66687774658203,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": 1.1483803987503052,
      "logits/rejected": 1.1319715976715088,
      "logps/chosen": -319.7860107421875,
      "logps/ref_chosen": -281.8494873046875,
      "logps/ref_rejected": -234.3673858642578,
      "logps/rejected": -313.0364990234375,
      "loss": 4.7521,
      "margin_dpo/margin_mean": 40.7325553894043,
      "margin_dpo/margin_std": 72.36601257324219,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -21.377685546875,
      "KL/mean": -51.446372985839844,
      "KL/rejected_KL_mean": -81.51506042480469,
      "KL/std": 65.1338882446289,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.0066869230940938,
      "fcm_dpo/delta": -0.054045744240283966,
      "fcm_dpo/margin": 60.13738250732422,
      "fcm_dpo/q_t": 0.40563881397247314,
      "grad_norm": 34.016937255859375,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": 1.2824490070343018,
      "logits/rejected": 1.3872209787368774,
      "logps/chosen": -269.4204406738281,
      "logps/ref_chosen": -248.0427703857422,
      "logps/ref_rejected": -263.464599609375,
      "logps/rejected": -344.9796447753906,
      "loss": 4.3462,
      "margin_dpo/margin_mean": 60.13738250732422,
      "margin_dpo/margin_std": 72.67241668701172,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -19.408716201782227,
      "KL/mean": -45.50986862182617,
      "KL/rejected_KL_mean": -71.61103057861328,
      "KL/std": 61.40985107421875,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.006565909367054701,
      "fcm_dpo/delta": -0.009758894331753254,
      "fcm_dpo/margin": 52.20230484008789,
      "fcm_dpo/q_t": 0.41902127861976624,
      "grad_norm": 32.72817611694336,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": 1.1887701749801636,
      "logits/rejected": 1.1062426567077637,
      "logps/chosen": -330.69464111328125,
      "logps/ref_chosen": -311.2859191894531,
      "logps/ref_rejected": -268.4949645996094,
      "logps/rejected": -340.1059875488281,
      "loss": 4.5195,
      "margin_dpo/margin_mean": 52.20230484008789,
      "margin_dpo/margin_std": 70.68122100830078,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -26.70029640197754,
      "KL/mean": -54.83576583862305,
      "KL/rejected_KL_mean": -82.97122955322266,
      "KL/std": 64.91513061523438,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.006530831102281809,
      "fcm_dpo/delta": -0.01081875991076231,
      "fcm_dpo/margin": 56.27093505859375,
      "fcm_dpo/q_t": 0.41270995140075684,
      "grad_norm": 23.624927520751953,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": 1.3132789134979248,
      "logits/rejected": 1.3089410066604614,
      "logps/chosen": -296.91278076171875,
      "logps/ref_chosen": -270.21246337890625,
      "logps/ref_rejected": -239.1932830810547,
      "logps/rejected": -322.16448974609375,
      "loss": 4.4242,
      "margin_dpo/margin_mean": 56.27093505859375,
      "margin_dpo/margin_std": 68.74674224853516,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -33.75720977783203,
      "KL/mean": -53.1615104675293,
      "KL/rejected_KL_mean": -72.56580352783203,
      "KL/std": 62.78734588623047,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.006500259041786194,
      "fcm_dpo/delta": 0.021968282759189606,
      "fcm_dpo/margin": 38.80859375,
      "fcm_dpo/q_t": 0.44050633907318115,
      "grad_norm": 29.50109100341797,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": 1.3509091138839722,
      "logits/rejected": 1.353635549545288,
      "logps/chosen": -322.280029296875,
      "logps/ref_chosen": -288.5228271484375,
      "logps/ref_rejected": -253.90597534179688,
      "logps/rejected": -326.4718017578125,
      "loss": 4.8248,
      "margin_dpo/margin_mean": 38.80859375,
      "margin_dpo/margin_std": 71.71615600585938,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -31.094886779785156,
      "KL/mean": -59.8298225402832,
      "KL/rejected_KL_mean": -88.56475830078125,
      "KL/std": 66.10234832763672,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.006514878012239933,
      "fcm_dpo/delta": -0.022287029772996902,
      "fcm_dpo/margin": 57.46986389160156,
      "fcm_dpo/q_t": 0.4128711223602295,
      "grad_norm": 42.245880126953125,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 1.3176361322402954,
      "logits/rejected": 1.4433343410491943,
      "logps/chosen": -302.9896240234375,
      "logps/ref_chosen": -271.8947448730469,
      "logps/ref_rejected": -260.5709228515625,
      "logps/rejected": -349.1356506347656,
      "loss": 4.4532,
      "margin_dpo/margin_mean": 57.46985626220703,
      "margin_dpo/margin_std": 78.60163116455078,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -36.41690444946289,
      "KL/mean": -59.0675163269043,
      "KL/rejected_KL_mean": -81.71812438964844,
      "KL/std": 62.573577880859375,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.0064763231202960014,
      "fcm_dpo/delta": 0.007375068496912718,
      "fcm_dpo/margin": 45.301212310791016,
      "fcm_dpo/q_t": 0.43097594380378723,
      "grad_norm": 34.023258209228516,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 1.4505891799926758,
      "logits/rejected": 1.5759941339492798,
      "logps/chosen": -306.5923767089844,
      "logps/ref_chosen": -270.17547607421875,
      "logps/ref_rejected": -257.6781311035156,
      "logps/rejected": -339.396240234375,
      "loss": 4.6583,
      "margin_dpo/margin_mean": 45.301212310791016,
      "margin_dpo/margin_std": 68.16083526611328,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -33.978614807128906,
      "KL/mean": -59.29860305786133,
      "KL/rejected_KL_mean": -84.61858367919922,
      "KL/std": 65.61349487304688,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.006474010646343231,
      "fcm_dpo/delta": -0.003122996538877487,
      "fcm_dpo/margin": 50.63996887207031,
      "fcm_dpo/q_t": 0.42297840118408203,
      "grad_norm": 28.804590225219727,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 1.252758264541626,
      "logits/rejected": 1.4063106775283813,
      "logps/chosen": -297.7408752441406,
      "logps/ref_chosen": -263.76226806640625,
      "logps/ref_rejected": -263.8758850097656,
      "logps/rejected": -348.4944763183594,
      "loss": 4.6242,
      "margin_dpo/margin_mean": 50.63996887207031,
      "margin_dpo/margin_std": 80.85758972167969,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -32.47196960449219,
      "KL/mean": -57.24422073364258,
      "KL/rejected_KL_mean": -82.0164794921875,
      "KL/std": 61.9453239440918,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.006435226649045944,
      "fcm_dpo/delta": -0.012770434841513634,
      "fcm_dpo/margin": 49.544517517089844,
      "fcm_dpo/q_t": 0.42454472184181213,
      "grad_norm": 36.39311981201172,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": 1.048301339149475,
      "logits/rejected": 0.9876877069473267,
      "logps/chosen": -307.14434814453125,
      "logps/ref_chosen": -274.67242431640625,
      "logps/ref_rejected": -242.1553955078125,
      "logps/rejected": -324.171875,
      "loss": 4.6037,
      "margin_dpo/margin_mean": 49.544517517089844,
      "margin_dpo/margin_std": 72.88914489746094,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -34.53045654296875,
      "KL/mean": -57.726802825927734,
      "KL/rejected_KL_mean": -80.92315673828125,
      "KL/std": 66.43037414550781,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.006418707314878702,
      "fcm_dpo/delta": -0.02673560008406639,
      "fcm_dpo/margin": 46.39269256591797,
      "fcm_dpo/q_t": 0.4300708770751953,
      "grad_norm": 31.59515953063965,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": 1.507441520690918,
      "logits/rejected": 1.6033644676208496,
      "logps/chosen": -329.71832275390625,
      "logps/ref_chosen": -295.1878662109375,
      "logps/ref_rejected": -271.3431091308594,
      "logps/rejected": -352.2662658691406,
      "loss": 4.7127,
      "margin_dpo/margin_mean": 46.39269256591797,
      "margin_dpo/margin_std": 77.8595962524414,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -32.32948684692383,
      "KL/mean": -56.868263244628906,
      "KL/rejected_KL_mean": -81.40704345703125,
      "KL/std": 64.89295959472656,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.006277768407016993,
      "fcm_dpo/delta": 0.0006561724003404379,
      "fcm_dpo/margin": 49.07756423950195,
      "fcm_dpo/q_t": 0.4272412657737732,
      "grad_norm": 32.50770568847656,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": 1.221778154373169,
      "logits/rejected": 1.2639273405075073,
      "logps/chosen": -322.0973205566406,
      "logps/ref_chosen": -289.767822265625,
      "logps/ref_rejected": -267.0824279785156,
      "logps/rejected": -348.48944091796875,
      "loss": 4.653,
      "margin_dpo/margin_mean": 49.07756423950195,
      "margin_dpo/margin_std": 77.81268310546875,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -29.39300537109375,
      "KL/mean": -56.98039627075195,
      "KL/rejected_KL_mean": -84.56779479980469,
      "KL/std": 65.70218658447266,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.006232062354683876,
      "fcm_dpo/delta": -0.0027414504438638687,
      "fcm_dpo/margin": 55.17478942871094,
      "fcm_dpo/q_t": 0.41939306259155273,
      "grad_norm": 32.32416534423828,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": 1.590125322341919,
      "logits/rejected": 1.6105413436889648,
      "logps/chosen": -327.81207275390625,
      "logps/ref_chosen": -298.4190673828125,
      "logps/ref_rejected": -266.64447021484375,
      "logps/rejected": -351.2122497558594,
      "loss": 4.5149,
      "margin_dpo/margin_mean": 55.17478942871094,
      "margin_dpo/margin_std": 76.68122863769531,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -35.7225341796875,
      "KL/mean": -63.44451141357422,
      "KL/rejected_KL_mean": -91.16650390625,
      "KL/std": 67.27351379394531,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.006221615243703127,
      "fcm_dpo/delta": -0.0099312923848629,
      "fcm_dpo/margin": 55.4439582824707,
      "fcm_dpo/q_t": 0.4192855954170227,
      "grad_norm": 49.50067901611328,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": 1.365885615348816,
      "logits/rejected": 1.362638235092163,
      "logps/chosen": -337.952392578125,
      "logps/ref_chosen": -302.2298583984375,
      "logps/ref_rejected": -272.7162170410156,
      "logps/rejected": -363.8826904296875,
      "loss": 4.5238,
      "margin_dpo/margin_mean": 55.44395446777344,
      "margin_dpo/margin_std": 78.52589416503906,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -27.04343032836914,
      "KL/mean": -52.53056335449219,
      "KL/rejected_KL_mean": -78.01768493652344,
      "KL/std": 63.6679573059082,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.006176614668220282,
      "fcm_dpo/delta": 0.000790424644947052,
      "fcm_dpo/margin": 50.974266052246094,
      "fcm_dpo/q_t": 0.4257015287876129,
      "grad_norm": 43.25516891479492,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": 1.3486686944961548,
      "logits/rejected": 1.3264087438583374,
      "logps/chosen": -273.8968505859375,
      "logps/ref_chosen": -246.85342407226562,
      "logps/ref_rejected": -216.5264892578125,
      "logps/rejected": -294.5441589355469,
      "loss": 4.5975,
      "margin_dpo/margin_mean": 50.974266052246094,
      "margin_dpo/margin_std": 73.50650024414062,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -31.23714828491211,
      "KL/mean": -57.99238204956055,
      "KL/rejected_KL_mean": -84.74761962890625,
      "KL/std": 65.0190658569336,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.006196119822561741,
      "fcm_dpo/delta": -0.005425549112260342,
      "fcm_dpo/margin": 53.51048278808594,
      "fcm_dpo/q_t": 0.4209233522415161,
      "grad_norm": 41.187705993652344,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": 1.2759814262390137,
      "logits/rejected": 1.3592501878738403,
      "logps/chosen": -293.88677978515625,
      "logps/ref_chosen": -262.649658203125,
      "logps/ref_rejected": -251.24945068359375,
      "logps/rejected": -335.9970397949219,
      "loss": 4.529,
      "margin_dpo/margin_mean": 53.51048278808594,
      "margin_dpo/margin_std": 70.92572784423828,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -29.581274032592773,
      "KL/mean": -57.73560333251953,
      "KL/rejected_KL_mean": -85.88993835449219,
      "KL/std": 64.10833740234375,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.006164859049022198,
      "fcm_dpo/delta": -0.008141260594129562,
      "fcm_dpo/margin": 56.30867004394531,
      "fcm_dpo/q_t": 0.4172815680503845,
      "grad_norm": 33.09791946411133,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": 1.3359382152557373,
      "logits/rejected": 1.3729983568191528,
      "logps/chosen": -322.9676208496094,
      "logps/ref_chosen": -293.3863220214844,
      "logps/ref_rejected": -238.86026000976562,
      "logps/rejected": -324.7502136230469,
      "loss": 4.4439,
      "margin_dpo/margin_mean": 56.30867385864258,
      "margin_dpo/margin_std": 65.52816009521484,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -27.324487686157227,
      "KL/mean": -53.60966873168945,
      "KL/rejected_KL_mean": -79.89485168457031,
      "KL/std": 64.10770416259766,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.0062089599668979645,
      "fcm_dpo/delta": 0.004322844557464123,
      "fcm_dpo/margin": 52.57035827636719,
      "fcm_dpo/q_t": 0.42285874485969543,
      "grad_norm": 37.46158981323242,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 1.3401055335998535,
      "logits/rejected": 1.3668043613433838,
      "logps/chosen": -322.82830810546875,
      "logps/ref_chosen": -295.5038146972656,
      "logps/ref_rejected": -278.524169921875,
      "logps/rejected": -358.4190368652344,
      "loss": 4.6015,
      "margin_dpo/margin_mean": 52.57035827636719,
      "margin_dpo/margin_std": 80.17140197753906,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -31.233789443969727,
      "KL/mean": -53.93647003173828,
      "KL/rejected_KL_mean": -76.63916015625,
      "KL/std": 64.6318359375,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.00616700341925025,
      "fcm_dpo/delta": -0.004503438714891672,
      "fcm_dpo/margin": 45.40536880493164,
      "fcm_dpo/q_t": 0.43364208936691284,
      "grad_norm": 38.812923431396484,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 1.6332988739013672,
      "logits/rejected": 1.5750645399093628,
      "logps/chosen": -311.9891662597656,
      "logps/ref_chosen": -280.7553405761719,
      "logps/ref_rejected": -237.47286987304688,
      "logps/rejected": -314.1120300292969,
      "loss": 4.7141,
      "margin_dpo/margin_mean": 45.40536880493164,
      "margin_dpo/margin_std": 74.22958374023438,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -29.310546875,
      "KL/mean": -57.61056900024414,
      "KL/rejected_KL_mean": -85.91058349609375,
      "KL/std": 65.92823791503906,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.006200039759278297,
      "fcm_dpo/delta": 0.010488786734640598,
      "fcm_dpo/margin": 56.600040435791016,
      "fcm_dpo/q_t": 0.41681724786758423,
      "grad_norm": 30.939517974853516,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 1.7061829566955566,
      "logits/rejected": 1.7258108854293823,
      "logps/chosen": -328.40185546875,
      "logps/ref_chosen": -299.09130859375,
      "logps/ref_rejected": -268.9637756347656,
      "logps/rejected": -354.87432861328125,
      "loss": 4.4698,
      "margin_dpo/margin_mean": 56.600040435791016,
      "margin_dpo/margin_std": 72.45549774169922,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -39.896034240722656,
      "KL/mean": -58.02458953857422,
      "KL/rejected_KL_mean": -76.15313720703125,
      "KL/std": 63.74956512451172,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.0062033627182245255,
      "fcm_dpo/delta": 0.001788802444934845,
      "fcm_dpo/margin": 36.25709915161133,
      "fcm_dpo/q_t": 0.4467847943305969,
      "grad_norm": 47.29400634765625,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": 1.5097765922546387,
      "logits/rejected": 1.4593255519866943,
      "logps/chosen": -316.59210205078125,
      "logps/ref_chosen": -276.696044921875,
      "logps/ref_rejected": -225.78790283203125,
      "logps/rejected": -301.9410400390625,
      "loss": 4.9605,
      "margin_dpo/margin_mean": 36.25709915161133,
      "margin_dpo/margin_std": 81.2262954711914,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -26.79886817932129,
      "KL/mean": -57.069007873535156,
      "KL/rejected_KL_mean": -87.33915710449219,
      "KL/std": 65.11820220947266,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.006056767888367176,
      "fcm_dpo/delta": -0.032869596034288406,
      "fcm_dpo/margin": 60.54029083251953,
      "fcm_dpo/q_t": 0.41299647092819214,
      "grad_norm": 39.03342819213867,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": 1.5260910987854004,
      "logits/rejected": 1.6223206520080566,
      "logps/chosen": -257.3838806152344,
      "logps/ref_chosen": -230.58499145507812,
      "logps/ref_rejected": -229.8074493408203,
      "logps/rejected": -317.1466064453125,
      "loss": 4.4011,
      "margin_dpo/margin_mean": 60.540287017822266,
      "margin_dpo/margin_std": 69.70601654052734,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -30.265159606933594,
      "KL/mean": -56.1893310546875,
      "KL/rejected_KL_mean": -82.1135025024414,
      "KL/std": 68.89250183105469,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.00600747624412179,
      "fcm_dpo/delta": -0.008517580106854439,
      "fcm_dpo/margin": 51.84834289550781,
      "fcm_dpo/q_t": 0.427543044090271,
      "grad_norm": 23.125743865966797,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": 1.418918490409851,
      "logits/rejected": 1.4608922004699707,
      "logps/chosen": -300.8211975097656,
      "logps/ref_chosen": -270.5559997558594,
      "logps/ref_rejected": -266.328369140625,
      "logps/rejected": -348.4418640136719,
      "loss": 4.6702,
      "margin_dpo/margin_mean": 51.84834289550781,
      "margin_dpo/margin_std": 86.80340576171875,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -30.674449920654297,
      "KL/mean": -56.10780334472656,
      "KL/rejected_KL_mean": -81.54115295410156,
      "KL/std": 63.30500030517578,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.006059874314814806,
      "fcm_dpo/delta": 0.01805487833917141,
      "fcm_dpo/margin": 50.866722106933594,
      "fcm_dpo/q_t": 0.42698514461517334,
      "grad_norm": 34.27556228637695,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": 1.3227207660675049,
      "logits/rejected": 1.4082999229431152,
      "logps/chosen": -263.9130554199219,
      "logps/ref_chosen": -233.2386016845703,
      "logps/ref_rejected": -258.154541015625,
      "logps/rejected": -339.6956787109375,
      "loss": 4.6595,
      "margin_dpo/margin_mean": 50.866722106933594,
      "margin_dpo/margin_std": 83.05711364746094,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -18.17897605895996,
      "KL/mean": -47.91671371459961,
      "KL/rejected_KL_mean": -77.65446472167969,
      "KL/std": 64.77115631103516,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.006034876685589552,
      "fcm_dpo/delta": -0.02551138773560524,
      "fcm_dpo/margin": 59.47547912597656,
      "fcm_dpo/q_t": 0.41513729095458984,
      "grad_norm": 27.70412254333496,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": 1.5885391235351562,
      "logits/rejected": 1.620835781097412,
      "logps/chosen": -286.72515869140625,
      "logps/ref_chosen": -268.5461730957031,
      "logps/ref_rejected": -266.5316467285156,
      "logps/rejected": -344.1861267089844,
      "loss": 4.4801,
      "margin_dpo/margin_mean": 59.47548294067383,
      "margin_dpo/margin_std": 79.43744659423828,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -30.534988403320312,
      "KL/mean": -57.53690719604492,
      "KL/rejected_KL_mean": -84.5388412475586,
      "KL/std": 66.0572509765625,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.005921962205320597,
      "fcm_dpo/delta": -0.01666666381061077,
      "fcm_dpo/margin": 54.00384521484375,
      "fcm_dpo/q_t": 0.42398443818092346,
      "grad_norm": 65.05178833007812,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": 1.3509434461593628,
      "logits/rejected": 1.378374695777893,
      "logps/chosen": -325.4117736816406,
      "logps/ref_chosen": -294.8768005371094,
      "logps/ref_rejected": -281.09771728515625,
      "logps/rejected": -365.6365661621094,
      "loss": 4.5657,
      "margin_dpo/margin_mean": 54.00384521484375,
      "margin_dpo/margin_std": 73.71467590332031,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -41.40918731689453,
      "KL/mean": -64.10562896728516,
      "KL/rejected_KL_mean": -86.80208587646484,
      "KL/std": 70.4186019897461,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.005927873309701681,
      "fcm_dpo/delta": 0.0143583994358778,
      "fcm_dpo/margin": 45.39289855957031,
      "fcm_dpo/q_t": 0.4369352459907532,
      "grad_norm": 35.845619201660156,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": 1.502881407737732,
      "logits/rejected": 1.5229520797729492,
      "logps/chosen": -353.7070007324219,
      "logps/ref_chosen": -312.2977600097656,
      "logps/ref_rejected": -304.6536560058594,
      "logps/rejected": -391.45574951171875,
      "loss": 4.8087,
      "margin_dpo/margin_mean": 45.39289855957031,
      "margin_dpo/margin_std": 85.92943572998047,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -38.76654815673828,
      "KL/mean": -57.95475387573242,
      "KL/rejected_KL_mean": -77.14295959472656,
      "KL/std": 67.43707275390625,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.005983759183436632,
      "fcm_dpo/delta": 0.006432502064853907,
      "fcm_dpo/margin": 38.37641525268555,
      "fcm_dpo/q_t": 0.44610095024108887,
      "grad_norm": 29.208524703979492,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 1.1763830184936523,
      "logits/rejected": 1.2363395690917969,
      "logps/chosen": -332.97076416015625,
      "logps/ref_chosen": -294.2042236328125,
      "logps/ref_rejected": -296.2102355957031,
      "logps/rejected": -373.35321044921875,
      "loss": 4.9188,
      "margin_dpo/margin_mean": 38.37641525268555,
      "margin_dpo/margin_std": 81.46822357177734,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -30.773679733276367,
      "KL/mean": -54.798736572265625,
      "KL/rejected_KL_mean": -78.82379913330078,
      "KL/std": 66.06912994384766,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.005997148342430592,
      "fcm_dpo/delta": 0.007573171518743038,
      "fcm_dpo/margin": 48.05011749267578,
      "fcm_dpo/q_t": 0.4319855570793152,
      "grad_norm": 38.09358596801758,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.4234950542449951,
      "logits/rejected": 1.442479133605957,
      "logps/chosen": -299.0863037109375,
      "logps/ref_chosen": -268.3126525878906,
      "logps/ref_rejected": -255.5543212890625,
      "logps/rejected": -334.37811279296875,
      "loss": 4.695,
      "margin_dpo/margin_mean": 48.05011749267578,
      "margin_dpo/margin_std": 76.60541534423828,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -33.80150604248047,
      "KL/mean": -63.746864318847656,
      "KL/rejected_KL_mean": -93.69223022460938,
      "KL/std": 67.40896606445312,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.005948436446487904,
      "fcm_dpo/delta": -0.024670947343111038,
      "fcm_dpo/margin": 59.8907356262207,
      "fcm_dpo/q_t": 0.4156443476676941,
      "grad_norm": 29.85625648498535,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": 1.2524160146713257,
      "logits/rejected": 1.2757524251937866,
      "logps/chosen": -297.5519714355469,
      "logps/ref_chosen": -263.75042724609375,
      "logps/ref_rejected": -234.5144500732422,
      "logps/rejected": -328.2066650390625,
      "loss": 4.4999,
      "margin_dpo/margin_mean": 59.89072799682617,
      "margin_dpo/margin_std": 83.02149963378906,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -30.637588500976562,
      "KL/mean": -52.648719787597656,
      "KL/rejected_KL_mean": -74.65985870361328,
      "KL/std": 64.72459411621094,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.005930397193878889,
      "fcm_dpo/delta": 0.012042054906487465,
      "fcm_dpo/margin": 44.02225875854492,
      "fcm_dpo/q_t": 0.43769946694374084,
      "grad_norm": 49.863128662109375,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": 1.4099836349487305,
      "logits/rejected": 1.388910174369812,
      "logps/chosen": -317.4540100097656,
      "logps/ref_chosen": -286.81646728515625,
      "logps/ref_rejected": -260.4169616699219,
      "logps/rejected": -335.0768127441406,
      "loss": 4.7688,
      "margin_dpo/margin_mean": 44.022254943847656,
      "margin_dpo/margin_std": 73.6692123413086,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -29.114213943481445,
      "KL/mean": -55.259300231933594,
      "KL/rejected_KL_mean": -81.40438079833984,
      "KL/std": 67.7076644897461,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.005934357643127441,
      "fcm_dpo/delta": -0.023426614701747894,
      "fcm_dpo/margin": 52.2901496887207,
      "fcm_dpo/q_t": 0.42686495184898376,
      "grad_norm": 27.968448638916016,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": 1.2623775005340576,
      "logits/rejected": 1.263635516166687,
      "logps/chosen": -322.4251403808594,
      "logps/ref_chosen": -293.3109130859375,
      "logps/ref_rejected": -264.1449279785156,
      "logps/rejected": -345.54931640625,
      "loss": 4.655,
      "margin_dpo/margin_mean": 52.2901496887207,
      "margin_dpo/margin_std": 82.27152252197266,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -46.52915954589844,
      "KL/mean": -68.65123748779297,
      "KL/rejected_KL_mean": -90.77330780029297,
      "KL/std": 65.12398529052734,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.005822364706546068,
      "fcm_dpo/delta": -0.003225984051823616,
      "fcm_dpo/margin": 44.2441520690918,
      "fcm_dpo/q_t": 0.4392317831516266,
      "grad_norm": 38.33371353149414,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 1.3433775901794434,
      "logits/rejected": 1.4395381212234497,
      "logps/chosen": -316.2855224609375,
      "logps/ref_chosen": -269.75634765625,
      "logps/ref_rejected": -241.08438110351562,
      "logps/rejected": -331.857666015625,
      "loss": 4.832,
      "margin_dpo/margin_mean": 44.24415588378906,
      "margin_dpo/margin_std": 84.15264892578125,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -33.95994567871094,
      "KL/mean": -55.601619720458984,
      "KL/rejected_KL_mean": -77.2432861328125,
      "KL/std": 65.44961547851562,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.0059359390288591385,
      "fcm_dpo/delta": 0.03386285901069641,
      "fcm_dpo/margin": 43.283355712890625,
      "fcm_dpo/q_t": 0.4397016763687134,
      "grad_norm": 29.368244171142578,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": 1.5255879163742065,
      "logits/rejected": 1.5564048290252686,
      "logps/chosen": -271.33172607421875,
      "logps/ref_chosen": -237.37179565429688,
      "logps/ref_rejected": -253.5463409423828,
      "logps/rejected": -330.7896423339844,
      "loss": 4.7862,
      "margin_dpo/margin_mean": 43.28335952758789,
      "margin_dpo/margin_std": 75.25077056884766,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -40.24750518798828,
      "KL/mean": -66.32674407958984,
      "KL/rejected_KL_mean": -92.4059829711914,
      "KL/std": 65.7067642211914,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.006111672148108482,
      "fcm_dpo/delta": 0.0293545164167881,
      "fcm_dpo/margin": 52.15848159790039,
      "fcm_dpo/q_t": 0.4255383014678955,
      "grad_norm": 40.590476989746094,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": 1.1298191547393799,
      "logits/rejected": 1.1785821914672852,
      "logps/chosen": -301.55517578125,
      "logps/ref_chosen": -261.3076477050781,
      "logps/ref_rejected": -252.3170166015625,
      "logps/rejected": -344.7229919433594,
      "loss": 4.636,
      "margin_dpo/margin_mean": 52.158485412597656,
      "margin_dpo/margin_std": 83.68705749511719,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -20.861732482910156,
      "KL/mean": -50.94083023071289,
      "KL/rejected_KL_mean": -81.0199203491211,
      "KL/std": 67.01274871826172,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.006185787729918957,
      "fcm_dpo/delta": 0.013499358668923378,
      "fcm_dpo/margin": 60.15819549560547,
      "fcm_dpo/q_t": 0.4128686785697937,
      "grad_norm": 30.557289123535156,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": 1.5213916301727295,
      "logits/rejected": 1.5185266733169556,
      "logps/chosen": -289.63348388671875,
      "logps/ref_chosen": -268.7717590332031,
      "logps/ref_rejected": -232.3352813720703,
      "logps/rejected": -313.3551940917969,
      "loss": 4.4273,
      "margin_dpo/margin_mean": 60.15819549560547,
      "margin_dpo/margin_std": 78.73454284667969,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -31.528644561767578,
      "KL/mean": -62.07075500488281,
      "KL/rejected_KL_mean": -92.61287689208984,
      "KL/std": 69.99024200439453,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.006258091423660517,
      "fcm_dpo/delta": -0.004390798509120941,
      "fcm_dpo/margin": 61.084232330322266,
      "fcm_dpo/q_t": 0.41055190563201904,
      "grad_norm": 46.977210998535156,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": 1.3392860889434814,
      "logits/rejected": 1.3284192085266113,
      "logps/chosen": -325.14178466796875,
      "logps/ref_chosen": -293.6131286621094,
      "logps/ref_rejected": -281.87738037109375,
      "logps/rejected": -374.490234375,
      "loss": 4.4325,
      "margin_dpo/margin_mean": 61.084232330322266,
      "margin_dpo/margin_std": 82.9106674194336,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -34.095001220703125,
      "KL/mean": -58.95244216918945,
      "KL/rejected_KL_mean": -83.80989074707031,
      "KL/std": 68.02757263183594,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.006173284724354744,
      "fcm_dpo/delta": 0.0001188395544886589,
      "fcm_dpo/margin": 49.71488571166992,
      "fcm_dpo/q_t": 0.42815113067626953,
      "grad_norm": 38.775726318359375,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 1.0285401344299316,
      "logits/rejected": 1.1454577445983887,
      "logps/chosen": -285.85205078125,
      "logps/ref_chosen": -251.75704956054688,
      "logps/ref_rejected": -255.76441955566406,
      "logps/rejected": -339.5743103027344,
      "loss": 4.6591,
      "margin_dpo/margin_mean": 49.71488571166992,
      "margin_dpo/margin_std": 79.7890853881836,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -49.50113296508789,
      "KL/mean": -71.08265686035156,
      "KL/rejected_KL_mean": -92.66419219970703,
      "KL/std": 70.87718200683594,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.006193068809807301,
      "fcm_dpo/delta": -0.008520380593836308,
      "fcm_dpo/margin": 43.16305923461914,
      "fcm_dpo/q_t": 0.4375413656234741,
      "grad_norm": 35.230308532714844,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 1.406817078590393,
      "logits/rejected": 1.5178782939910889,
      "logps/chosen": -356.1487731933594,
      "logps/ref_chosen": -306.64764404296875,
      "logps/ref_rejected": -301.4248352050781,
      "logps/rejected": -394.0890197753906,
      "loss": 4.8161,
      "margin_dpo/margin_mean": 43.163063049316406,
      "margin_dpo/margin_std": 82.33653259277344,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -30.55238151550293,
      "KL/mean": -56.85309600830078,
      "KL/rejected_KL_mean": -83.15381622314453,
      "KL/std": 67.6126480102539,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.0062232050113379955,
      "fcm_dpo/delta": 0.01414551306515932,
      "fcm_dpo/margin": 52.6014404296875,
      "fcm_dpo/q_t": 0.42304959893226624,
      "grad_norm": 29.2943115234375,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": 1.4825918674468994,
      "logits/rejected": 1.5440407991409302,
      "logps/chosen": -293.2213134765625,
      "logps/ref_chosen": -262.6689147949219,
      "logps/ref_rejected": -260.8409118652344,
      "logps/rejected": -343.9947204589844,
      "loss": 4.5664,
      "margin_dpo/margin_mean": 52.6014404296875,
      "margin_dpo/margin_std": 75.2231216430664,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -33.05850601196289,
      "KL/mean": -55.59626007080078,
      "KL/rejected_KL_mean": -78.13400268554688,
      "KL/std": 68.03715515136719,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.006305000279098749,
      "fcm_dpo/delta": 0.02910422533750534,
      "fcm_dpo/margin": 45.07551956176758,
      "fcm_dpo/q_t": 0.43363693356513977,
      "grad_norm": 42.36564636230469,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": 1.4014748334884644,
      "logits/rejected": 1.3892173767089844,
      "logps/chosen": -332.6844787597656,
      "logps/ref_chosen": -299.6259765625,
      "logps/ref_rejected": -261.2701416015625,
      "logps/rejected": -339.40411376953125,
      "loss": 4.7464,
      "margin_dpo/margin_mean": 45.07551956176758,
      "margin_dpo/margin_std": 79.74722290039062,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -34.1021728515625,
      "KL/mean": -62.08058166503906,
      "KL/rejected_KL_mean": -90.05899047851562,
      "KL/std": 69.13385772705078,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.006418094038963318,
      "fcm_dpo/delta": -0.009951267391443253,
      "fcm_dpo/margin": 55.95680236816406,
      "fcm_dpo/q_t": 0.41614484786987305,
      "grad_norm": 31.544309616088867,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": 1.3380707502365112,
      "logits/rejected": 1.3846410512924194,
      "logps/chosen": -325.0526123046875,
      "logps/ref_chosen": -290.950439453125,
      "logps/ref_rejected": -255.15374755859375,
      "logps/rejected": -345.21270751953125,
      "loss": 4.5182,
      "margin_dpo/margin_mean": 55.95680236816406,
      "margin_dpo/margin_std": 81.17728424072266,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -30.38825798034668,
      "KL/mean": -51.85435485839844,
      "KL/rejected_KL_mean": -73.32046508789062,
      "KL/std": 66.37906646728516,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.006285062525421381,
      "fcm_dpo/delta": -0.010914524085819721,
      "fcm_dpo/margin": 42.932220458984375,
      "fcm_dpo/q_t": 0.43635475635528564,
      "grad_norm": 34.43415069580078,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": 1.4310325384140015,
      "logits/rejected": 1.5134990215301514,
      "logps/chosen": -303.65411376953125,
      "logps/ref_chosen": -273.2658996582031,
      "logps/ref_rejected": -263.9664306640625,
      "logps/rejected": -337.286865234375,
      "loss": 4.7561,
      "margin_dpo/margin_mean": 42.932220458984375,
      "margin_dpo/margin_std": 71.39842224121094,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -44.03504943847656,
      "KL/mean": -71.5798568725586,
      "KL/rejected_KL_mean": -99.12466430664062,
      "KL/std": 69.19415283203125,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.006047086324542761,
      "fcm_dpo/delta": -0.06337644159793854,
      "fcm_dpo/margin": 55.089622497558594,
      "fcm_dpo/q_t": 0.42114800214767456,
      "grad_norm": 24.601913452148438,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": 1.3760749101638794,
      "logits/rejected": 1.445542573928833,
      "logps/chosen": -306.71722412109375,
      "logps/ref_chosen": -262.6821594238281,
      "logps/ref_rejected": -226.46859741210938,
      "logps/rejected": -325.59326171875,
      "loss": 4.573,
      "margin_dpo/margin_mean": 55.089622497558594,
      "margin_dpo/margin_std": 76.09601593017578,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -40.785282135009766,
      "KL/mean": -64.11566925048828,
      "KL/rejected_KL_mean": -87.44606018066406,
      "KL/std": 68.2197036743164,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.00599468220025301,
      "fcm_dpo/delta": 0.011973203159868717,
      "fcm_dpo/margin": 46.66077423095703,
      "fcm_dpo/q_t": 0.4338519871234894,
      "grad_norm": 29.438217163085938,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": 1.161898136138916,
      "logits/rejected": 1.173927903175354,
      "logps/chosen": -304.60986328125,
      "logps/ref_chosen": -263.8246154785156,
      "logps/ref_rejected": -237.9121551513672,
      "logps/rejected": -325.3582458496094,
      "loss": 4.7303,
      "margin_dpo/margin_mean": 46.66077423095703,
      "margin_dpo/margin_std": 78.64490509033203,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -32.60622024536133,
      "KL/mean": -53.932796478271484,
      "KL/rejected_KL_mean": -75.25936889648438,
      "KL/std": 67.92752075195312,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.0060234153643250465,
      "fcm_dpo/delta": 0.0033203144557774067,
      "fcm_dpo/margin": 42.653141021728516,
      "fcm_dpo/q_t": 0.43946680426597595,
      "grad_norm": 35.388031005859375,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 1.2212731838226318,
      "logits/rejected": 1.256709337234497,
      "logps/chosen": -310.75958251953125,
      "logps/ref_chosen": -278.1533203125,
      "logps/ref_rejected": -263.39691162109375,
      "logps/rejected": -338.65631103515625,
      "loss": 4.7978,
      "margin_dpo/margin_mean": 42.65314483642578,
      "margin_dpo/margin_std": 75.15667724609375,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -38.41587829589844,
      "KL/mean": -66.45494079589844,
      "KL/rejected_KL_mean": -94.49398803710938,
      "KL/std": 71.41858673095703,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.006034653168171644,
      "fcm_dpo/delta": -0.00029916176572442055,
      "fcm_dpo/margin": 56.0781135559082,
      "fcm_dpo/q_t": 0.4207199811935425,
      "grad_norm": 39.72799301147461,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": 1.6147540807724,
      "logits/rejected": 1.6720943450927734,
      "logps/chosen": -356.87091064453125,
      "logps/ref_chosen": -318.455078125,
      "logps/ref_rejected": -288.5691833496094,
      "logps/rejected": -383.0631408691406,
      "loss": 4.5454,
      "margin_dpo/margin_mean": 56.07810974121094,
      "margin_dpo/margin_std": 79.39434051513672,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -41.41194534301758,
      "KL/mean": -66.7266616821289,
      "KL/rejected_KL_mean": -92.04136657714844,
      "KL/std": 70.17897033691406,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.006033113691955805,
      "fcm_dpo/delta": -0.013566947542130947,
      "fcm_dpo/margin": 50.629417419433594,
      "fcm_dpo/q_t": 0.42806947231292725,
      "grad_norm": 41.588932037353516,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 1.285040020942688,
      "logits/rejected": 1.2778687477111816,
      "logps/chosen": -325.6011047363281,
      "logps/ref_chosen": -284.1891784667969,
      "logps/ref_rejected": -247.94589233398438,
      "logps/rejected": -339.98724365234375,
      "loss": 4.6778,
      "margin_dpo/margin_mean": 50.629417419433594,
      "margin_dpo/margin_std": 81.14497375488281,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -41.95210647583008,
      "KL/mean": -58.345672607421875,
      "KL/rejected_KL_mean": -74.73924255371094,
      "KL/std": 68.75979614257812,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.005949968937784433,
      "fcm_dpo/delta": -0.010718228295445442,
      "fcm_dpo/margin": 32.78713607788086,
      "fcm_dpo/q_t": 0.45472219586372375,
      "grad_norm": 39.65448760986328,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 1.4258158206939697,
      "logits/rejected": 1.4016984701156616,
      "logps/chosen": -345.26019287109375,
      "logps/ref_chosen": -303.30810546875,
      "logps/ref_rejected": -260.1866455078125,
      "logps/rejected": -334.9259033203125,
      "loss": 5.0672,
      "margin_dpo/margin_mean": 32.787132263183594,
      "margin_dpo/margin_std": 84.1899642944336,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -34.616188049316406,
      "KL/mean": -63.967079162597656,
      "KL/rejected_KL_mean": -93.3179702758789,
      "KL/std": 69.77045440673828,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.005903993267565966,
      "fcm_dpo/delta": -0.008995930664241314,
      "fcm_dpo/margin": 58.7017822265625,
      "fcm_dpo/q_t": 0.4183250963687897,
      "grad_norm": 26.328838348388672,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": 1.438184380531311,
      "logits/rejected": 1.624953269958496,
      "logps/chosen": -281.4233093261719,
      "logps/ref_chosen": -246.80709838867188,
      "logps/ref_rejected": -253.13223266601562,
      "logps/rejected": -346.4501953125,
      "loss": 4.503,
      "margin_dpo/margin_mean": 58.701786041259766,
      "margin_dpo/margin_std": 77.34344482421875,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -40.567115783691406,
      "KL/mean": -68.51258850097656,
      "KL/rejected_KL_mean": -96.45805358886719,
      "KL/std": 69.24854278564453,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.005843131337314844,
      "fcm_dpo/delta": -0.019068751484155655,
      "fcm_dpo/margin": 55.890953063964844,
      "fcm_dpo/q_t": 0.4227598011493683,
      "grad_norm": 32.21244430541992,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": 1.342195987701416,
      "logits/rejected": 1.4221916198730469,
      "logps/chosen": -302.1685791015625,
      "logps/ref_chosen": -261.6014709472656,
      "logps/ref_rejected": -262.74761962890625,
      "logps/rejected": -359.2057189941406,
      "loss": 4.5659,
      "margin_dpo/margin_mean": 55.890953063964844,
      "margin_dpo/margin_std": 77.97010803222656,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -40.940673828125,
      "KL/mean": -58.819862365722656,
      "KL/rejected_KL_mean": -76.69905090332031,
      "KL/std": 69.59741973876953,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.005793419200927019,
      "fcm_dpo/delta": 0.004706468433141708,
      "fcm_dpo/margin": 35.75838851928711,
      "fcm_dpo/q_t": 0.4515298008918762,
      "grad_norm": 29.70002555847168,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": 1.395818829536438,
      "logits/rejected": 1.5184588432312012,
      "logps/chosen": -293.089111328125,
      "logps/ref_chosen": -252.14840698242188,
      "logps/ref_rejected": -244.1017608642578,
      "logps/rejected": -320.8007507324219,
      "loss": 5.0242,
      "margin_dpo/margin_mean": 35.75838851928711,
      "margin_dpo/margin_std": 87.11705017089844,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -42.84786605834961,
      "KL/mean": -70.09430694580078,
      "KL/rejected_KL_mean": -97.34074401855469,
      "KL/std": 70.56648254394531,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.005826142150908709,
      "fcm_dpo/delta": 0.00946696475148201,
      "fcm_dpo/margin": 54.49287414550781,
      "fcm_dpo/q_t": 0.4250933527946472,
      "grad_norm": 40.87082290649414,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": 1.2719687223434448,
      "logits/rejected": 1.3822062015533447,
      "logps/chosen": -329.0104064941406,
      "logps/ref_chosen": -286.16253662109375,
      "logps/ref_rejected": -272.3275146484375,
      "logps/rejected": -369.66827392578125,
      "loss": 4.6011,
      "margin_dpo/margin_mean": 54.49287414550781,
      "margin_dpo/margin_std": 80.34129333496094,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -34.94264221191406,
      "KL/mean": -57.91682434082031,
      "KL/rejected_KL_mean": -80.89100646972656,
      "KL/std": 66.35358428955078,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.005869910586625338,
      "fcm_dpo/delta": 0.011351291090250015,
      "fcm_dpo/margin": 45.9483757019043,
      "fcm_dpo/q_t": 0.43602466583251953,
      "grad_norm": 33.91718673706055,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": 1.3714754581451416,
      "logits/rejected": 1.4286210536956787,
      "logps/chosen": -233.0954132080078,
      "logps/ref_chosen": -198.1527862548828,
      "logps/ref_rejected": -209.80569458007812,
      "logps/rejected": -290.69671630859375,
      "loss": 4.7519,
      "margin_dpo/margin_mean": 45.9483757019043,
      "margin_dpo/margin_std": 76.4412612915039,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -43.91596221923828,
      "KL/mean": -69.5147476196289,
      "KL/rejected_KL_mean": -95.113525390625,
      "KL/std": 68.429931640625,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.0059114256873726845,
      "fcm_dpo/delta": 0.006022127345204353,
      "fcm_dpo/margin": 51.19756317138672,
      "fcm_dpo/q_t": 0.4293052852153778,
      "grad_norm": 27.665699005126953,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": 1.4295636415481567,
      "logits/rejected": 1.500382900238037,
      "logps/chosen": -300.4231262207031,
      "logps/ref_chosen": -256.50714111328125,
      "logps/ref_rejected": -237.40769958496094,
      "logps/rejected": -332.5212097167969,
      "loss": 4.6695,
      "margin_dpo/margin_mean": 51.19755935668945,
      "margin_dpo/margin_std": 80.88502502441406,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -32.26213836669922,
      "KL/mean": -60.65143966674805,
      "KL/rejected_KL_mean": -89.0407485961914,
      "KL/std": 69.44224548339844,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.005900029093027115,
      "fcm_dpo/delta": -0.01149833481758833,
      "fcm_dpo/margin": 56.77861404418945,
      "fcm_dpo/q_t": 0.421244353055954,
      "grad_norm": 33.19036865234375,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": 1.4738943576812744,
      "logits/rejected": 1.4208471775054932,
      "logps/chosen": -324.70941162109375,
      "logps/ref_chosen": -292.447265625,
      "logps/ref_rejected": -255.47872924804688,
      "logps/rejected": -344.51947021484375,
      "loss": 4.5711,
      "margin_dpo/margin_mean": 56.77861404418945,
      "margin_dpo/margin_std": 84.26338195800781,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -35.188323974609375,
      "KL/mean": -61.71752166748047,
      "KL/rejected_KL_mean": -88.2467041015625,
      "KL/std": 70.39640045166016,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.005903935991227627,
      "fcm_dpo/delta": 0.019214006140828133,
      "fcm_dpo/margin": 53.058387756347656,
      "fcm_dpo/q_t": 0.42590513825416565,
      "grad_norm": 31.160329818725586,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": 1.600233793258667,
      "logits/rejected": 1.588295340538025,
      "logps/chosen": -328.9482727050781,
      "logps/ref_chosen": -293.7599792480469,
      "logps/ref_rejected": -259.4142761230469,
      "logps/rejected": -347.6610107421875,
      "loss": 4.6162,
      "margin_dpo/margin_mean": 53.058387756347656,
      "margin_dpo/margin_std": 79.46435546875,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -41.158424377441406,
      "KL/mean": -65.04624938964844,
      "KL/rejected_KL_mean": -88.93407440185547,
      "KL/std": 68.92320251464844,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.005948488600552082,
      "fcm_dpo/delta": -0.012084469199180603,
      "fcm_dpo/margin": 47.77565002441406,
      "fcm_dpo/q_t": 0.43246906995773315,
      "grad_norm": 29.21001625061035,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 1.3149760961532593,
      "logits/rejected": 1.2524644136428833,
      "logps/chosen": -342.07220458984375,
      "logps/ref_chosen": -300.91375732421875,
      "logps/ref_rejected": -256.4080810546875,
      "logps/rejected": -345.3421325683594,
      "loss": 4.6761,
      "margin_dpo/margin_mean": 47.775657653808594,
      "margin_dpo/margin_std": 70.1992416381836,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -28.942506790161133,
      "KL/mean": -61.01874542236328,
      "KL/rejected_KL_mean": -93.09498596191406,
      "KL/std": 71.69976043701172,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.005899815820157528,
      "fcm_dpo/delta": 0.0046116081066429615,
      "fcm_dpo/margin": 64.15248107910156,
      "fcm_dpo/q_t": 0.41122859716415405,
      "grad_norm": 35.52924728393555,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 1.1095117330551147,
      "logits/rejected": 1.1934186220169067,
      "logps/chosen": -285.5028076171875,
      "logps/ref_chosen": -256.5603332519531,
      "logps/ref_rejected": -271.043701171875,
      "logps/rejected": -364.138671875,
      "loss": 4.4409,
      "margin_dpo/margin_mean": 64.15248107910156,
      "margin_dpo/margin_std": 88.41272735595703,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -36.236793518066406,
      "KL/mean": -63.600379943847656,
      "KL/rejected_KL_mean": -90.96397399902344,
      "KL/std": 67.58488464355469,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.005911659449338913,
      "fcm_dpo/delta": 0.005211292300373316,
      "fcm_dpo/margin": 54.72719192504883,
      "fcm_dpo/q_t": 0.4238627254962921,
      "grad_norm": 32.33614730834961,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": 1.295014500617981,
      "logits/rejected": 1.1960289478302002,
      "logps/chosen": -330.937255859375,
      "logps/ref_chosen": -294.700439453125,
      "logps/ref_rejected": -239.67922973632812,
      "logps/rejected": -330.6432189941406,
      "loss": 4.5886,
      "margin_dpo/margin_mean": 54.72718811035156,
      "margin_dpo/margin_std": 79.90623474121094,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -34.245845794677734,
      "KL/mean": -65.21289825439453,
      "KL/rejected_KL_mean": -96.17996215820312,
      "KL/std": 72.66947174072266,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.005849492736160755,
      "fcm_dpo/delta": -0.025133563205599785,
      "fcm_dpo/margin": 61.93410873413086,
      "fcm_dpo/q_t": 0.4147513210773468,
      "grad_norm": 45.9963264465332,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": 1.3069310188293457,
      "logits/rejected": 1.3113911151885986,
      "logps/chosen": -327.92169189453125,
      "logps/ref_chosen": -293.67584228515625,
      "logps/ref_rejected": -260.85675048828125,
      "logps/rejected": -357.0367126464844,
      "loss": 4.4599,
      "margin_dpo/margin_mean": 61.93410873413086,
      "margin_dpo/margin_std": 81.77556610107422,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -53.62449264526367,
      "KL/mean": -76.43040466308594,
      "KL/rejected_KL_mean": -99.23634338378906,
      "KL/std": 68.5497817993164,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.005887560546398163,
      "fcm_dpo/delta": 0.02403995953500271,
      "fcm_dpo/margin": 45.611846923828125,
      "fcm_dpo/q_t": 0.436967134475708,
      "grad_norm": 33.84764862060547,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": 1.255437970161438,
      "logits/rejected": 1.3208142518997192,
      "logps/chosen": -332.5198669433594,
      "logps/ref_chosen": -278.8953857421875,
      "logps/ref_rejected": -283.8898620605469,
      "logps/rejected": -383.12615966796875,
      "loss": 4.7843,
      "margin_dpo/margin_mean": 45.611846923828125,
      "margin_dpo/margin_std": 80.90498352050781,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -33.240142822265625,
      "KL/mean": -63.88751983642578,
      "KL/rejected_KL_mean": -94.53489685058594,
      "KL/std": 71.28411865234375,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.005929033271968365,
      "fcm_dpo/delta": -0.006351620424538851,
      "fcm_dpo/margin": 61.29475402832031,
      "fcm_dpo/q_t": 0.41466769576072693,
      "grad_norm": 33.59861373901367,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": 1.3618028163909912,
      "logits/rejected": 1.4174528121948242,
      "logps/chosen": -298.884033203125,
      "logps/ref_chosen": -265.6439208984375,
      "logps/ref_rejected": -255.40138244628906,
      "logps/rejected": -349.9363098144531,
      "loss": 4.4506,
      "margin_dpo/margin_mean": 61.29475402832031,
      "margin_dpo/margin_std": 80.7364730834961,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -40.754112243652344,
      "KL/mean": -69.73933410644531,
      "KL/rejected_KL_mean": -98.72454833984375,
      "KL/std": 71.52763366699219,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.0058882213197648525,
      "fcm_dpo/delta": -0.0021855137310922146,
      "fcm_dpo/margin": 57.97043228149414,
      "fcm_dpo/q_t": 0.4192063808441162,
      "grad_norm": 33.443485260009766,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": 1.4809504747390747,
      "logits/rejected": 1.5085532665252686,
      "logps/chosen": -320.9786071777344,
      "logps/ref_chosen": -280.2244873046875,
      "logps/ref_rejected": -248.72366333007812,
      "logps/rejected": -347.44818115234375,
      "loss": 4.5242,
      "margin_dpo/margin_mean": 57.97043228149414,
      "margin_dpo/margin_std": 79.93954467773438,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -36.874267578125,
      "KL/mean": -66.8705825805664,
      "KL/rejected_KL_mean": -96.86690521240234,
      "KL/std": 70.92996215820312,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.005872255656868219,
      "fcm_dpo/delta": -0.015139848925173283,
      "fcm_dpo/margin": 59.99262237548828,
      "fcm_dpo/q_t": 0.4169343113899231,
      "grad_norm": 32.345157623291016,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": 1.215214490890503,
      "logits/rejected": 1.2097758054733276,
      "logps/chosen": -329.81097412109375,
      "logps/ref_chosen": -292.9366760253906,
      "logps/ref_rejected": -261.88763427734375,
      "logps/rejected": -358.7545471191406,
      "loss": 4.469,
      "margin_dpo/margin_mean": 59.99263000488281,
      "margin_dpo/margin_std": 77.49986267089844,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -29.079792022705078,
      "KL/mean": -63.07053756713867,
      "KL/rejected_KL_mean": -97.061279296875,
      "KL/std": 72.36804962158203,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.0057658846490085125,
      "fcm_dpo/delta": -0.027743961662054062,
      "fcm_dpo/margin": 67.98148345947266,
      "fcm_dpo/q_t": 0.40727487206459045,
      "grad_norm": 31.235498428344727,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": 1.2657430171966553,
      "logits/rejected": 1.3550244569778442,
      "logps/chosen": -313.98553466796875,
      "logps/ref_chosen": -284.90576171875,
      "logps/ref_rejected": -256.59197998046875,
      "logps/rejected": -353.6532287597656,
      "loss": 4.3467,
      "margin_dpo/margin_mean": 67.98148345947266,
      "margin_dpo/margin_std": 78.14146423339844,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -38.880332946777344,
      "KL/mean": -62.38481140136719,
      "KL/rejected_KL_mean": -85.8893051147461,
      "KL/std": 69.9710693359375,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.005694333929568529,
      "fcm_dpo/delta": -0.008864663541316986,
      "fcm_dpo/margin": 47.00896453857422,
      "fcm_dpo/q_t": 0.43668699264526367,
      "grad_norm": 35.99516677856445,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": 1.3092613220214844,
      "logits/rejected": 1.321373462677002,
      "logps/chosen": -314.7962646484375,
      "logps/ref_chosen": -275.91595458984375,
      "logps/ref_rejected": -264.79736328125,
      "logps/rejected": -350.6866455078125,
      "loss": 4.7825,
      "margin_dpo/margin_mean": 47.00896453857422,
      "margin_dpo/margin_std": 82.2657241821289,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -33.75083923339844,
      "KL/mean": -63.94988250732422,
      "KL/rejected_KL_mean": -94.14892578125,
      "KL/std": 71.10142517089844,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.005669101607054472,
      "fcm_dpo/delta": 0.0002967662294395268,
      "fcm_dpo/margin": 60.398094177246094,
      "fcm_dpo/q_t": 0.41952913999557495,
      "grad_norm": 23.352693557739258,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 1.172903060913086,
      "logits/rejected": 1.2363715171813965,
      "logps/chosen": -278.69287109375,
      "logps/ref_chosen": -244.9420166015625,
      "logps/ref_rejected": -235.61978149414062,
      "logps/rejected": -329.7687072753906,
      "loss": 4.5316,
      "margin_dpo/margin_mean": 60.398094177246094,
      "margin_dpo/margin_std": 86.25730895996094,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -39.005516052246094,
      "KL/mean": -64.75503540039062,
      "KL/rejected_KL_mean": -90.50456237792969,
      "KL/std": 70.41693115234375,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.0056970068253576756,
      "fcm_dpo/delta": -0.0046531930565834045,
      "fcm_dpo/margin": 51.4990348815918,
      "fcm_dpo/q_t": 0.43016308546066284,
      "grad_norm": 40.558528900146484,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 1.176418662071228,
      "logits/rejected": 1.1575579643249512,
      "logps/chosen": -328.3553771972656,
      "logps/ref_chosen": -289.3498229980469,
      "logps/ref_rejected": -243.70205688476562,
      "logps/rejected": -334.20660400390625,
      "loss": 4.6706,
      "margin_dpo/margin_mean": 51.4990348815918,
      "margin_dpo/margin_std": 79.15789794921875,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -37.14433670043945,
      "KL/mean": -70.23841857910156,
      "KL/rejected_KL_mean": -103.33248901367188,
      "KL/std": 73.222900390625,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.0056442320346832275,
      "fcm_dpo/delta": 0.0044305892661213875,
      "fcm_dpo/margin": 66.1881332397461,
      "fcm_dpo/q_t": 0.41161203384399414,
      "grad_norm": 27.89653968811035,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": 1.6276371479034424,
      "logits/rejected": 1.6381112337112427,
      "logps/chosen": -358.7978515625,
      "logps/ref_chosen": -321.65350341796875,
      "logps/ref_rejected": -282.73773193359375,
      "logps/rejected": -386.0702209472656,
      "loss": 4.4143,
      "margin_dpo/margin_mean": 66.18814086914062,
      "margin_dpo/margin_std": 81.6332778930664,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -35.79389953613281,
      "KL/mean": -66.21890258789062,
      "KL/rejected_KL_mean": -96.64389038085938,
      "KL/std": 74.63967895507812,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.005615277215838432,
      "fcm_dpo/delta": 0.0006926862988620996,
      "fcm_dpo/margin": 60.849998474121094,
      "fcm_dpo/q_t": 0.41957980394363403,
      "grad_norm": 34.66533279418945,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": 1.2122138738632202,
      "logits/rejected": 1.1651296615600586,
      "logps/chosen": -345.54437255859375,
      "logps/ref_chosen": -309.75048828125,
      "logps/ref_rejected": -267.41925048828125,
      "logps/rejected": -364.0631408691406,
      "loss": 4.5276,
      "margin_dpo/margin_mean": 60.85000228881836,
      "margin_dpo/margin_std": 84.82569885253906,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -40.26706314086914,
      "KL/mean": -60.995731353759766,
      "KL/rejected_KL_mean": -81.72441864013672,
      "KL/std": 71.74934387207031,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.005669465754181147,
      "fcm_dpo/delta": 0.007436351850628853,
      "fcm_dpo/margin": 41.45735168457031,
      "fcm_dpo/q_t": 0.4448689818382263,
      "grad_norm": 46.221500396728516,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 1.4722293615341187,
      "logits/rejected": 1.451279878616333,
      "logps/chosen": -351.26116943359375,
      "logps/ref_chosen": -310.9941101074219,
      "logps/ref_rejected": -279.94915771484375,
      "logps/rejected": -361.673583984375,
      "loss": 4.8963,
      "margin_dpo/margin_mean": 41.45735168457031,
      "margin_dpo/margin_std": 84.26883697509766,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -40.73126983642578,
      "KL/mean": -66.5917739868164,
      "KL/rejected_KL_mean": -92.45230102539062,
      "KL/std": 69.7247085571289,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.00574857834726572,
      "fcm_dpo/delta": 0.003375548869371414,
      "fcm_dpo/margin": 51.72104263305664,
      "fcm_dpo/q_t": 0.4301758408546448,
      "grad_norm": 40.411285400390625,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": 1.5146502256393433,
      "logits/rejected": 1.4434853792190552,
      "logps/chosen": -349.91473388671875,
      "logps/ref_chosen": -309.18341064453125,
      "logps/ref_rejected": -256.9963073730469,
      "logps/rejected": -349.4485778808594,
      "loss": 4.6821,
      "margin_dpo/margin_mean": 51.72104263305664,
      "margin_dpo/margin_std": 83.79369354248047,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -41.74312973022461,
      "KL/mean": -67.52073669433594,
      "KL/rejected_KL_mean": -93.29833984375,
      "KL/std": 72.34806060791016,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.0056892698630690575,
      "fcm_dpo/delta": -0.02217222936451435,
      "fcm_dpo/margin": 51.555206298828125,
      "fcm_dpo/q_t": 0.4309225082397461,
      "grad_norm": 29.26421356201172,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": 1.3720002174377441,
      "logits/rejected": 1.4351372718811035,
      "logps/chosen": -324.018310546875,
      "logps/ref_chosen": -282.275146484375,
      "logps/ref_rejected": -255.17941284179688,
      "logps/rejected": -348.4777526855469,
      "loss": 4.7096,
      "margin_dpo/margin_mean": 51.555206298828125,
      "margin_dpo/margin_std": 84.42403411865234,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -38.90581512451172,
      "KL/mean": -62.6286506652832,
      "KL/rejected_KL_mean": -86.35147094726562,
      "KL/std": 67.94470977783203,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.005712832789868116,
      "fcm_dpo/delta": 0.03435080498456955,
      "fcm_dpo/margin": 47.44567108154297,
      "fcm_dpo/q_t": 0.43644294142723083,
      "grad_norm": 30.020174026489258,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": 1.4383010864257812,
      "logits/rejected": 1.4928736686706543,
      "logps/chosen": -331.78302001953125,
      "logps/ref_chosen": -292.8772277832031,
      "logps/ref_rejected": -266.78509521484375,
      "logps/rejected": -353.1365661621094,
      "loss": 4.7435,
      "margin_dpo/margin_mean": 47.44567108154297,
      "margin_dpo/margin_std": 80.14208984375,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -37.284393310546875,
      "KL/mean": -61.246639251708984,
      "KL/rejected_KL_mean": -85.20887756347656,
      "KL/std": 71.65672302246094,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.005775141529738903,
      "fcm_dpo/delta": -0.0016941586509346962,
      "fcm_dpo/margin": 47.92448425292969,
      "fcm_dpo/q_t": 0.43488824367523193,
      "grad_norm": 40.82341384887695,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": 1.3757528066635132,
      "logits/rejected": 1.4355530738830566,
      "logps/chosen": -310.1781005859375,
      "logps/ref_chosen": -272.8937072753906,
      "logps/ref_rejected": -293.058349609375,
      "logps/rejected": -378.2672119140625,
      "loss": 4.7343,
      "margin_dpo/margin_mean": 47.92448425292969,
      "margin_dpo/margin_std": 79.6957778930664,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -37.807315826416016,
      "KL/mean": -63.437442779541016,
      "KL/rejected_KL_mean": -89.06757354736328,
      "KL/std": 68.95721435546875,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.005743257701396942,
      "fcm_dpo/delta": -0.009766137227416039,
      "fcm_dpo/margin": 51.2602653503418,
      "fcm_dpo/q_t": 0.43055325746536255,
      "grad_norm": 40.31827926635742,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": 1.3602495193481445,
      "logits/rejected": 1.367412805557251,
      "logps/chosen": -296.94891357421875,
      "logps/ref_chosen": -259.1416015625,
      "logps/ref_rejected": -229.64996337890625,
      "logps/rejected": -318.7175598144531,
      "loss": 4.7043,
      "margin_dpo/margin_mean": 51.2602653503418,
      "margin_dpo/margin_std": 86.9591064453125,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -44.88898468017578,
      "KL/mean": -69.46295166015625,
      "KL/rejected_KL_mean": -94.03691101074219,
      "KL/std": 72.96025085449219,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.005757888313382864,
      "fcm_dpo/delta": 0.010965266264975071,
      "fcm_dpo/margin": 49.14793014526367,
      "fcm_dpo/q_t": 0.43409690260887146,
      "grad_norm": 64.16248321533203,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 1.3960915803909302,
      "logits/rejected": 1.323775053024292,
      "logps/chosen": -341.2269287109375,
      "logps/ref_chosen": -296.33795166015625,
      "logps/ref_rejected": -283.7359924316406,
      "logps/rejected": -377.7728576660156,
      "loss": 4.7263,
      "margin_dpo/margin_mean": 49.14793014526367,
      "margin_dpo/margin_std": 82.04457092285156,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -40.70269775390625,
      "KL/mean": -68.68380737304688,
      "KL/rejected_KL_mean": -96.66490173339844,
      "KL/std": 71.64056396484375,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.005815763026475906,
      "fcm_dpo/delta": 0.01836465299129486,
      "fcm_dpo/margin": 55.96220016479492,
      "fcm_dpo/q_t": 0.4236753284931183,
      "grad_norm": 61.32617950439453,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 1.2255052328109741,
      "logits/rejected": 1.2598758935928345,
      "logps/chosen": -289.6759033203125,
      "logps/ref_chosen": -248.97320556640625,
      "logps/ref_rejected": -259.5140075683594,
      "logps/rejected": -356.17889404296875,
      "loss": 4.568,
      "margin_dpo/margin_mean": 55.96220397949219,
      "margin_dpo/margin_std": 81.33918762207031,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -28.753982543945312,
      "KL/mean": -63.350120544433594,
      "KL/rejected_KL_mean": -97.94627380371094,
      "KL/std": 71.96202087402344,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.005783876869827509,
      "fcm_dpo/delta": -0.03661331906914711,
      "fcm_dpo/margin": 69.19227600097656,
      "fcm_dpo/q_t": 0.4055514633655548,
      "grad_norm": 33.33644104003906,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": 1.3843127489089966,
      "logits/rejected": 1.5462632179260254,
      "logps/chosen": -309.42626953125,
      "logps/ref_chosen": -280.67230224609375,
      "logps/ref_rejected": -272.4359130859375,
      "logps/rejected": -370.3821716308594,
      "loss": 4.3445,
      "margin_dpo/margin_mean": 69.19227600097656,
      "margin_dpo/margin_std": 82.95043182373047,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -35.152679443359375,
      "KL/mean": -61.34556198120117,
      "KL/rejected_KL_mean": -87.5384521484375,
      "KL/std": 69.7143783569336,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.005756077356636524,
      "fcm_dpo/delta": 0.02370220422744751,
      "fcm_dpo/margin": 52.38575744628906,
      "fcm_dpo/q_t": 0.4291101396083832,
      "grad_norm": 30.689350128173828,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": 1.1176592111587524,
      "logits/rejected": 1.1696782112121582,
      "logps/chosen": -302.8111572265625,
      "logps/ref_chosen": -267.658447265625,
      "logps/ref_rejected": -262.9810791015625,
      "logps/rejected": -350.51953125,
      "loss": 4.6526,
      "margin_dpo/margin_mean": 52.38575744628906,
      "margin_dpo/margin_std": 80.9411849975586,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -42.52230453491211,
      "KL/mean": -61.41611099243164,
      "KL/rejected_KL_mean": -80.30989837646484,
      "KL/std": 68.72921752929688,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.005827743094414473,
      "fcm_dpo/delta": 0.015056919306516647,
      "fcm_dpo/margin": 37.787601470947266,
      "fcm_dpo/q_t": 0.4483274221420288,
      "grad_norm": 47.5373420715332,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": 1.4168736934661865,
      "logits/rejected": 1.600912094116211,
      "logps/chosen": -308.76885986328125,
      "logps/ref_chosen": -266.2465515136719,
      "logps/ref_rejected": -281.1450500488281,
      "logps/rejected": -361.4549255371094,
      "loss": 4.9705,
      "margin_dpo/margin_mean": 37.78759765625,
      "margin_dpo/margin_std": 86.831787109375,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -37.53956985473633,
      "KL/mean": -62.57978057861328,
      "KL/rejected_KL_mean": -87.6199951171875,
      "KL/std": 69.87211608886719,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.005918873939663172,
      "fcm_dpo/delta": 0.008942903019487858,
      "fcm_dpo/margin": 50.080421447753906,
      "fcm_dpo/q_t": 0.43059298396110535,
      "grad_norm": 41.67280197143555,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": 1.4343469142913818,
      "logits/rejected": 1.430830717086792,
      "logps/chosen": -311.84539794921875,
      "logps/ref_chosen": -274.30584716796875,
      "logps/ref_rejected": -242.94247436523438,
      "logps/rejected": -330.5624694824219,
      "loss": 4.6875,
      "margin_dpo/margin_mean": 50.080421447753906,
      "margin_dpo/margin_std": 81.83329772949219,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -31.415958404541016,
      "KL/mean": -62.22372055053711,
      "KL/rejected_KL_mean": -93.03147888183594,
      "KL/std": 74.65743255615234,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.005736582912504673,
      "fcm_dpo/delta": -0.04927733540534973,
      "fcm_dpo/margin": 61.615516662597656,
      "fcm_dpo/q_t": 0.41625073552131653,
      "grad_norm": 30.383468627929688,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": 1.393118143081665,
      "logits/rejected": 1.4672538042068481,
      "logps/chosen": -340.6824035644531,
      "logps/ref_chosen": -309.2664489746094,
      "logps/ref_rejected": -298.4182434082031,
      "logps/rejected": -391.4497375488281,
      "loss": 4.5061,
      "margin_dpo/margin_mean": 61.615516662597656,
      "margin_dpo/margin_std": 83.25586700439453,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -32.46100616455078,
      "KL/mean": -61.694480895996094,
      "KL/rejected_KL_mean": -90.9279556274414,
      "KL/std": 71.06009674072266,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.005657571367919445,
      "fcm_dpo/delta": 0.007430531550198793,
      "fcm_dpo/margin": 58.466941833496094,
      "fcm_dpo/q_t": 0.42239296436309814,
      "grad_norm": 30.680625915527344,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 1.396806240081787,
      "logits/rejected": 1.4459258317947388,
      "logps/chosen": -296.4819030761719,
      "logps/ref_chosen": -264.0209045410156,
      "logps/ref_rejected": -264.3447570800781,
      "logps/rejected": -355.272705078125,
      "loss": 4.5735,
      "margin_dpo/margin_mean": 58.466941833496094,
      "margin_dpo/margin_std": 84.47205352783203,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -45.005165100097656,
      "KL/mean": -66.85578918457031,
      "KL/rejected_KL_mean": -88.70642852783203,
      "KL/std": 69.91039276123047,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.00565434992313385,
      "fcm_dpo/delta": 3.866571933031082e-05,
      "fcm_dpo/margin": 43.701263427734375,
      "fcm_dpo/q_t": 0.44198447465896606,
      "grad_norm": 53.31806182861328,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": 1.352836012840271,
      "logits/rejected": 1.4095062017440796,
      "logps/chosen": -308.2240295410156,
      "logps/ref_chosen": -263.2188415527344,
      "logps/ref_rejected": -236.08546447753906,
      "logps/rejected": -324.7918701171875,
      "loss": 4.8772,
      "margin_dpo/margin_mean": 43.701263427734375,
      "margin_dpo/margin_std": 87.63800811767578,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -33.140480041503906,
      "KL/mean": -62.86387634277344,
      "KL/rejected_KL_mean": -92.58728790283203,
      "KL/std": 68.41466522216797,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.0056841736659407616,
      "fcm_dpo/delta": 0.005020654760301113,
      "fcm_dpo/margin": 59.44679260253906,
      "fcm_dpo/q_t": 0.4199007451534271,
      "grad_norm": 48.97894287109375,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": 1.32963228225708,
      "logits/rejected": 1.3083891868591309,
      "logps/chosen": -306.93841552734375,
      "logps/ref_chosen": -273.79791259765625,
      "logps/ref_rejected": -242.4090118408203,
      "logps/rejected": -334.99627685546875,
      "loss": 4.5354,
      "margin_dpo/margin_mean": 59.44679260253906,
      "margin_dpo/margin_std": 82.35969543457031,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -36.285400390625,
      "KL/mean": -64.5611572265625,
      "KL/rejected_KL_mean": -92.83692169189453,
      "KL/std": 69.67570495605469,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.005722576752305031,
      "fcm_dpo/delta": -0.0016763133462518454,
      "fcm_dpo/margin": 56.55152893066406,
      "fcm_dpo/q_t": 0.4238140881061554,
      "grad_norm": 36.144710540771484,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 1.2004024982452393,
      "logits/rejected": 1.2577799558639526,
      "logps/chosen": -311.6005554199219,
      "logps/ref_chosen": -275.3151550292969,
      "logps/ref_rejected": -278.1514587402344,
      "logps/rejected": -370.9884338378906,
      "loss": 4.6285,
      "margin_dpo/margin_mean": 56.55152893066406,
      "margin_dpo/margin_std": 90.65345001220703,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -43.44788360595703,
      "KL/mean": -70.53215026855469,
      "KL/rejected_KL_mean": -97.61642456054688,
      "KL/std": 72.71483612060547,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.0057303570210933685,
      "fcm_dpo/delta": -0.0041380757465958595,
      "fcm_dpo/margin": 54.16854476928711,
      "fcm_dpo/q_t": 0.4268883168697357,
      "grad_norm": 26.626068115234375,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.2742648124694824,
      "logits/rejected": 1.2590354681015015,
      "logps/chosen": -318.647216796875,
      "logps/ref_chosen": -275.1993408203125,
      "logps/ref_rejected": -261.32122802734375,
      "logps/rejected": -358.9376220703125,
      "loss": 4.5966,
      "margin_dpo/margin_mean": 54.16854476928711,
      "margin_dpo/margin_std": 75.0491943359375,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -37.037078857421875,
      "KL/mean": -71.72711181640625,
      "KL/rejected_KL_mean": -106.41716766357422,
      "KL/std": 72.38037109375,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.005560027901083231,
      "fcm_dpo/delta": -0.029484987258911133,
      "fcm_dpo/margin": 69.38008880615234,
      "fcm_dpo/q_t": 0.4086316227912903,
      "grad_norm": 44.154415130615234,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": 1.3461742401123047,
      "logits/rejected": 1.358787178993225,
      "logps/chosen": -336.8648681640625,
      "logps/ref_chosen": -299.8277587890625,
      "logps/ref_rejected": -251.24798583984375,
      "logps/rejected": -357.6651611328125,
      "loss": 4.3433,
      "margin_dpo/margin_mean": 69.38008880615234,
      "margin_dpo/margin_std": 76.5917739868164,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -33.104591369628906,
      "KL/mean": -58.68251419067383,
      "KL/rejected_KL_mean": -84.26043701171875,
      "KL/std": 69.8023681640625,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.005620099604129791,
      "fcm_dpo/delta": 0.02052416279911995,
      "fcm_dpo/margin": 51.15583038330078,
      "fcm_dpo/q_t": 0.43219760060310364,
      "grad_norm": 42.19123840332031,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": 1.4071629047393799,
      "logits/rejected": 1.4042648077011108,
      "logps/chosen": -313.5657043457031,
      "logps/ref_chosen": -280.4610900878906,
      "logps/ref_rejected": -228.34823608398438,
      "logps/rejected": -312.608642578125,
      "loss": 4.7083,
      "margin_dpo/margin_mean": 51.15583038330078,
      "margin_dpo/margin_std": 85.43704223632812,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -30.38258171081543,
      "KL/mean": -55.82136917114258,
      "KL/rejected_KL_mean": -81.2601547241211,
      "KL/std": 68.99714660644531,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.005732883233577013,
      "fcm_dpo/delta": 0.021570980548858643,
      "fcm_dpo/margin": 50.87757110595703,
      "fcm_dpo/q_t": 0.43169111013412476,
      "grad_norm": 55.2331657409668,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": 1.3367154598236084,
      "logits/rejected": 1.4419281482696533,
      "logps/chosen": -349.982666015625,
      "logps/ref_chosen": -319.60009765625,
      "logps/ref_rejected": -316.42822265625,
      "logps/rejected": -397.6883544921875,
      "loss": 4.6709,
      "margin_dpo/margin_mean": 50.87757110595703,
      "margin_dpo/margin_std": 77.82229614257812,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -37.89224624633789,
      "KL/mean": -64.57542419433594,
      "KL/rejected_KL_mean": -91.25860595703125,
      "KL/std": 68.56454467773438,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.005701184738427401,
      "fcm_dpo/delta": -0.016113528981804848,
      "fcm_dpo/margin": 53.366329193115234,
      "fcm_dpo/q_t": 0.4281626045703888,
      "grad_norm": 34.436256408691406,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": 1.1115540266036987,
      "logits/rejected": 1.23086416721344,
      "logps/chosen": -298.3100891113281,
      "logps/ref_chosen": -260.4178466796875,
      "logps/ref_rejected": -270.04498291015625,
      "logps/rejected": -361.3036193847656,
      "loss": 4.6287,
      "margin_dpo/margin_mean": 53.366329193115234,
      "margin_dpo/margin_std": 77.86839294433594,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -39.975929260253906,
      "KL/mean": -66.40296173095703,
      "KL/rejected_KL_mean": -92.83000183105469,
      "KL/std": 70.88655090332031,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.005648612044751644,
      "fcm_dpo/delta": -0.01798979938030243,
      "fcm_dpo/margin": 52.85405731201172,
      "fcm_dpo/q_t": 0.42931780219078064,
      "grad_norm": 59.245426177978516,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": 1.5093213319778442,
      "logits/rejected": 1.5127885341644287,
      "logps/chosen": -320.678955078125,
      "logps/ref_chosen": -280.7030029296875,
      "logps/ref_rejected": -250.76385498046875,
      "logps/rejected": -343.5938415527344,
      "loss": 4.6771,
      "margin_dpo/margin_mean": 52.85405731201172,
      "margin_dpo/margin_std": 84.82481384277344,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -34.25897216796875,
      "KL/mean": -59.6822509765625,
      "KL/rejected_KL_mean": -85.10551452636719,
      "KL/std": 71.90367126464844,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.005564483813941479,
      "fcm_dpo/delta": -0.012457918375730515,
      "fcm_dpo/margin": 50.84654998779297,
      "fcm_dpo/q_t": 0.43293097615242004,
      "grad_norm": 27.78412628173828,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": 1.2895883321762085,
      "logits/rejected": 1.406543493270874,
      "logps/chosen": -335.1454162597656,
      "logps/ref_chosen": -300.8864440917969,
      "logps/ref_rejected": -288.5682067871094,
      "logps/rejected": -373.6737365722656,
      "loss": 4.7195,
      "margin_dpo/margin_mean": 50.84654998779297,
      "margin_dpo/margin_std": 81.06475830078125,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -49.08461380004883,
      "KL/mean": -69.30685424804688,
      "KL/rejected_KL_mean": -89.52908325195312,
      "KL/std": 72.63346862792969,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.005588540807366371,
      "fcm_dpo/delta": 0.010477852076292038,
      "fcm_dpo/margin": 40.44446563720703,
      "fcm_dpo/q_t": 0.44711777567863464,
      "grad_norm": 47.24563217163086,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": 1.2969677448272705,
      "logits/rejected": 1.3580262660980225,
      "logps/chosen": -342.3869934082031,
      "logps/ref_chosen": -293.3023681640625,
      "logps/ref_rejected": -270.0457763671875,
      "logps/rejected": -359.5748596191406,
      "loss": 4.9574,
      "margin_dpo/margin_mean": 40.4444694519043,
      "margin_dpo/margin_std": 92.06279754638672,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -43.60993957519531,
      "KL/mean": -67.60672760009766,
      "KL/rejected_KL_mean": -91.603515625,
      "KL/std": 72.94109344482422,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.0056368703953921795,
      "fcm_dpo/delta": 0.01475178450345993,
      "fcm_dpo/margin": 47.993587493896484,
      "fcm_dpo/q_t": 0.43614646792411804,
      "grad_norm": 59.89192199707031,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": 1.1046593189239502,
      "logits/rejected": 1.1742043495178223,
      "logps/chosen": -327.0091857910156,
      "logps/ref_chosen": -283.39923095703125,
      "logps/ref_rejected": -264.5092468261719,
      "logps/rejected": -356.1127624511719,
      "loss": 4.7389,
      "margin_dpo/margin_mean": 47.993587493896484,
      "margin_dpo/margin_std": 77.74752807617188,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -37.8900146484375,
      "KL/mean": -66.0036849975586,
      "KL/rejected_KL_mean": -94.11735534667969,
      "KL/std": 68.33687591552734,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.005634288303554058,
      "fcm_dpo/delta": -0.007323849014937878,
      "fcm_dpo/margin": 56.227325439453125,
      "fcm_dpo/q_t": 0.4250444769859314,
      "grad_norm": 34.885162353515625,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 1.1642831563949585,
      "logits/rejected": 1.2289671897888184,
      "logps/chosen": -292.35784912109375,
      "logps/ref_chosen": -254.4678192138672,
      "logps/ref_rejected": -252.00326538085938,
      "logps/rejected": -346.12060546875,
      "loss": 4.6016,
      "margin_dpo/margin_mean": 56.227325439453125,
      "margin_dpo/margin_std": 80.86348724365234,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_KL/chosen_KL_mean": -41.54841613769531,
      "eval_KL/mean": -67.07406616210938,
      "eval_KL/rejected_KL_mean": -92.59970092773438,
      "eval_KL/std": 70.14669036865234,
      "eval_fcm_dpo/beta": 0.0056360261514782906,
      "eval_logits/chosen": 1.5636427402496338,
      "eval_logits/rejected": 1.6450908184051514,
      "eval_logps/chosen": -321.9651184082031,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -357.2452392578125,
      "eval_loss": 0.58974689245224,
      "eval_margin_dpo/margin_mean": 51.0512809753418,
      "eval_margin_dpo/margin_std": 81.34390258789062,
      "eval_runtime": 88.541,
      "eval_samples_per_second": 22.588,
      "eval_steps_per_second": 2.824,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -34.37708282470703,
      "KL/mean": -67.0552749633789,
      "KL/rejected_KL_mean": -99.73347473144531,
      "KL/std": 73.49190521240234,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.005607966333627701,
      "fcm_dpo/delta": -0.003262819955125451,
      "fcm_dpo/margin": 65.35639953613281,
      "fcm_dpo/q_t": 0.41333135962486267,
      "grad_norm": 32.96726989746094,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 1.34162437915802,
      "logits/rejected": 1.3759033679962158,
      "logps/chosen": -344.97662353515625,
      "logps/ref_chosen": -310.5995788574219,
      "logps/ref_rejected": -292.7754821777344,
      "logps/rejected": -392.5089416503906,
      "loss": 4.4224,
      "margin_dpo/margin_mean": 65.35639953613281,
      "margin_dpo/margin_std": 81.28590393066406,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -34.880958557128906,
      "KL/mean": -62.652347564697266,
      "KL/rejected_KL_mean": -90.42375183105469,
      "KL/std": 71.70995330810547,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.0056101661175489426,
      "fcm_dpo/delta": -0.002124865772202611,
      "fcm_dpo/margin": 55.542781829833984,
      "fcm_dpo/q_t": 0.4263724982738495,
      "grad_norm": 44.68589782714844,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": 1.3792533874511719,
      "logits/rejected": 1.3068594932556152,
      "logps/chosen": -334.7323303222656,
      "logps/ref_chosen": -299.8513488769531,
      "logps/ref_rejected": -278.5664367675781,
      "logps/rejected": -368.99017333984375,
      "loss": 4.6195,
      "margin_dpo/margin_mean": 55.542781829833984,
      "margin_dpo/margin_std": 83.6781997680664,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -30.88512420654297,
      "KL/mean": -68.1521987915039,
      "KL/rejected_KL_mean": -105.41928100585938,
      "KL/std": 73.96147155761719,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.005608746316283941,
      "fcm_dpo/delta": -0.019092462956905365,
      "fcm_dpo/margin": 74.53414916992188,
      "fcm_dpo/q_t": 0.40090423822402954,
      "grad_norm": 33.62675857543945,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": 1.3099597692489624,
      "logits/rejected": 1.3480051755905151,
      "logps/chosen": -333.3167724609375,
      "logps/ref_chosen": -302.4316101074219,
      "logps/ref_rejected": -270.951171875,
      "logps/rejected": -376.37042236328125,
      "loss": 4.22,
      "margin_dpo/margin_mean": 74.53414916992188,
      "margin_dpo/margin_std": 73.57200622558594,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -44.7662353515625,
      "KL/mean": -70.37663269042969,
      "KL/rejected_KL_mean": -95.9870376586914,
      "KL/std": 74.23789978027344,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.005546754691749811,
      "fcm_dpo/delta": 0.01605689525604248,
      "fcm_dpo/margin": 51.22079849243164,
      "fcm_dpo/q_t": 0.4331476092338562,
      "grad_norm": 41.1744270324707,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": 1.346960186958313,
      "logits/rejected": 1.3870068788528442,
      "logps/chosen": -334.6693420410156,
      "logps/ref_chosen": -289.9031066894531,
      "logps/ref_rejected": -252.84918212890625,
      "logps/rejected": -348.8362121582031,
      "loss": 4.7236,
      "margin_dpo/margin_mean": 51.220802307128906,
      "margin_dpo/margin_std": 85.0601577758789,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -35.77894592285156,
      "KL/mean": -64.92236328125,
      "KL/rejected_KL_mean": -94.06578063964844,
      "KL/std": 72.94497680664062,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.005575691815465689,
      "fcm_dpo/delta": -0.004206720739603043,
      "fcm_dpo/margin": 58.28684997558594,
      "fcm_dpo/q_t": 0.423540860414505,
      "grad_norm": 25.92606544494629,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": 1.4113208055496216,
      "logits/rejected": 1.599323034286499,
      "logps/chosen": -283.9530029296875,
      "logps/ref_chosen": -248.17404174804688,
      "logps/ref_rejected": -252.23764038085938,
      "logps/rejected": -346.30340576171875,
      "loss": 4.5551,
      "margin_dpo/margin_mean": 58.286842346191406,
      "margin_dpo/margin_std": 79.76528930664062,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -43.40061950683594,
      "KL/mean": -66.99781036376953,
      "KL/rejected_KL_mean": -90.59500122070312,
      "KL/std": 72.16825103759766,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.00563565781340003,
      "fcm_dpo/delta": 0.02900759130716324,
      "fcm_dpo/margin": 47.19438171386719,
      "fcm_dpo/q_t": 0.4374856948852539,
      "grad_norm": 34.088172912597656,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": 1.3219295740127563,
      "logits/rejected": 1.3950344324111938,
      "logps/chosen": -319.81671142578125,
      "logps/ref_chosen": -276.4161376953125,
      "logps/ref_rejected": -264.0082702636719,
      "logps/rejected": -354.603271484375,
      "loss": 4.8022,
      "margin_dpo/margin_mean": 47.19438171386719,
      "margin_dpo/margin_std": 87.58171844482422,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -49.26393508911133,
      "KL/mean": -77.42133331298828,
      "KL/rejected_KL_mean": -105.5787353515625,
      "KL/std": 74.86487579345703,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.005747040268033743,
      "fcm_dpo/delta": 0.021842535585165024,
      "fcm_dpo/margin": 56.314796447753906,
      "fcm_dpo/q_t": 0.42419612407684326,
      "grad_norm": 48.5234375,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": 1.3789080381393433,
      "logits/rejected": 1.4008229970932007,
      "logps/chosen": -306.7012939453125,
      "logps/ref_chosen": -257.4373474121094,
      "logps/ref_rejected": -219.66390991210938,
      "logps/rejected": -325.2426452636719,
      "loss": 4.5933,
      "margin_dpo/margin_mean": 56.314788818359375,
      "margin_dpo/margin_std": 85.56969451904297,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -36.52445983886719,
      "KL/mean": -65.276123046875,
      "KL/rejected_KL_mean": -94.02780151367188,
      "KL/std": 71.32727813720703,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.005806931760162115,
      "fcm_dpo/delta": -0.016825567930936813,
      "fcm_dpo/margin": 57.503334045410156,
      "fcm_dpo/q_t": 0.4210928976535797,
      "grad_norm": 37.30312728881836,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": 1.390464186668396,
      "logits/rejected": 1.5240916013717651,
      "logps/chosen": -325.1778259277344,
      "logps/ref_chosen": -288.6533508300781,
      "logps/ref_rejected": -291.54107666015625,
      "logps/rejected": -385.5688781738281,
      "loss": 4.5495,
      "margin_dpo/margin_mean": 57.503334045410156,
      "margin_dpo/margin_std": 79.41692352294922,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -40.43635940551758,
      "KL/mean": -70.10758209228516,
      "KL/rejected_KL_mean": -99.77880859375,
      "KL/std": 74.0046615600586,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.005669078323990107,
      "fcm_dpo/delta": -0.015406528487801552,
      "fcm_dpo/margin": 59.342437744140625,
      "fcm_dpo/q_t": 0.42115840315818787,
      "grad_norm": 33.51537322998047,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": 1.418125867843628,
      "logits/rejected": 1.5287835597991943,
      "logps/chosen": -318.1737365722656,
      "logps/ref_chosen": -277.7373962402344,
      "logps/ref_rejected": -272.988525390625,
      "logps/rejected": -372.767333984375,
      "loss": 4.5749,
      "margin_dpo/margin_mean": 59.34243392944336,
      "margin_dpo/margin_std": 89.5152816772461,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -40.43669891357422,
      "KL/mean": -66.39927673339844,
      "KL/rejected_KL_mean": -92.36186218261719,
      "KL/std": 73.42118835449219,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.005742058623582125,
      "fcm_dpo/delta": 0.019838577136397362,
      "fcm_dpo/margin": 51.925167083740234,
      "fcm_dpo/q_t": 0.43019813299179077,
      "grad_norm": 64.85350799560547,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 1.5594767332077026,
      "logits/rejected": 1.539419412612915,
      "logps/chosen": -314.65777587890625,
      "logps/ref_chosen": -274.2210998535156,
      "logps/ref_rejected": -264.33465576171875,
      "logps/rejected": -356.696533203125,
      "loss": 4.6908,
      "margin_dpo/margin_mean": 51.925167083740234,
      "margin_dpo/margin_std": 86.36168670654297,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -34.97002410888672,
      "KL/mean": -66.30883026123047,
      "KL/rejected_KL_mean": -97.64764404296875,
      "KL/std": 73.95851135253906,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.005672582425177097,
      "fcm_dpo/delta": -0.029585428535938263,
      "fcm_dpo/margin": 62.67761993408203,
      "fcm_dpo/q_t": 0.4158910810947418,
      "grad_norm": 31.859609603881836,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 1.4028679132461548,
      "logits/rejected": 1.4709159135818481,
      "logps/chosen": -307.8380126953125,
      "logps/ref_chosen": -272.86798095703125,
      "logps/ref_rejected": -250.35772705078125,
      "logps/rejected": -348.00537109375,
      "loss": 4.4736,
      "margin_dpo/margin_mean": 62.6776237487793,
      "margin_dpo/margin_std": 78.85665893554688,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -49.12342071533203,
      "KL/mean": -74.26603698730469,
      "KL/rejected_KL_mean": -99.40866088867188,
      "KL/std": 73.6407241821289,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.005613192915916443,
      "fcm_dpo/delta": 0.011941947974264622,
      "fcm_dpo/margin": 50.28524398803711,
      "fcm_dpo/q_t": 0.43374866247177124,
      "grad_norm": 50.395530700683594,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": 1.1836318969726562,
      "logits/rejected": 1.286041021347046,
      "logps/chosen": -317.0426330566406,
      "logps/ref_chosen": -267.919189453125,
      "logps/ref_rejected": -271.73162841796875,
      "logps/rejected": -371.1402893066406,
      "loss": 4.7492,
      "margin_dpo/margin_mean": 50.28525161743164,
      "margin_dpo/margin_std": 88.66204833984375,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -36.451839447021484,
      "KL/mean": -60.05599594116211,
      "KL/rejected_KL_mean": -83.66015625,
      "KL/std": 69.52487182617188,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.005707796663045883,
      "fcm_dpo/delta": 0.023781919851899147,
      "fcm_dpo/margin": 47.20831298828125,
      "fcm_dpo/q_t": 0.43670517206192017,
      "grad_norm": 36.19025802612305,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 1.2643343210220337,
      "logits/rejected": 1.3224858045578003,
      "logps/chosen": -301.4098815917969,
      "logps/ref_chosen": -264.95806884765625,
      "logps/ref_rejected": -267.40093994140625,
      "logps/rejected": -351.06109619140625,
      "loss": 4.7445,
      "margin_dpo/margin_mean": 47.20831298828125,
      "margin_dpo/margin_std": 79.60798645019531,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -46.366737365722656,
      "KL/mean": -74.14826965332031,
      "KL/rejected_KL_mean": -101.9298095703125,
      "KL/std": 74.98602294921875,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.005864008329808712,
      "fcm_dpo/delta": 0.010818198323249817,
      "fcm_dpo/margin": 55.56306838989258,
      "fcm_dpo/q_t": 0.42432108521461487,
      "grad_norm": 28.803497314453125,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": 1.3398324251174927,
      "logits/rejected": 1.400763750076294,
      "logps/chosen": -336.37249755859375,
      "logps/ref_chosen": -290.00579833984375,
      "logps/ref_rejected": -266.8810119628906,
      "logps/rejected": -368.8108215332031,
      "loss": 4.6375,
      "margin_dpo/margin_mean": 55.56306838989258,
      "margin_dpo/margin_std": 90.18624877929688,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -34.370086669921875,
      "KL/mean": -67.09663391113281,
      "KL/rejected_KL_mean": -99.82318878173828,
      "KL/std": 72.9776611328125,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.005797143559902906,
      "fcm_dpo/delta": -0.023910801857709885,
      "fcm_dpo/margin": 65.4531021118164,
      "fcm_dpo/q_t": 0.41106075048446655,
      "grad_norm": 27.662456512451172,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": 1.427484393119812,
      "logits/rejected": 1.5233029127120972,
      "logps/chosen": -324.18109130859375,
      "logps/ref_chosen": -289.81103515625,
      "logps/ref_rejected": -278.2758483886719,
      "logps/rejected": -378.09906005859375,
      "loss": 4.4204,
      "margin_dpo/margin_mean": 65.4531021118164,
      "margin_dpo/margin_std": 85.86985778808594,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -35.22258758544922,
      "KL/mean": -66.39048767089844,
      "KL/rejected_KL_mean": -97.55840301513672,
      "KL/std": 72.54383087158203,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.005628857761621475,
      "fcm_dpo/delta": -0.012326779775321484,
      "fcm_dpo/margin": 62.33583450317383,
      "fcm_dpo/q_t": 0.4172388017177582,
      "grad_norm": 32.515567779541016,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": 1.443591594696045,
      "logits/rejected": 1.4231038093566895,
      "logps/chosen": -339.87139892578125,
      "logps/ref_chosen": -304.6488037109375,
      "logps/ref_rejected": -264.2703857421875,
      "logps/rejected": -361.82879638671875,
      "loss": 4.5026,
      "margin_dpo/margin_mean": 62.33583068847656,
      "margin_dpo/margin_std": 84.13420104980469,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -38.17135238647461,
      "KL/mean": -65.28013610839844,
      "KL/rejected_KL_mean": -92.388916015625,
      "KL/std": 71.19725036621094,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.005636176094412804,
      "fcm_dpo/delta": -0.023509033024311066,
      "fcm_dpo/margin": 54.21757125854492,
      "fcm_dpo/q_t": 0.4280126690864563,
      "grad_norm": 51.79258728027344,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": 1.4839136600494385,
      "logits/rejected": 1.5833758115768433,
      "logps/chosen": -340.0602722167969,
      "logps/ref_chosen": -301.8889465332031,
      "logps/ref_rejected": -275.0347595214844,
      "logps/rejected": -367.4236755371094,
      "loss": 4.67,
      "margin_dpo/margin_mean": 54.21757125854492,
      "margin_dpo/margin_std": 86.58592224121094,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -47.7001838684082,
      "KL/mean": -71.09051513671875,
      "KL/rejected_KL_mean": -94.48085021972656,
      "KL/std": 72.64429473876953,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.005531955044716597,
      "fcm_dpo/delta": -0.01827726513147354,
      "fcm_dpo/margin": 46.780662536621094,
      "fcm_dpo/q_t": 0.43858498334884644,
      "grad_norm": 43.77811050415039,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": 1.0857739448547363,
      "logits/rejected": 1.099918007850647,
      "logps/chosen": -316.9475402832031,
      "logps/ref_chosen": -269.2473449707031,
      "logps/ref_rejected": -275.5192565917969,
      "logps/rejected": -370.0000915527344,
      "loss": 4.7885,
      "margin_dpo/margin_mean": 46.780662536621094,
      "margin_dpo/margin_std": 77.29371643066406,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -47.12254333496094,
      "KL/mean": -78.98845672607422,
      "KL/rejected_KL_mean": -110.85435485839844,
      "KL/std": 77.59358215332031,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.0054054479114711285,
      "fcm_dpo/delta": -0.02243437059223652,
      "fcm_dpo/margin": 63.7318229675293,
      "fcm_dpo/q_t": 0.4188068211078644,
      "grad_norm": 25.274011611938477,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": 1.4264103174209595,
      "logits/rejected": 1.4069331884384155,
      "logps/chosen": -338.282958984375,
      "logps/ref_chosen": -291.1604309082031,
      "logps/ref_rejected": -245.18325805664062,
      "logps/rejected": -356.03759765625,
      "loss": 4.5511,
      "margin_dpo/margin_mean": 63.7318229675293,
      "margin_dpo/margin_std": 93.59545135498047,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -44.63287353515625,
      "KL/mean": -74.6416244506836,
      "KL/rejected_KL_mean": -104.65037536621094,
      "KL/std": 75.04874420166016,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.0054064164869487286,
      "fcm_dpo/delta": 0.011926915496587753,
      "fcm_dpo/margin": 60.017494201660156,
      "fcm_dpo/q_t": 0.4232439398765564,
      "grad_norm": 43.212554931640625,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 1.2428594827651978,
      "logits/rejected": 1.2047325372695923,
      "logps/chosen": -303.1026916503906,
      "logps/ref_chosen": -258.4698181152344,
      "logps/ref_rejected": -242.916015625,
      "logps/rejected": -347.56640625,
      "loss": 4.5694,
      "margin_dpo/margin_mean": 60.017486572265625,
      "margin_dpo/margin_std": 86.2717056274414,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -30.887887954711914,
      "KL/mean": -63.9599494934082,
      "KL/rejected_KL_mean": -97.03201293945312,
      "KL/std": 71.0981674194336,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.00541547778993845,
      "fcm_dpo/delta": 0.0007146215066313744,
      "fcm_dpo/margin": 66.1441421508789,
      "fcm_dpo/q_t": 0.4150181710720062,
      "grad_norm": 36.27775573730469,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.3024277687072754,
      "logits/rejected": 1.303057074546814,
      "logps/chosen": -321.0286560058594,
      "logps/ref_chosen": -290.1407775878906,
      "logps/ref_rejected": -239.65676879882812,
      "logps/rejected": -336.6887512207031,
      "loss": 4.4369,
      "margin_dpo/margin_mean": 66.1441421508789,
      "margin_dpo/margin_std": 81.36579895019531,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -43.311485290527344,
      "KL/mean": -66.23017120361328,
      "KL/rejected_KL_mean": -89.14887237548828,
      "KL/std": 72.75209045410156,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.005456462502479553,
      "fcm_dpo/delta": 0.011666701175272465,
      "fcm_dpo/margin": 45.837371826171875,
      "fcm_dpo/q_t": 0.44073203206062317,
      "grad_norm": 37.061256408691406,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": 1.343682885169983,
      "logits/rejected": 1.4286375045776367,
      "logps/chosen": -320.2475280761719,
      "logps/ref_chosen": -276.93603515625,
      "logps/ref_rejected": -250.398681640625,
      "logps/rejected": -339.5475769042969,
      "loss": 4.8488,
      "margin_dpo/margin_mean": 45.837371826171875,
      "margin_dpo/margin_std": 89.12725067138672,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -38.20805740356445,
      "KL/mean": -70.51373291015625,
      "KL/rejected_KL_mean": -102.81941223144531,
      "KL/std": 74.57706451416016,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.0054898327216506,
      "fcm_dpo/delta": 0.011693090200424194,
      "fcm_dpo/margin": 64.61135864257812,
      "fcm_dpo/q_t": 0.4166249930858612,
      "grad_norm": 42.620731353759766,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": 1.3972219228744507,
      "logits/rejected": 1.5013253688812256,
      "logps/chosen": -323.6381530761719,
      "logps/ref_chosen": -285.43011474609375,
      "logps/ref_rejected": -258.0743408203125,
      "logps/rejected": -360.8937683105469,
      "loss": 4.4902,
      "margin_dpo/margin_mean": 64.61135864257812,
      "margin_dpo/margin_std": 88.50875854492188,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -33.05979919433594,
      "KL/mean": -64.45976257324219,
      "KL/rejected_KL_mean": -95.8597183227539,
      "KL/std": 70.65982818603516,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.005541723221540451,
      "fcm_dpo/delta": -0.008421042934060097,
      "fcm_dpo/margin": 62.79994201660156,
      "fcm_dpo/q_t": 0.4173696041107178,
      "grad_norm": 32.33796310424805,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": 1.2541685104370117,
      "logits/rejected": 1.241849660873413,
      "logps/chosen": -332.4048156738281,
      "logps/ref_chosen": -299.3450012207031,
      "logps/ref_rejected": -244.60169982910156,
      "logps/rejected": -340.46142578125,
      "loss": 4.4653,
      "margin_dpo/margin_mean": 62.79993438720703,
      "margin_dpo/margin_std": 76.90019989013672,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -40.287635803222656,
      "KL/mean": -66.60220336914062,
      "KL/rejected_KL_mean": -92.91676330566406,
      "KL/std": 75.05889892578125,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.005518430843949318,
      "fcm_dpo/delta": -0.006673981435596943,
      "fcm_dpo/margin": 52.62913131713867,
      "fcm_dpo/q_t": 0.43163007497787476,
      "grad_norm": 36.838809967041016,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": 1.5159351825714111,
      "logits/rejected": 1.613511323928833,
      "logps/chosen": -313.23736572265625,
      "logps/ref_chosen": -272.9497375488281,
      "logps/ref_rejected": -274.6335754394531,
      "logps/rejected": -367.5503234863281,
      "loss": 4.695,
      "margin_dpo/margin_mean": 52.62913131713867,
      "margin_dpo/margin_std": 81.81671142578125,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -34.78303146362305,
      "KL/mean": -63.5547981262207,
      "KL/rejected_KL_mean": -92.3265609741211,
      "KL/std": 72.42831420898438,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.005558468401432037,
      "fcm_dpo/delta": 0.023662401363253593,
      "fcm_dpo/margin": 57.54353713989258,
      "fcm_dpo/q_t": 0.42448821663856506,
      "grad_norm": 36.073062896728516,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 1.1892077922821045,
      "logits/rejected": 1.225877285003662,
      "logps/chosen": -316.7245788574219,
      "logps/ref_chosen": -281.9415588378906,
      "logps/ref_rejected": -253.20486450195312,
      "logps/rejected": -345.53143310546875,
      "loss": 4.5553,
      "margin_dpo/margin_mean": 57.54353713989258,
      "margin_dpo/margin_std": 77.29570770263672,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -32.19007873535156,
      "KL/mean": -70.86051177978516,
      "KL/rejected_KL_mean": -109.53093719482422,
      "KL/std": 74.940673828125,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.00551729928702116,
      "fcm_dpo/delta": -0.06149332597851753,
      "fcm_dpo/margin": 77.34085083007812,
      "fcm_dpo/q_t": 0.39934274554252625,
      "grad_norm": 34.53076934814453,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": 1.3778717517852783,
      "logits/rejected": 1.3804051876068115,
      "logps/chosen": -312.8066711425781,
      "logps/ref_chosen": -280.6166076660156,
      "logps/ref_rejected": -263.7532958984375,
      "logps/rejected": -373.28424072265625,
      "loss": 4.2301,
      "margin_dpo/margin_mean": 77.34085845947266,
      "margin_dpo/margin_std": 80.02632141113281,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -39.57822799682617,
      "KL/mean": -76.4451675415039,
      "KL/rejected_KL_mean": -113.31210327148438,
      "KL/std": 75.20032501220703,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.005359685514122248,
      "fcm_dpo/delta": -0.012872972525656223,
      "fcm_dpo/margin": 73.73387145996094,
      "fcm_dpo/q_t": 0.40639638900756836,
      "grad_norm": 41.09707260131836,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": 1.4769716262817383,
      "logits/rejected": 1.6110912561416626,
      "logps/chosen": -325.740478515625,
      "logps/ref_chosen": -286.1622314453125,
      "logps/ref_rejected": -273.3803405761719,
      "logps/rejected": -386.69244384765625,
      "loss": 4.3062,
      "margin_dpo/margin_mean": 73.73387145996094,
      "margin_dpo/margin_std": 77.8399429321289,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -39.76506805419922,
      "KL/mean": -67.99298095703125,
      "KL/rejected_KL_mean": -96.22091674804688,
      "KL/std": 72.51812744140625,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.005215507000684738,
      "fcm_dpo/delta": -0.019996024668216705,
      "fcm_dpo/margin": 56.45586395263672,
      "fcm_dpo/q_t": 0.4306040406227112,
      "grad_norm": 45.82267761230469,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": 1.476718783378601,
      "logits/rejected": 1.5051639080047607,
      "logps/chosen": -306.1016540527344,
      "logps/ref_chosen": -266.3365783691406,
      "logps/ref_rejected": -255.01956176757812,
      "logps/rejected": -351.240478515625,
      "loss": 4.7176,
      "margin_dpo/margin_mean": 56.45586395263672,
      "margin_dpo/margin_std": 95.49129486083984,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -40.09527587890625,
      "KL/mean": -65.8067855834961,
      "KL/rejected_KL_mean": -91.51829528808594,
      "KL/std": 72.14607238769531,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.005191179923713207,
      "fcm_dpo/delta": -0.007898986339569092,
      "fcm_dpo/margin": 51.42304229736328,
      "fcm_dpo/q_t": 0.43663978576660156,
      "grad_norm": 55.412479400634766,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 1.218741536140442,
      "logits/rejected": 1.2435691356658936,
      "logps/chosen": -322.24664306640625,
      "logps/ref_chosen": -282.1513977050781,
      "logps/ref_rejected": -263.4792175292969,
      "logps/rejected": -354.99755859375,
      "loss": 4.7679,
      "margin_dpo/margin_mean": 51.42304229736328,
      "margin_dpo/margin_std": 86.4907455444336,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -43.98268508911133,
      "KL/mean": -67.72498321533203,
      "KL/rejected_KL_mean": -91.46729278564453,
      "KL/std": 73.0689697265625,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.005150371231138706,
      "fcm_dpo/delta": 0.009864707477390766,
      "fcm_dpo/margin": 47.484619140625,
      "fcm_dpo/q_t": 0.44222405552864075,
      "grad_norm": 39.449092864990234,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 1.2784985303878784,
      "logits/rejected": 1.4400746822357178,
      "logps/chosen": -312.69580078125,
      "logps/ref_chosen": -268.713134765625,
      "logps/ref_rejected": -266.13446044921875,
      "logps/rejected": -357.6017761230469,
      "loss": 4.8166,
      "margin_dpo/margin_mean": 47.484619140625,
      "margin_dpo/margin_std": 83.7557601928711,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -33.25787353515625,
      "KL/mean": -64.4706802368164,
      "KL/rejected_KL_mean": -95.68348693847656,
      "KL/std": 73.49506378173828,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.005200041923671961,
      "fcm_dpo/delta": 0.009530635550618172,
      "fcm_dpo/margin": 62.425594329833984,
      "fcm_dpo/q_t": 0.423394113779068,
      "grad_norm": 22.286808013916016,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": 1.5207293033599854,
      "logits/rejected": 1.5718684196472168,
      "logps/chosen": -317.2420959472656,
      "logps/ref_chosen": -283.9842224121094,
      "logps/ref_rejected": -253.9568634033203,
      "logps/rejected": -349.64031982421875,
      "loss": 4.552,
      "margin_dpo/margin_mean": 62.42559051513672,
      "margin_dpo/margin_std": 85.50220489501953,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -49.30999755859375,
      "KL/mean": -74.53396606445312,
      "KL/rejected_KL_mean": -99.7579345703125,
      "KL/std": 72.8354263305664,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.005280315410345793,
      "fcm_dpo/delta": 0.01891099289059639,
      "fcm_dpo/margin": 50.44794845581055,
      "fcm_dpo/q_t": 0.436132550239563,
      "grad_norm": 25.11457061767578,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": 1.0965906381607056,
      "logits/rejected": 1.1195716857910156,
      "logps/chosen": -291.7532043457031,
      "logps/ref_chosen": -242.4431915283203,
      "logps/ref_rejected": -252.87979125976562,
      "logps/rejected": -352.6377258300781,
      "loss": 4.7533,
      "margin_dpo/margin_mean": 50.44794845581055,
      "margin_dpo/margin_std": 84.1836929321289,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -43.49471664428711,
      "KL/mean": -73.01007843017578,
      "KL/rejected_KL_mean": -102.52542877197266,
      "KL/std": 73.61245727539062,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.005330992862582207,
      "fcm_dpo/delta": 0.006001044064760208,
      "fcm_dpo/margin": 59.03071212768555,
      "fcm_dpo/q_t": 0.42585131525993347,
      "grad_norm": 26.62639617919922,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": 1.4436386823654175,
      "logits/rejected": 1.3862559795379639,
      "logps/chosen": -331.77886962890625,
      "logps/ref_chosen": -288.2841796875,
      "logps/ref_rejected": -252.9123992919922,
      "logps/rejected": -355.43780517578125,
      "loss": 4.6014,
      "margin_dpo/margin_mean": 59.03071212768555,
      "margin_dpo/margin_std": 86.00122833251953,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -36.33592987060547,
      "KL/mean": -63.280540466308594,
      "KL/rejected_KL_mean": -90.22515869140625,
      "KL/std": 72.15150451660156,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.005374121945351362,
      "fcm_dpo/delta": 0.00406399741768837,
      "fcm_dpo/margin": 53.88923263549805,
      "fcm_dpo/q_t": 0.4311869442462921,
      "grad_norm": 61.77814483642578,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": 1.2704858779907227,
      "logits/rejected": 1.3609488010406494,
      "logps/chosen": -285.35076904296875,
      "logps/ref_chosen": -249.01483154296875,
      "logps/ref_rejected": -243.81192016601562,
      "logps/rejected": -334.0370788574219,
      "loss": 4.6657,
      "margin_dpo/margin_mean": 53.88923263549805,
      "margin_dpo/margin_std": 80.23640441894531,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -42.60380554199219,
      "KL/mean": -61.585479736328125,
      "KL/rejected_KL_mean": -80.56715393066406,
      "KL/std": 73.85432434082031,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.0053419070318341255,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.96335220336914,
      "fcm_dpo/q_t": 0.45194098353385925,
      "grad_norm": 36.67753982543945,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": 1.3122345209121704,
      "logits/rejected": 1.416359782218933,
      "logps/chosen": -355.5339050292969,
      "logps/ref_chosen": -312.9300537109375,
      "logps/ref_rejected": -296.2721862792969,
      "logps/rejected": -376.8393249511719,
      "loss": 4.9675,
      "margin_dpo/margin_mean": 37.96335220336914,
      "margin_dpo/margin_std": 81.49271392822266,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -35.319557189941406,
      "KL/mean": -67.47904205322266,
      "KL/rejected_KL_mean": -99.63851928710938,
      "KL/std": 75.74437713623047,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.0053402879275381565,
      "fcm_dpo/delta": -0.006010908633470535,
      "fcm_dpo/margin": 64.31895446777344,
      "fcm_dpo/q_t": 0.41860732436180115,
      "grad_norm": 31.58122444152832,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": 1.4615464210510254,
      "logits/rejected": 1.5106630325317383,
      "logps/chosen": -303.63348388671875,
      "logps/ref_chosen": -268.31390380859375,
      "logps/ref_rejected": -268.7603759765625,
      "logps/rejected": -368.39892578125,
      "loss": 4.5127,
      "margin_dpo/margin_mean": 64.31895446777344,
      "margin_dpo/margin_std": 86.06070709228516,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -50.74787139892578,
      "KL/mean": -66.31439971923828,
      "KL/rejected_KL_mean": -81.88093566894531,
      "KL/std": 71.43739318847656,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.005316280759871006,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.133054733276367,
      "fcm_dpo/q_t": 0.4608237147331238,
      "grad_norm": 94.07452392578125,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": 1.1274949312210083,
      "logits/rejected": 1.1915290355682373,
      "logps/chosen": -305.9797668457031,
      "logps/ref_chosen": -255.23190307617188,
      "logps/ref_rejected": -241.1186981201172,
      "logps/rejected": -322.9996337890625,
      "loss": 5.1202,
      "margin_dpo/margin_mean": 31.133054733276367,
      "margin_dpo/margin_std": 85.15692901611328,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -42.66007614135742,
      "KL/mean": -69.0288314819336,
      "KL/rejected_KL_mean": -95.39759826660156,
      "KL/std": 70.25579833984375,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.005457358900457621,
      "fcm_dpo/delta": 0.05171309411525726,
      "fcm_dpo/margin": 52.737518310546875,
      "fcm_dpo/q_t": 0.4323086738586426,
      "grad_norm": 35.11560821533203,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 1.2940787076950073,
      "logits/rejected": 1.3134608268737793,
      "logps/chosen": -293.6404113769531,
      "logps/ref_chosen": -250.9803466796875,
      "logps/ref_rejected": -239.9314422607422,
      "logps/rejected": -335.3290710449219,
      "loss": 4.6982,
      "margin_dpo/margin_mean": 52.737518310546875,
      "margin_dpo/margin_std": 85.96511840820312,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -50.95021057128906,
      "KL/mean": -72.17295837402344,
      "KL/rejected_KL_mean": -93.39569091796875,
      "KL/std": 70.75137329101562,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.005503377411514521,
      "fcm_dpo/delta": -0.017070434987545013,
      "fcm_dpo/margin": 42.445491790771484,
      "fcm_dpo/q_t": 0.44478821754455566,
      "grad_norm": 28.864763259887695,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 1.4687436819076538,
      "logits/rejected": 1.413029432296753,
      "logps/chosen": -351.9529724121094,
      "logps/ref_chosen": -301.0027770996094,
      "logps/ref_rejected": -263.628662109375,
      "logps/rejected": -357.02435302734375,
      "loss": 4.9051,
      "margin_dpo/margin_mean": 42.445491790771484,
      "margin_dpo/margin_std": 84.00487518310547,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -39.51060104370117,
      "KL/mean": -70.18974304199219,
      "KL/rejected_KL_mean": -100.8688735961914,
      "KL/std": 73.34302520751953,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.005407760851085186,
      "fcm_dpo/delta": -0.026254694908857346,
      "fcm_dpo/margin": 61.35826873779297,
      "fcm_dpo/q_t": 0.42123499512672424,
      "grad_norm": 33.42005157470703,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 1.3732541799545288,
      "logits/rejected": 1.4656100273132324,
      "logps/chosen": -340.359375,
      "logps/ref_chosen": -300.8487854003906,
      "logps/ref_rejected": -296.7118835449219,
      "logps/rejected": -397.5807800292969,
      "loss": 4.5184,
      "margin_dpo/margin_mean": 61.35826873779297,
      "margin_dpo/margin_std": 76.00444030761719,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -52.707637786865234,
      "KL/mean": -76.46123504638672,
      "KL/rejected_KL_mean": -100.21482849121094,
      "KL/std": 70.8086929321289,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.005435292609035969,
      "fcm_dpo/delta": 0.022825397551059723,
      "fcm_dpo/margin": 47.50719451904297,
      "fcm_dpo/q_t": 0.43925732374191284,
      "grad_norm": 40.03047180175781,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": 1.4757999181747437,
      "logits/rejected": 1.5388679504394531,
      "logps/chosen": -324.3560485839844,
      "logps/ref_chosen": -271.6484069824219,
      "logps/ref_rejected": -238.21197509765625,
      "logps/rejected": -338.4267883300781,
      "loss": 4.8399,
      "margin_dpo/margin_mean": 47.50719451904297,
      "margin_dpo/margin_std": 93.34744262695312,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -44.355552673339844,
      "KL/mean": -69.06871795654297,
      "KL/rejected_KL_mean": -93.78187561035156,
      "KL/std": 71.48081970214844,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.005491041578352451,
      "fcm_dpo/delta": -0.011893033981323242,
      "fcm_dpo/margin": 49.42631912231445,
      "fcm_dpo/q_t": 0.435776025056839,
      "grad_norm": 39.4298210144043,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": 1.1655833721160889,
      "logits/rejected": 1.2435380220413208,
      "logps/chosen": -310.6520690917969,
      "logps/ref_chosen": -266.2965087890625,
      "logps/ref_rejected": -263.7326965332031,
      "logps/rejected": -357.51458740234375,
      "loss": 4.7564,
      "margin_dpo/margin_mean": 49.42631530761719,
      "margin_dpo/margin_std": 81.57635498046875,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -40.9454231262207,
      "KL/mean": -67.91551208496094,
      "KL/rejected_KL_mean": -94.8855972290039,
      "KL/std": 74.42050170898438,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.005468276329338551,
      "fcm_dpo/delta": 0.02183919958770275,
      "fcm_dpo/margin": 53.94019317626953,
      "fcm_dpo/q_t": 0.4310198426246643,
      "grad_norm": 34.89653396606445,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": 1.1511905193328857,
      "logits/rejected": 1.153980016708374,
      "logps/chosen": -333.30621337890625,
      "logps/ref_chosen": -292.36077880859375,
      "logps/ref_rejected": -264.35662841796875,
      "logps/rejected": -359.24224853515625,
      "loss": 4.6868,
      "margin_dpo/margin_mean": 53.94019317626953,
      "margin_dpo/margin_std": 89.14730834960938,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -41.8572998046875,
      "KL/mean": -63.48856735229492,
      "KL/rejected_KL_mean": -85.11982727050781,
      "KL/std": 72.95282745361328,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.005471288692206144,
      "fcm_dpo/delta": -0.025690380483865738,
      "fcm_dpo/margin": 43.262535095214844,
      "fcm_dpo/q_t": 0.44415631890296936,
      "grad_norm": 32.03999710083008,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": 1.2278082370758057,
      "logits/rejected": 1.3330305814743042,
      "logps/chosen": -288.5771179199219,
      "logps/ref_chosen": -246.71978759765625,
      "logps/ref_rejected": -261.339599609375,
      "logps/rejected": -346.4594421386719,
      "loss": 4.8858,
      "margin_dpo/margin_mean": 43.262535095214844,
      "margin_dpo/margin_std": 81.8985824584961,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -33.342376708984375,
      "KL/mean": -64.02377319335938,
      "KL/rejected_KL_mean": -94.70516204833984,
      "KL/std": 71.90693664550781,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.00540059944614768,
      "fcm_dpo/delta": -0.0035153739154338837,
      "fcm_dpo/margin": 61.3627815246582,
      "fcm_dpo/q_t": 0.4216492176055908,
      "grad_norm": 43.415122985839844,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": 1.2869246006011963,
      "logits/rejected": 1.2912991046905518,
      "logps/chosen": -331.9010925292969,
      "logps/ref_chosen": -298.5587158203125,
      "logps/ref_rejected": -264.5877685546875,
      "logps/rejected": -359.29296875,
      "loss": 4.5535,
      "margin_dpo/margin_mean": 61.36278533935547,
      "margin_dpo/margin_std": 85.9680404663086,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -27.897111892700195,
      "KL/mean": -60.01478958129883,
      "KL/rejected_KL_mean": -92.13245391845703,
      "KL/std": 70.31501007080078,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.005377801135182381,
      "fcm_dpo/delta": 0.00554183404892683,
      "fcm_dpo/margin": 64.2353515625,
      "fcm_dpo/q_t": 0.41788265109062195,
      "grad_norm": 32.423030853271484,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": 1.1403666734695435,
      "logits/rejected": 1.1312565803527832,
      "logps/chosen": -277.7503662109375,
      "logps/ref_chosen": -249.8532257080078,
      "logps/ref_rejected": -233.2117156982422,
      "logps/rejected": -325.34417724609375,
      "loss": 4.4658,
      "margin_dpo/margin_mean": 64.2353515625,
      "margin_dpo/margin_std": 78.74893188476562,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -38.17671585083008,
      "KL/mean": -65.96183776855469,
      "KL/rejected_KL_mean": -93.7469482421875,
      "KL/std": 73.09352111816406,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.005345382262021303,
      "fcm_dpo/delta": -0.003636152483522892,
      "fcm_dpo/margin": 55.57023620605469,
      "fcm_dpo/q_t": 0.42944756150245667,
      "grad_norm": 34.737266540527344,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": 1.304903507232666,
      "logits/rejected": 1.4629226922988892,
      "logps/chosen": -296.99835205078125,
      "logps/ref_chosen": -258.8216247558594,
      "logps/ref_rejected": -260.4953308105469,
      "logps/rejected": -354.24224853515625,
      "loss": 4.657,
      "margin_dpo/margin_mean": 55.57023620605469,
      "margin_dpo/margin_std": 84.48978424072266,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -34.249107360839844,
      "KL/mean": -65.9332046508789,
      "KL/rejected_KL_mean": -97.61729431152344,
      "KL/std": 71.14087677001953,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.005350839346647263,
      "fcm_dpo/delta": -0.007685650605708361,
      "fcm_dpo/margin": 63.36819076538086,
      "fcm_dpo/q_t": 0.4195067584514618,
      "grad_norm": 29.561708450317383,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": 1.4659944772720337,
      "logits/rejected": 1.3658264875411987,
      "logps/chosen": -358.46563720703125,
      "logps/ref_chosen": -324.2165222167969,
      "logps/ref_rejected": -237.05453491210938,
      "logps/rejected": -334.6718444824219,
      "loss": 4.5097,
      "margin_dpo/margin_mean": 63.36819076538086,
      "margin_dpo/margin_std": 83.65818786621094,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -49.38461685180664,
      "KL/mean": -73.5987777709961,
      "KL/rejected_KL_mean": -97.81291198730469,
      "KL/std": 74.511474609375,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.005427522584795952,
      "fcm_dpo/delta": 0.03262011334300041,
      "fcm_dpo/margin": 48.428306579589844,
      "fcm_dpo/q_t": 0.43783727288246155,
      "grad_norm": 44.741050720214844,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 1.3428983688354492,
      "logits/rejected": 1.3335058689117432,
      "logps/chosen": -327.785888671875,
      "logps/ref_chosen": -278.4012451171875,
      "logps/ref_rejected": -247.72671508789062,
      "logps/rejected": -345.5396728515625,
      "loss": 4.7676,
      "margin_dpo/margin_mean": 48.42830276489258,
      "margin_dpo/margin_std": 82.47622680664062,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -39.26097869873047,
      "KL/mean": -66.04202270507812,
      "KL/rejected_KL_mean": -92.82305908203125,
      "KL/std": 70.11302947998047,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.005454204976558685,
      "fcm_dpo/delta": -0.009797046892344952,
      "fcm_dpo/margin": 53.562076568603516,
      "fcm_dpo/q_t": 0.4307946562767029,
      "grad_norm": 42.751670837402344,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.2704087495803833,
      "logits/rejected": 1.291514277458191,
      "logps/chosen": -331.3555908203125,
      "logps/ref_chosen": -292.0946350097656,
      "logps/ref_rejected": -252.24237060546875,
      "logps/rejected": -345.0654602050781,
      "loss": 4.6742,
      "margin_dpo/margin_mean": 53.562076568603516,
      "margin_dpo/margin_std": 80.89630889892578,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -32.64827346801758,
      "KL/mean": -58.666717529296875,
      "KL/rejected_KL_mean": -84.68517303466797,
      "KL/std": 73.98143005371094,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.005409210920333862,
      "fcm_dpo/delta": -0.01027128379791975,
      "fcm_dpo/margin": 52.036895751953125,
      "fcm_dpo/q_t": 0.4333280026912689,
      "grad_norm": 21.756975173950195,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.3500585556030273,
      "logits/rejected": 1.4132287502288818,
      "logps/chosen": -323.25909423828125,
      "logps/ref_chosen": -290.61077880859375,
      "logps/ref_rejected": -279.8963928222656,
      "logps/rejected": -364.5815734863281,
      "loss": 4.6933,
      "margin_dpo/margin_mean": 52.036895751953125,
      "margin_dpo/margin_std": 79.72173309326172,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -39.42767333984375,
      "KL/mean": -69.54899597167969,
      "KL/rejected_KL_mean": -99.67030334472656,
      "KL/std": 72.37506866455078,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.005495256744325161,
      "fcm_dpo/delta": 0.035183586180210114,
      "fcm_dpo/margin": 60.24263000488281,
      "fcm_dpo/q_t": 0.42164164781570435,
      "grad_norm": 37.136356353759766,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": 1.4674394130706787,
      "logits/rejected": 1.4397181272506714,
      "logps/chosen": -355.7429504394531,
      "logps/ref_chosen": -316.3152770996094,
      "logps/ref_rejected": -300.5086975097656,
      "logps/rejected": -400.17901611328125,
      "loss": 4.5261,
      "margin_dpo/margin_mean": 60.24263000488281,
      "margin_dpo/margin_std": 81.05773162841797,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -40.36354446411133,
      "KL/mean": -68.06829833984375,
      "KL/rejected_KL_mean": -95.77305603027344,
      "KL/std": 75.7471923828125,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.005551543552428484,
      "fcm_dpo/delta": -0.010645093396306038,
      "fcm_dpo/margin": 55.40950012207031,
      "fcm_dpo/q_t": 0.4273066520690918,
      "grad_norm": 33.33755874633789,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": 1.6698086261749268,
      "logits/rejected": 1.7771351337432861,
      "logps/chosen": -330.9088439941406,
      "logps/ref_chosen": -290.5452575683594,
      "logps/ref_rejected": -297.2102355957031,
      "logps/rejected": -392.9832763671875,
      "loss": 4.6191,
      "margin_dpo/margin_mean": 55.40950012207031,
      "margin_dpo/margin_std": 80.9318618774414,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -34.59906768798828,
      "KL/mean": -57.92844009399414,
      "KL/rejected_KL_mean": -81.2578125,
      "KL/std": 70.01815795898438,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.005563527811318636,
      "fcm_dpo/delta": 0.027157753705978394,
      "fcm_dpo/margin": 46.65873718261719,
      "fcm_dpo/q_t": 0.43863487243652344,
      "grad_norm": 53.62447738647461,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": 1.2376124858856201,
      "logits/rejected": 1.3392072916030884,
      "logps/chosen": -298.0194091796875,
      "logps/ref_chosen": -263.4203186035156,
      "logps/ref_rejected": -270.30291748046875,
      "logps/rejected": -351.56072998046875,
      "loss": 4.7822,
      "margin_dpo/margin_mean": 46.65874099731445,
      "margin_dpo/margin_std": 81.40591430664062,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -41.21111297607422,
      "KL/mean": -74.95854187011719,
      "KL/rejected_KL_mean": -108.7059555053711,
      "KL/std": 77.47089385986328,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.005615747533738613,
      "fcm_dpo/delta": -0.014343283139169216,
      "fcm_dpo/margin": 67.49485778808594,
      "fcm_dpo/q_t": 0.4116773307323456,
      "grad_norm": 43.248870849609375,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": 1.241938829421997,
      "logits/rejected": 1.4058455228805542,
      "logps/chosen": -353.24847412109375,
      "logps/ref_chosen": -312.0373840332031,
      "logps/ref_rejected": -290.50946044921875,
      "logps/rejected": -399.21539306640625,
      "loss": 4.4392,
      "margin_dpo/margin_mean": 67.49485778808594,
      "margin_dpo/margin_std": 91.81751251220703,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -43.950862884521484,
      "KL/mean": -73.94917297363281,
      "KL/rejected_KL_mean": -103.9474868774414,
      "KL/std": 75.67977142333984,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.005626625847071409,
      "fcm_dpo/delta": 0.020244503393769264,
      "fcm_dpo/margin": 59.99661636352539,
      "fcm_dpo/q_t": 0.42078155279159546,
      "grad_norm": 35.218299865722656,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": 1.346535325050354,
      "logits/rejected": 1.383528232574463,
      "logps/chosen": -341.1228332519531,
      "logps/ref_chosen": -297.1719970703125,
      "logps/ref_rejected": -269.15557861328125,
      "logps/rejected": -373.1030578613281,
      "loss": 4.5524,
      "margin_dpo/margin_mean": 59.99662399291992,
      "margin_dpo/margin_std": 88.69428253173828,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -39.65129089355469,
      "KL/mean": -66.38029479980469,
      "KL/rejected_KL_mean": -93.10931396484375,
      "KL/std": 75.7659912109375,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.0056646582670509815,
      "fcm_dpo/delta": -0.0014877889771014452,
      "fcm_dpo/margin": 53.458011627197266,
      "fcm_dpo/q_t": 0.42914968729019165,
      "grad_norm": 24.431915283203125,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": 1.3997750282287598,
      "logits/rejected": 1.3364354372024536,
      "logps/chosen": -327.0162048339844,
      "logps/ref_chosen": -287.3648986816406,
      "logps/ref_rejected": -271.1472473144531,
      "logps/rejected": -364.25653076171875,
      "loss": 4.6632,
      "margin_dpo/margin_mean": 53.458011627197266,
      "margin_dpo/margin_std": 83.98710632324219,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -47.9534797668457,
      "KL/mean": -68.57614135742188,
      "KL/rejected_KL_mean": -89.19880676269531,
      "KL/std": 72.66455078125,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.005715405102819204,
      "fcm_dpo/delta": 0.032105423510074615,
      "fcm_dpo/margin": 41.24533462524414,
      "fcm_dpo/q_t": 0.44492238759994507,
      "grad_norm": 38.6359748840332,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": 1.2302157878875732,
      "logits/rejected": 1.312467336654663,
      "logps/chosen": -336.55963134765625,
      "logps/ref_chosen": -288.6061096191406,
      "logps/ref_rejected": -285.6366271972656,
      "logps/rejected": -374.83544921875,
      "loss": 4.9193,
      "margin_dpo/margin_mean": 41.24532699584961,
      "margin_dpo/margin_std": 88.28429412841797,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -45.02811813354492,
      "KL/mean": -65.65428161621094,
      "KL/rejected_KL_mean": -86.28044128417969,
      "KL/std": 71.26193237304688,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.005793233402073383,
      "fcm_dpo/delta": 0.0011298726312816143,
      "fcm_dpo/margin": 41.252315521240234,
      "fcm_dpo/q_t": 0.4440900385379791,
      "grad_norm": 45.50015640258789,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 1.3111213445663452,
      "logits/rejected": 1.2763772010803223,
      "logps/chosen": -355.28009033203125,
      "logps/ref_chosen": -310.251953125,
      "logps/ref_rejected": -252.962158203125,
      "logps/rejected": -339.2425842285156,
      "loss": 4.931,
      "margin_dpo/margin_mean": 41.25231170654297,
      "margin_dpo/margin_std": 90.51017761230469,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -40.65153121948242,
      "KL/mean": -67.74393463134766,
      "KL/rejected_KL_mean": -94.8363265991211,
      "KL/std": 74.61626434326172,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.005803259089589119,
      "fcm_dpo/delta": -0.006100666243582964,
      "fcm_dpo/margin": 54.184791564941406,
      "fcm_dpo/q_t": 0.426129549741745,
      "grad_norm": 35.061283111572266,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 1.2899760007858276,
      "logits/rejected": 1.4286366701126099,
      "logps/chosen": -277.4319763183594,
      "logps/ref_chosen": -236.7804412841797,
      "logps/ref_rejected": -229.67550659179688,
      "logps/rejected": -324.5118408203125,
      "loss": 4.6624,
      "margin_dpo/margin_mean": 54.18479919433594,
      "margin_dpo/margin_std": 89.54027557373047,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -35.26939010620117,
      "KL/mean": -69.46683502197266,
      "KL/rejected_KL_mean": -103.66427612304688,
      "KL/std": 75.2166519165039,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.005735974758863449,
      "fcm_dpo/delta": -0.007208782713860273,
      "fcm_dpo/margin": 68.39486694335938,
      "fcm_dpo/q_t": 0.40824374556541443,
      "grad_norm": 32.09534454345703,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": 1.3994579315185547,
      "logits/rejected": 1.3183456659317017,
      "logps/chosen": -370.5729064941406,
      "logps/ref_chosen": -335.30352783203125,
      "logps/ref_rejected": -250.6796875,
      "logps/rejected": -354.3439636230469,
      "loss": 4.4142,
      "margin_dpo/margin_mean": 68.39486694335938,
      "margin_dpo/margin_std": 93.53690338134766,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -43.66324234008789,
      "KL/mean": -67.69544219970703,
      "KL/rejected_KL_mean": -91.72764587402344,
      "KL/std": 72.91073608398438,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.005779605358839035,
      "fcm_dpo/delta": 0.02346043847501278,
      "fcm_dpo/margin": 48.06439971923828,
      "fcm_dpo/q_t": 0.43457457423210144,
      "grad_norm": 61.08562469482422,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": 1.4621200561523438,
      "logits/rejected": 1.3093297481536865,
      "logps/chosen": -331.88348388671875,
      "logps/ref_chosen": -288.22021484375,
      "logps/ref_rejected": -227.96673583984375,
      "logps/rejected": -319.6943664550781,
      "loss": 4.7352,
      "margin_dpo/margin_mean": 48.06439971923828,
      "margin_dpo/margin_std": 81.41033172607422,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -44.847442626953125,
      "KL/mean": -71.01887512207031,
      "KL/rejected_KL_mean": -97.19029235839844,
      "KL/std": 72.06629943847656,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.005762441083788872,
      "fcm_dpo/delta": -0.026485417038202286,
      "fcm_dpo/margin": 52.342857360839844,
      "fcm_dpo/q_t": 0.42821836471557617,
      "grad_norm": 43.461761474609375,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": 1.4079946279525757,
      "logits/rejected": 1.4409602880477905,
      "logps/chosen": -305.18438720703125,
      "logps/ref_chosen": -260.3369140625,
      "logps/ref_rejected": -254.2083740234375,
      "logps/rejected": -351.3986511230469,
      "loss": 4.6579,
      "margin_dpo/margin_mean": 52.342857360839844,
      "margin_dpo/margin_std": 81.2108154296875,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -28.497591018676758,
      "KL/mean": -62.084896087646484,
      "KL/rejected_KL_mean": -95.67220306396484,
      "KL/std": 73.44735717773438,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.005517670884728432,
      "fcm_dpo/delta": -0.05884559452533722,
      "fcm_dpo/margin": 67.17460632324219,
      "fcm_dpo/q_t": 0.4123648405075073,
      "grad_norm": 25.142335891723633,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.480944275856018,
      "logits/rejected": 1.5334943532943726,
      "logps/chosen": -311.56707763671875,
      "logps/ref_chosen": -283.0694885253906,
      "logps/ref_rejected": -289.1370544433594,
      "logps/rejected": -384.80926513671875,
      "loss": 4.4371,
      "margin_dpo/margin_mean": 67.17459869384766,
      "margin_dpo/margin_std": 82.15501403808594,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -31.041288375854492,
      "KL/mean": -65.6865234375,
      "KL/rejected_KL_mean": -100.33175659179688,
      "KL/std": 73.89168548583984,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.005386871751397848,
      "fcm_dpo/delta": -0.01800444908440113,
      "fcm_dpo/margin": 69.29046630859375,
      "fcm_dpo/q_t": 0.41118213534355164,
      "grad_norm": 26.608245849609375,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": 0.993057906627655,
      "logits/rejected": 1.0527089834213257,
      "logps/chosen": -287.92633056640625,
      "logps/ref_chosen": -256.8850402832031,
      "logps/ref_rejected": -247.0802459716797,
      "logps/rejected": -347.4120178222656,
      "loss": 4.3829,
      "margin_dpo/margin_mean": 69.29046630859375,
      "margin_dpo/margin_std": 80.81978607177734,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -41.60496139526367,
      "KL/mean": -67.72435760498047,
      "KL/rejected_KL_mean": -93.8437728881836,
      "KL/std": 73.697265625,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.005383265670388937,
      "fcm_dpo/delta": 0.01237080805003643,
      "fcm_dpo/margin": 52.238792419433594,
      "fcm_dpo/q_t": 0.4337858557701111,
      "grad_norm": 29.328641891479492,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": 1.164002776145935,
      "logits/rejected": 1.3239754438400269,
      "logps/chosen": -299.1290283203125,
      "logps/ref_chosen": -257.5240783691406,
      "logps/ref_rejected": -274.82122802734375,
      "logps/rejected": -368.6650085449219,
      "loss": 4.7328,
      "margin_dpo/margin_mean": 52.238792419433594,
      "margin_dpo/margin_std": 86.84254455566406,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -35.342124938964844,
      "KL/mean": -57.937896728515625,
      "KL/rejected_KL_mean": -80.53366088867188,
      "KL/std": 71.39872741699219,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.005425299052149057,
      "fcm_dpo/delta": -0.013705480843782425,
      "fcm_dpo/margin": 45.19153594970703,
      "fcm_dpo/q_t": 0.44210389256477356,
      "grad_norm": 37.754859924316406,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": 1.4868371486663818,
      "logits/rejected": 1.384922742843628,
      "logps/chosen": -347.6018371582031,
      "logps/ref_chosen": -312.2597351074219,
      "logps/ref_rejected": -259.74383544921875,
      "logps/rejected": -340.2774658203125,
      "loss": 4.8098,
      "margin_dpo/margin_mean": 45.19153594970703,
      "margin_dpo/margin_std": 76.44750213623047,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -31.64827537536621,
      "KL/mean": -61.37698745727539,
      "KL/rejected_KL_mean": -91.10569763183594,
      "KL/std": 75.57807922363281,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.005401932168751955,
      "fcm_dpo/delta": -0.0012713768519461155,
      "fcm_dpo/margin": 59.457427978515625,
      "fcm_dpo/q_t": 0.42392513155937195,
      "grad_norm": 31.80645179748535,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": 1.1904560327529907,
      "logits/rejected": 1.1043834686279297,
      "logps/chosen": -348.68798828125,
      "logps/ref_chosen": -317.03973388671875,
      "logps/ref_rejected": -260.12628173828125,
      "logps/rejected": -351.23193359375,
      "loss": 4.6011,
      "margin_dpo/margin_mean": 59.457427978515625,
      "margin_dpo/margin_std": 89.42403411865234,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -29.787275314331055,
      "KL/mean": -62.9322624206543,
      "KL/rejected_KL_mean": -96.07725524902344,
      "KL/std": 74.51081848144531,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.005305787548422813,
      "fcm_dpo/delta": -0.04282934218645096,
      "fcm_dpo/margin": 66.28998565673828,
      "fcm_dpo/q_t": 0.4167814254760742,
      "grad_norm": 36.12846755981445,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 1.311298131942749,
      "logits/rejected": 1.2498576641082764,
      "logps/chosen": -334.18121337890625,
      "logps/ref_chosen": -304.3939514160156,
      "logps/ref_rejected": -257.6587219238281,
      "logps/rejected": -353.7359619140625,
      "loss": 4.4867,
      "margin_dpo/margin_mean": 66.28998565673828,
      "margin_dpo/margin_std": 84.85897827148438,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -41.68468475341797,
      "KL/mean": -66.48046112060547,
      "KL/rejected_KL_mean": -91.27623748779297,
      "KL/std": 74.9468994140625,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.005222363863140345,
      "fcm_dpo/delta": 0.011921152472496033,
      "fcm_dpo/margin": 49.59154510498047,
      "fcm_dpo/q_t": 0.4387713074684143,
      "grad_norm": 66.71401977539062,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 1.1977286338806152,
      "logits/rejected": 1.2768800258636475,
      "logps/chosen": -326.6285400390625,
      "logps/ref_chosen": -284.94384765625,
      "logps/ref_rejected": -297.4357604980469,
      "logps/rejected": -388.7120056152344,
      "loss": 4.7733,
      "margin_dpo/margin_mean": 49.591552734375,
      "margin_dpo/margin_std": 83.81500244140625,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -31.604597091674805,
      "KL/mean": -62.937870025634766,
      "KL/rejected_KL_mean": -94.2711410522461,
      "KL/std": 72.7457504272461,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.005277193151414394,
      "fcm_dpo/delta": 0.009456639178097248,
      "fcm_dpo/margin": 62.66654968261719,
      "fcm_dpo/q_t": 0.42240428924560547,
      "grad_norm": 48.23680877685547,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": 1.4072538614273071,
      "logits/rejected": 1.5504512786865234,
      "logps/chosen": -263.9588623046875,
      "logps/ref_chosen": -232.35427856445312,
      "logps/ref_rejected": -245.6057891845703,
      "logps/rejected": -339.8769226074219,
      "loss": 4.552,
      "margin_dpo/margin_mean": 62.66654968261719,
      "margin_dpo/margin_std": 88.19377136230469,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -37.94249725341797,
      "KL/mean": -69.90180206298828,
      "KL/rejected_KL_mean": -101.86109924316406,
      "KL/std": 73.83494567871094,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.005309076979756355,
      "fcm_dpo/delta": 0.00706434715539217,
      "fcm_dpo/margin": 63.91862106323242,
      "fcm_dpo/q_t": 0.4197175204753876,
      "grad_norm": 31.56553840637207,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": 1.2976325750350952,
      "logits/rejected": 1.3610479831695557,
      "logps/chosen": -315.9493713378906,
      "logps/ref_chosen": -278.0068664550781,
      "logps/ref_rejected": -257.19256591796875,
      "logps/rejected": -359.0537109375,
      "loss": 4.5161,
      "margin_dpo/margin_mean": 63.91862106323242,
      "margin_dpo/margin_std": 86.35836029052734,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -48.252838134765625,
      "KL/mean": -71.77349853515625,
      "KL/rejected_KL_mean": -95.294189453125,
      "KL/std": 72.57025146484375,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.005307510960847139,
      "fcm_dpo/delta": -0.0005860645323991776,
      "fcm_dpo/margin": 47.04135513305664,
      "fcm_dpo/q_t": 0.4404796361923218,
      "grad_norm": 37.01661682128906,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": 1.3139057159423828,
      "logits/rejected": 1.382585048675537,
      "logps/chosen": -329.0103454589844,
      "logps/ref_chosen": -280.75750732421875,
      "logps/ref_rejected": -264.34295654296875,
      "logps/rejected": -359.63714599609375,
      "loss": 4.804,
      "margin_dpo/margin_mean": 47.04135513305664,
      "margin_dpo/margin_std": 78.60551452636719,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -35.262657165527344,
      "KL/mean": -59.975975036621094,
      "KL/rejected_KL_mean": -84.68930053710938,
      "KL/std": 69.97857666015625,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.005357269663363695,
      "fcm_dpo/delta": 0.024888351559638977,
      "fcm_dpo/margin": 49.426639556884766,
      "fcm_dpo/q_t": 0.4374130070209503,
      "grad_norm": 31.577306747436523,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": 1.2353053092956543,
      "logits/rejected": 1.37124764919281,
      "logps/chosen": -276.7340087890625,
      "logps/ref_chosen": -241.4713592529297,
      "logps/ref_rejected": -244.0782928466797,
      "logps/rejected": -328.767578125,
      "loss": 4.7644,
      "margin_dpo/margin_mean": 49.42664337158203,
      "margin_dpo/margin_std": 82.26480865478516,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -25.37890625,
      "KL/mean": -54.56439971923828,
      "KL/rejected_KL_mean": -83.7498779296875,
      "KL/std": 70.49406433105469,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.0053556268103420734,
      "fcm_dpo/delta": -0.016656864434480667,
      "fcm_dpo/margin": 58.370975494384766,
      "fcm_dpo/q_t": 0.4259790778160095,
      "grad_norm": 22.241384506225586,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": 1.4514435529708862,
      "logits/rejected": 1.4686757326126099,
      "logps/chosen": -344.1532287597656,
      "logps/ref_chosen": -318.7742919921875,
      "logps/ref_rejected": -293.66595458984375,
      "logps/rejected": -377.41583251953125,
      "loss": 4.6088,
      "margin_dpo/margin_mean": 58.370975494384766,
      "margin_dpo/margin_std": 84.55082702636719,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -35.83843231201172,
      "KL/mean": -67.55340576171875,
      "KL/rejected_KL_mean": -99.26837921142578,
      "KL/std": 74.76976013183594,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.005404382944107056,
      "fcm_dpo/delta": 0.005659319926053286,
      "fcm_dpo/margin": 63.42994689941406,
      "fcm_dpo/q_t": 0.41933223605155945,
      "grad_norm": 33.6338005065918,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": 1.3146765232086182,
      "logits/rejected": 1.3630093336105347,
      "logps/chosen": -301.7741394042969,
      "logps/ref_chosen": -265.9356994628906,
      "logps/ref_rejected": -243.6986083984375,
      "logps/rejected": -342.96697998046875,
      "loss": 4.535,
      "margin_dpo/margin_mean": 63.4299430847168,
      "margin_dpo/margin_std": 90.45182800292969,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 2.690188397901363,
      "train_runtime": 4372.8359,
      "train_samples_per_second": 13.981,
      "train_steps_per_second": 0.109
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}