{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.04536247253417969,
      "KL/mean": -0.05389120429754257,
      "KL/rejected_KL_mean": -0.06241607666015625,
      "KL/std": 0.5513200759887695,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01704825460910797,
      "fcm_dpo/q_t": 0.499957412481308,
      "grad_norm": 14.618359565734863,
      "learning_rate": 0.0,
      "logits/chosen": 1.9854813814163208,
      "logits/rejected": 1.7737188339233398,
      "logps/chosen": -267.0936584472656,
      "logps/ref_chosen": -267.04827880859375,
      "logps/ref_rejected": -219.76657104492188,
      "logps/rejected": -219.82899475097656,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.01704716682434082,
      "margin_dpo/margin_std": 0.7467707991600037,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.07550811767578125,
      "KL/mean": 0.03060000389814377,
      "KL/rejected_KL_mean": -0.014301300048828125,
      "KL/std": 0.6403332352638245,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08979703485965729,
      "fcm_dpo/q_t": 0.4997755289077759,
      "grad_norm": 15.139681816101074,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 2.1993117332458496,
      "logits/rejected": 2.1368322372436523,
      "logps/chosen": -256.5101623535156,
      "logps/ref_chosen": -256.585693359375,
      "logps/ref_rejected": -239.92742919921875,
      "logps/rejected": -239.94171142578125,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08979731798171997,
      "margin_dpo/margin_std": 0.9745746850967407,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": -0.06587791442871094,
      "KL/mean": -0.08352743089199066,
      "KL/rejected_KL_mean": -0.101165771484375,
      "KL/std": 0.6164373755455017,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03528332710266113,
      "fcm_dpo/q_t": 0.4999118149280548,
      "grad_norm": 14.608208656311035,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": 2.256659507751465,
      "logits/rejected": 2.296133279800415,
      "logps/chosen": -265.7347412109375,
      "logps/ref_chosen": -265.66888427734375,
      "logps/ref_rejected": -284.3778991699219,
      "logps/rejected": -284.47906494140625,
      "loss": 5.5438,
      "margin_dpo/margin_mean": 0.03528323769569397,
      "margin_dpo/margin_std": 0.795761227607727,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.08188247680664062,
      "KL/mean": 0.06199420988559723,
      "KL/rejected_KL_mean": 0.042110443115234375,
      "KL/std": 0.6389985084533691,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.039766669273376465,
      "fcm_dpo/q_t": 0.49990057945251465,
      "grad_norm": 15.714363098144531,
      "learning_rate": 3.125e-08,
      "logits/chosen": 1.8713123798370361,
      "logits/rejected": 1.8089830875396729,
      "logps/chosen": -319.42559814453125,
      "logps/ref_chosen": -319.50750732421875,
      "logps/ref_rejected": -296.22314453125,
      "logps/rejected": -296.1810302734375,
      "loss": 5.5437,
      "margin_dpo/margin_mean": 0.03976723551750183,
      "margin_dpo/margin_std": 0.8642778396606445,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.011129379272460938,
      "KL/mean": 0.02145184576511383,
      "KL/rejected_KL_mean": 0.031780242919921875,
      "KL/std": 0.6017542481422424,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.02064874768257141,
      "fcm_dpo/q_t": 0.5000516176223755,
      "grad_norm": 15.625190734863281,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": 1.848024845123291,
      "logits/rejected": 1.8500643968582153,
      "logps/chosen": -292.8618469238281,
      "logps/ref_chosen": -292.87298583984375,
      "logps/ref_rejected": -270.3802795410156,
      "logps/rejected": -270.3485107421875,
      "loss": 5.5461,
      "margin_dpo/margin_mean": -0.020647287368774414,
      "margin_dpo/margin_std": 0.7839505672454834,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": -0.026639938354492188,
      "KL/mean": -0.023511432111263275,
      "KL/rejected_KL_mean": -0.020376205444335938,
      "KL/std": 0.6144284605979919,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.006271347403526306,
      "fcm_dpo/q_t": 0.5000157356262207,
      "grad_norm": 15.454659461975098,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 2.2118873596191406,
      "logits/rejected": 2.1529176235198975,
      "logps/chosen": -280.31207275390625,
      "logps/ref_chosen": -280.28546142578125,
      "logps/ref_rejected": -242.7611541748047,
      "logps/rejected": -242.7815399169922,
      "loss": 5.5455,
      "margin_dpo/margin_mean": -0.006272077560424805,
      "margin_dpo/margin_std": 0.8461670279502869,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.08706855773925781,
      "KL/mean": 0.05976106971502304,
      "KL/rejected_KL_mean": 0.032459259033203125,
      "KL/std": 0.6347144246101379,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05461118370294571,
      "fcm_dpo/q_t": 0.49986347556114197,
      "grad_norm": 15.701974868774414,
      "learning_rate": 6.25e-08,
      "logits/chosen": 1.9548264741897583,
      "logits/rejected": 1.8622612953186035,
      "logps/chosen": -255.01864624023438,
      "logps/ref_chosen": -255.10572814941406,
      "logps/ref_rejected": -251.15402221679688,
      "logps/rejected": -251.12155151367188,
      "loss": 5.5431,
      "margin_dpo/margin_mean": 0.05461117625236511,
      "margin_dpo/margin_std": 0.8225716948509216,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.010669708251953125,
      "KL/mean": 0.027865156531333923,
      "KL/rejected_KL_mean": 0.06640052795410156,
      "KL/std": 0.7284108996391296,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07707974314689636,
      "fcm_dpo/q_t": 0.5001927018165588,
      "grad_norm": 16.406755447387695,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": 2.2714736461639404,
      "logits/rejected": 2.0743818283081055,
      "logps/chosen": -345.1922302246094,
      "logps/ref_chosen": -345.18157958984375,
      "logps/ref_rejected": -277.3522644042969,
      "logps/rejected": -277.285888671875,
      "loss": 5.5484,
      "margin_dpo/margin_mean": -0.07707980275154114,
      "margin_dpo/margin_std": 1.0355305671691895,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.06557464599609375,
      "KL/mean": -0.04673897475004196,
      "KL/rejected_KL_mean": -0.02790069580078125,
      "KL/std": 0.6569837927818298,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03767143189907074,
      "fcm_dpo/q_t": 0.5000941753387451,
      "grad_norm": 14.084053039550781,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": 1.997477650642395,
      "logits/rejected": 1.968400001525879,
      "logps/chosen": -276.7959289550781,
      "logps/ref_chosen": -276.7303161621094,
      "logps/ref_rejected": -266.19952392578125,
      "logps/rejected": -266.2274169921875,
      "loss": 5.5468,
      "margin_dpo/margin_mean": -0.0376712828874588,
      "margin_dpo/margin_std": 0.8946220874786377,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": -0.07577896118164062,
      "KL/mean": -0.07722152769565582,
      "KL/rejected_KL_mean": -0.07865524291992188,
      "KL/std": 0.5632377862930298,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0028744637966156006,
      "fcm_dpo/q_t": 0.4999927878379822,
      "grad_norm": 13.755327224731445,
      "learning_rate": 9.375e-08,
      "logits/chosen": 1.9406746625900269,
      "logits/rejected": 1.9932451248168945,
      "logps/chosen": -246.63803100585938,
      "logps/ref_chosen": -246.562255859375,
      "logps/ref_rejected": -224.70401000976562,
      "logps/rejected": -224.78265380859375,
      "loss": 5.5451,
      "margin_dpo/margin_mean": 0.0028738677501678467,
      "margin_dpo/margin_std": 0.7276897430419922,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": -0.033191680908203125,
      "KL/mean": -0.059243060648441315,
      "KL/rejected_KL_mean": -0.08529472351074219,
      "KL/std": 0.6480485200881958,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05209481716156006,
      "fcm_dpo/q_t": 0.4998697340488434,
      "grad_norm": 15.365917205810547,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 2.193141460418701,
      "logits/rejected": 2.162012815475464,
      "logps/chosen": -290.5343017578125,
      "logps/ref_chosen": -290.5010986328125,
      "logps/ref_rejected": -253.9559326171875,
      "logps/rejected": -254.04119873046875,
      "loss": 5.5432,
      "margin_dpo/margin_mean": 0.052095331251621246,
      "margin_dpo/margin_std": 0.8792738914489746,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": 0.06145668029785156,
      "KL/mean": 0.05036916583776474,
      "KL/rejected_KL_mean": 0.03929328918457031,
      "KL/std": 0.6098710894584656,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.022167548537254333,
      "fcm_dpo/q_t": 0.4999445378780365,
      "grad_norm": 14.80578327178955,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": 1.6256630420684814,
      "logits/rejected": 1.6108452081680298,
      "logps/chosen": -278.4570617675781,
      "logps/ref_chosen": -278.5185241699219,
      "logps/ref_rejected": -273.56207275390625,
      "logps/rejected": -273.5227966308594,
      "loss": 5.5444,
      "margin_dpo/margin_mean": 0.022167682647705078,
      "margin_dpo/margin_std": 0.8828686475753784,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.00852203369140625,
      "KL/mean": -0.022243589162826538,
      "KL/rejected_KL_mean": -0.03595924377441406,
      "KL/std": 0.5769276022911072,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.027436017990112305,
      "fcm_dpo/q_t": 0.4999314248561859,
      "grad_norm": 14.823359489440918,
      "learning_rate": 1.25e-07,
      "logits/chosen": 1.781716227531433,
      "logits/rejected": 1.9066872596740723,
      "logps/chosen": -264.6656494140625,
      "logps/ref_chosen": -264.6571350097656,
      "logps/ref_rejected": -274.3492736816406,
      "logps/rejected": -274.3852844238281,
      "loss": 5.5441,
      "margin_dpo/margin_mean": 0.027435779571533203,
      "margin_dpo/margin_std": 0.7559254765510559,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.09064865112304688,
      "KL/mean": 0.06694958359003067,
      "KL/rejected_KL_mean": 0.043247222900390625,
      "KL/std": 0.5763744115829468,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.047401100397109985,
      "fcm_dpo/q_t": 0.49988147616386414,
      "grad_norm": 15.71843433380127,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": 1.954875111579895,
      "logits/rejected": 1.9788742065429688,
      "logps/chosen": -274.02392578125,
      "logps/ref_chosen": -274.1145935058594,
      "logps/ref_rejected": -261.0951843261719,
      "logps/rejected": -261.05194091796875,
      "loss": 5.5434,
      "margin_dpo/margin_mean": 0.04740062355995178,
      "margin_dpo/margin_std": 0.7873871922492981,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.06531333923339844,
      "KL/mean": -0.011118605732917786,
      "KL/rejected_KL_mean": 0.04307365417480469,
      "KL/std": 0.6332308650016785,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.10837936401367188,
      "fcm_dpo/q_t": 0.5002709627151489,
      "grad_norm": 16.118513107299805,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": 1.9805262088775635,
      "logits/rejected": 1.8356199264526367,
      "logps/chosen": -288.6500244140625,
      "logps/ref_chosen": -288.584716796875,
      "logps/ref_rejected": -224.8094482421875,
      "logps/rejected": -224.766357421875,
      "loss": 5.5496,
      "margin_dpo/margin_mean": -0.10837952792644501,
      "margin_dpo/margin_std": 0.8592747449874878,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": 0.07653045654296875,
      "KL/mean": 0.0009353607892990112,
      "KL/rejected_KL_mean": -0.07465934753417969,
      "KL/std": 0.6243972182273865,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1511818766593933,
      "fcm_dpo/q_t": 0.49962204694747925,
      "grad_norm": 16.586254119873047,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": 2.0272176265716553,
      "logits/rejected": 1.888408899307251,
      "logps/chosen": -283.7666015625,
      "logps/ref_chosen": -283.8431091308594,
      "logps/ref_rejected": -248.63645935058594,
      "logps/rejected": -248.71112060546875,
      "loss": 5.5392,
      "margin_dpo/margin_mean": 0.15118207037448883,
      "margin_dpo/margin_std": 0.8825235366821289,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.0829315185546875,
      "KL/mean": 0.07783805578947067,
      "KL/rejected_KL_mean": 0.07274627685546875,
      "KL/std": 0.5340047478675842,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.010174766182899475,
      "fcm_dpo/q_t": 0.4999745488166809,
      "grad_norm": 15.227861404418945,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 1.9501663446426392,
      "logits/rejected": 1.9114961624145508,
      "logps/chosen": -254.5506591796875,
      "logps/ref_chosen": -254.6335906982422,
      "logps/ref_rejected": -245.3981170654297,
      "logps/rejected": -245.3253631591797,
      "loss": 5.5448,
      "margin_dpo/margin_mean": 0.010174959897994995,
      "margin_dpo/margin_std": 0.7053954005241394,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.039371490478515625,
      "KL/mean": -0.015219412744045258,
      "KL/rejected_KL_mean": 0.0089263916015625,
      "KL/std": 0.6349751353263855,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.04830075800418854,
      "fcm_dpo/q_t": 0.5001207590103149,
      "grad_norm": 15.106447219848633,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": 1.8998022079467773,
      "logits/rejected": 1.8124054670333862,
      "logps/chosen": -287.7889099121094,
      "logps/ref_chosen": -287.7495422363281,
      "logps/ref_rejected": -273.03363037109375,
      "logps/rejected": -273.02471923828125,
      "loss": 5.5472,
      "margin_dpo/margin_mean": -0.048301324248313904,
      "margin_dpo/margin_std": 0.8308672904968262,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.06242179870605469,
      "KL/mean": -0.04194725304841995,
      "KL/rejected_KL_mean": -0.021474838256835938,
      "KL/std": 0.5633232593536377,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.04094491899013519,
      "fcm_dpo/q_t": 0.5001024007797241,
      "grad_norm": 14.858816146850586,
      "learning_rate": 1.875e-07,
      "logits/chosen": 1.9225668907165527,
      "logits/rejected": 1.8749691247940063,
      "logps/chosen": -273.4663391113281,
      "logps/ref_chosen": -273.4039001464844,
      "logps/ref_rejected": -260.09722900390625,
      "logps/rejected": -260.11871337890625,
      "loss": 5.5469,
      "margin_dpo/margin_mean": -0.04094433784484863,
      "margin_dpo/margin_std": 0.7406395673751831,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.08871841430664062,
      "KL/mean": 0.030117250978946686,
      "KL/rejected_KL_mean": -0.028478622436523438,
      "KL/std": 0.6939102411270142,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11721085011959076,
      "fcm_dpo/q_t": 0.49970701336860657,
      "grad_norm": 15.507994651794434,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.9182145595550537,
      "logits/rejected": 1.8762524127960205,
      "logps/chosen": -292.8734130859375,
      "logps/ref_chosen": -292.96209716796875,
      "logps/ref_rejected": -256.602783203125,
      "logps/rejected": -256.6312561035156,
      "loss": 5.5406,
      "margin_dpo/margin_mean": 0.11721087992191315,
      "margin_dpo/margin_std": 0.8616542816162109,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.056758880615234375,
      "KL/mean": 0.059789396822452545,
      "KL/rejected_KL_mean": 0.06281089782714844,
      "KL/std": 0.6676266193389893,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0060462504625320435,
      "fcm_dpo/q_t": 0.5000150799751282,
      "grad_norm": 15.374666213989258,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 2.0961177349090576,
      "logits/rejected": 2.0322794914245605,
      "logps/chosen": -292.27044677734375,
      "logps/ref_chosen": -292.3271789550781,
      "logps/ref_rejected": -274.20635986328125,
      "logps/rejected": -274.1435852050781,
      "loss": 5.5455,
      "margin_dpo/margin_mean": -0.006047368049621582,
      "margin_dpo/margin_std": 0.9542068839073181,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.04469871520996094,
      "KL/mean": 0.04655793309211731,
      "KL/rejected_KL_mean": 0.04842185974121094,
      "KL/std": 0.5714669227600098,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.003717765212059021,
      "fcm_dpo/q_t": 0.500009298324585,
      "grad_norm": 13.768776893615723,
      "learning_rate": 2.1875e-07,
      "logits/chosen": 1.9535518884658813,
      "logits/rejected": 1.9568406343460083,
      "logps/chosen": -253.2215118408203,
      "logps/ref_chosen": -253.2662353515625,
      "logps/ref_rejected": -241.4681396484375,
      "logps/rejected": -241.41970825195312,
      "loss": 5.5454,
      "margin_dpo/margin_mean": -0.0037176012992858887,
      "margin_dpo/margin_std": 0.7891198992729187,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.11308860778808594,
      "KL/mean": 0.07142220437526703,
      "KL/rejected_KL_mean": 0.02974700927734375,
      "KL/std": 0.6001766324043274,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08333727717399597,
      "fcm_dpo/q_t": 0.49979162216186523,
      "grad_norm": 15.181371688842773,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": 1.969049334526062,
      "logits/rejected": 1.9529852867126465,
      "logps/chosen": -275.3085021972656,
      "logps/ref_chosen": -275.4216003417969,
      "logps/ref_rejected": -264.44024658203125,
      "logps/rejected": -264.4104919433594,
      "loss": 5.5419,
      "margin_dpo/margin_mean": 0.08333773910999298,
      "margin_dpo/margin_std": 0.8054537773132324,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": 0.06640434265136719,
      "KL/mean": 0.03843782842159271,
      "KL/rejected_KL_mean": 0.010469436645507812,
      "KL/std": 0.6087324619293213,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.055929094552993774,
      "fcm_dpo/q_t": 0.4998601973056793,
      "grad_norm": 16.473119735717773,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": 1.8471585512161255,
      "logits/rejected": 1.8741003274917603,
      "logps/chosen": -293.31121826171875,
      "logps/ref_chosen": -293.3775634765625,
      "logps/ref_rejected": -301.59014892578125,
      "logps/rejected": -301.5796813964844,
      "loss": 5.543,
      "margin_dpo/margin_mean": 0.05592915415763855,
      "margin_dpo/margin_std": 0.8231167197227478,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.052745819091796875,
      "KL/mean": 0.015244759619235992,
      "KL/rejected_KL_mean": -0.02225494384765625,
      "KL/std": 0.6187722086906433,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07499389350414276,
      "fcm_dpo/q_t": 0.4998124837875366,
      "grad_norm": 14.490097045898438,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.7165096998214722,
      "logits/rejected": 1.7513575553894043,
      "logps/chosen": -240.8809356689453,
      "logps/ref_chosen": -240.93368530273438,
      "logps/ref_rejected": -235.3230743408203,
      "logps/rejected": -235.34530639648438,
      "loss": 5.5422,
      "margin_dpo/margin_mean": 0.07499483227729797,
      "margin_dpo/margin_std": 0.7974440455436707,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": -0.023538589477539062,
      "KL/mean": -0.004726484417915344,
      "KL/rejected_KL_mean": 0.014087677001953125,
      "KL/std": 0.6440406441688538,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0376235693693161,
      "fcm_dpo/q_t": 0.5000940561294556,
      "grad_norm": 15.590790748596191,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": 1.7846139669418335,
      "logits/rejected": 1.7740511894226074,
      "logps/chosen": -273.61883544921875,
      "logps/ref_chosen": -273.5953369140625,
      "logps/ref_rejected": -234.49148559570312,
      "logps/rejected": -234.47738647460938,
      "loss": 5.5468,
      "margin_dpo/margin_mean": -0.037623971700668335,
      "margin_dpo/margin_std": 0.8705086708068848,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.047672271728515625,
      "KL/mean": -0.009357526898384094,
      "KL/rejected_KL_mean": -0.06637191772460938,
      "KL/std": 0.7004381418228149,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11403846740722656,
      "fcm_dpo/q_t": 0.4997149109840393,
      "grad_norm": 15.177017211914062,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": 2.147540807723999,
      "logits/rejected": 1.9023590087890625,
      "logps/chosen": -314.2161865234375,
      "logps/ref_chosen": -314.26385498046875,
      "logps/ref_rejected": -242.06735229492188,
      "logps/rejected": -242.13372802734375,
      "loss": 5.5407,
      "margin_dpo/margin_mean": 0.11403858661651611,
      "margin_dpo/margin_std": 0.9722203016281128,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.07826614379882812,
      "KL/mean": 0.038661859929561615,
      "KL/rejected_KL_mean": -0.0009479522705078125,
      "KL/std": 0.6057465076446533,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0792028158903122,
      "fcm_dpo/q_t": 0.49980202317237854,
      "grad_norm": 15.426494598388672,
      "learning_rate": 2.8125e-07,
      "logits/chosen": 1.9799132347106934,
      "logits/rejected": 1.9838168621063232,
      "logps/chosen": -240.66055297851562,
      "logps/ref_chosen": -240.7388153076172,
      "logps/ref_rejected": -237.8331298828125,
      "logps/rejected": -237.83409118652344,
      "loss": 5.5421,
      "margin_dpo/margin_mean": 0.07920265197753906,
      "margin_dpo/margin_std": 0.8622682094573975,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.061359405517578125,
      "KL/mean": 0.025824226438999176,
      "KL/rejected_KL_mean": -0.009702682495117188,
      "KL/std": 0.650516927242279,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07106535136699677,
      "fcm_dpo/q_t": 0.49982237815856934,
      "grad_norm": 14.703625679016113,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": 1.940491795539856,
      "logits/rejected": 1.8248564004898071,
      "logps/chosen": -307.7680358886719,
      "logps/ref_chosen": -307.82940673828125,
      "logps/ref_rejected": -281.8109130859375,
      "logps/rejected": -281.82061767578125,
      "loss": 5.5424,
      "margin_dpo/margin_mean": 0.07106417417526245,
      "margin_dpo/margin_std": 0.8769566416740417,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.046413421630859375,
      "KL/mean": 0.04285681992769241,
      "KL/rejected_KL_mean": 0.03928947448730469,
      "KL/std": 0.5640113949775696,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.007130876183509827,
      "fcm_dpo/q_t": 0.49998220801353455,
      "grad_norm": 14.423657417297363,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.6589934825897217,
      "logits/rejected": 1.6497961282730103,
      "logps/chosen": -247.9673614501953,
      "logps/ref_chosen": -248.01376342773438,
      "logps/ref_rejected": -259.0014953613281,
      "logps/rejected": -258.96221923828125,
      "loss": 5.545,
      "margin_dpo/margin_mean": 0.007129386067390442,
      "margin_dpo/margin_std": 0.8481414914131165,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.123779296875,
      "KL/mean": 0.05536259710788727,
      "KL/rejected_KL_mean": -0.013055801391601562,
      "KL/std": 0.611191987991333,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13683655858039856,
      "fcm_dpo/q_t": 0.49965789914131165,
      "grad_norm": 13.655344009399414,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 1.69005286693573,
      "logits/rejected": 1.6008719205856323,
      "logps/chosen": -239.93902587890625,
      "logps/ref_chosen": -240.0628204345703,
      "logps/ref_rejected": -201.91021728515625,
      "logps/rejected": -201.92324829101562,
      "loss": 5.5398,
      "margin_dpo/margin_mean": 0.13683582842350006,
      "margin_dpo/margin_std": 0.8205834031105042,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.049465179443359375,
      "KL/mean": 0.06679318845272064,
      "KL/rejected_KL_mean": 0.08411979675292969,
      "KL/std": 0.6807079911231995,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03465672582387924,
      "fcm_dpo/q_t": 0.5000866651535034,
      "grad_norm": 15.599594116210938,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": 1.7600914239883423,
      "logits/rejected": 1.6627109050750732,
      "logps/chosen": -294.082763671875,
      "logps/ref_chosen": -294.1322326660156,
      "logps/ref_rejected": -265.8150634765625,
      "logps/rejected": -265.73095703125,
      "loss": 5.5467,
      "margin_dpo/margin_mean": -0.034656181931495667,
      "margin_dpo/margin_std": 0.8770751953125,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.32059288024902344,
      "KL/mean": 0.2050504833459854,
      "KL/rejected_KL_mean": 0.08950042724609375,
      "KL/std": 0.7067633271217346,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.23109108209609985,
      "fcm_dpo/q_t": 0.4994223117828369,
      "grad_norm": 15.355238914489746,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 2.0744423866271973,
      "logits/rejected": 2.048499584197998,
      "logps/chosen": -275.3345031738281,
      "logps/ref_chosen": -275.6551208496094,
      "logps/ref_rejected": -253.5972137451172,
      "logps/rejected": -253.50770568847656,
      "loss": 5.536,
      "margin_dpo/margin_mean": 0.2310909926891327,
      "margin_dpo/margin_std": 0.9401529431343079,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.24072837829589844,
      "KL/mean": 0.14348666369915009,
      "KL/rejected_KL_mean": 0.046245574951171875,
      "KL/std": 0.7096943855285645,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1944766640663147,
      "fcm_dpo/q_t": 0.4995138347148895,
      "grad_norm": 17.689180374145508,
      "learning_rate": 3.4375e-07,
      "logits/chosen": 1.9848268032073975,
      "logits/rejected": 1.8665993213653564,
      "logps/chosen": -332.7294006347656,
      "logps/ref_chosen": -332.9701232910156,
      "logps/ref_rejected": -278.2213439941406,
      "logps/rejected": -278.17510986328125,
      "loss": 5.5375,
      "margin_dpo/margin_mean": 0.19447600841522217,
      "margin_dpo/margin_std": 0.9495550394058228,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.14238929748535156,
      "KL/mean": 0.15987856686115265,
      "KL/rejected_KL_mean": 0.1773662567138672,
      "KL/std": 0.7061426639556885,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.034973323345184326,
      "fcm_dpo/q_t": 0.5000874400138855,
      "grad_norm": 13.648781776428223,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": 1.8412927389144897,
      "logits/rejected": 1.9047892093658447,
      "logps/chosen": -258.2864990234375,
      "logps/ref_chosen": -258.42889404296875,
      "logps/ref_rejected": -259.5119934082031,
      "logps/rejected": -259.3345947265625,
      "loss": 5.5467,
      "margin_dpo/margin_mean": -0.03497309982776642,
      "margin_dpo/margin_std": 1.0409122705459595,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.3343372344970703,
      "KL/mean": 0.23328572511672974,
      "KL/rejected_KL_mean": 0.13223838806152344,
      "KL/std": 0.7229053974151611,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2021031677722931,
      "fcm_dpo/q_t": 0.4994947910308838,
      "grad_norm": 17.792436599731445,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": 1.9906575679779053,
      "logits/rejected": 1.9371637105941772,
      "logps/chosen": -290.2850341796875,
      "logps/ref_chosen": -290.6193542480469,
      "logps/ref_rejected": -280.2503967285156,
      "logps/rejected": -280.1181335449219,
      "loss": 5.5372,
      "margin_dpo/margin_mean": 0.20210357010364532,
      "margin_dpo/margin_std": 0.9947282671928406,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.2956409454345703,
      "KL/mean": 0.22801515460014343,
      "KL/rejected_KL_mean": 0.1603984832763672,
      "KL/std": 0.6972114443778992,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13524112105369568,
      "fcm_dpo/q_t": 0.4996618926525116,
      "grad_norm": 15.18211555480957,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.812025785446167,
      "logits/rejected": 1.7968189716339111,
      "logps/chosen": -303.27716064453125,
      "logps/ref_chosen": -303.57281494140625,
      "logps/ref_rejected": -294.56878662109375,
      "logps/rejected": -294.4083557128906,
      "loss": 5.5399,
      "margin_dpo/margin_mean": 0.13524065911769867,
      "margin_dpo/margin_std": 0.945915699005127,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.2816295623779297,
      "KL/mean": 0.1602761447429657,
      "KL/rejected_KL_mean": 0.038921356201171875,
      "KL/std": 0.7572365999221802,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.24270696938037872,
      "fcm_dpo/q_t": 0.4993932843208313,
      "grad_norm": 15.26474666595459,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": 2.0457522869110107,
      "logits/rejected": 2.1038782596588135,
      "logps/chosen": -272.2718505859375,
      "logps/ref_chosen": -272.5534973144531,
      "logps/ref_rejected": -266.9136047363281,
      "logps/rejected": -266.8747253417969,
      "loss": 5.5356,
      "margin_dpo/margin_mean": 0.242706760764122,
      "margin_dpo/margin_std": 1.0484888553619385,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 0.46538543701171875,
      "KL/mean": 0.25401902198791504,
      "KL/rejected_KL_mean": 0.042652130126953125,
      "KL/std": 0.7714985609054565,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4227225184440613,
      "fcm_dpo/q_t": 0.4989432394504547,
      "grad_norm": 16.058277130126953,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": 1.7938494682312012,
      "logits/rejected": 1.7296651601791382,
      "logps/chosen": -264.921142578125,
      "logps/ref_chosen": -265.38653564453125,
      "logps/ref_rejected": -244.34774780273438,
      "logps/rejected": -244.30511474609375,
      "loss": 5.5284,
      "margin_dpo/margin_mean": 0.42272236943244934,
      "margin_dpo/margin_std": 0.9183180928230286,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 0.5182533264160156,
      "KL/mean": 0.28440096974372864,
      "KL/rejected_KL_mean": 0.05056190490722656,
      "KL/std": 0.764484703540802,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.46768736839294434,
      "fcm_dpo/q_t": 0.49883079528808594,
      "grad_norm": 14.914390563964844,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.9236502647399902,
      "logits/rejected": 1.9350950717926025,
      "logps/chosen": -274.3755187988281,
      "logps/ref_chosen": -274.893798828125,
      "logps/ref_rejected": -244.3120880126953,
      "logps/rejected": -244.26153564453125,
      "loss": 5.5266,
      "margin_dpo/margin_mean": 0.46768683195114136,
      "margin_dpo/margin_std": 0.9942057132720947,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 0.39699554443359375,
      "KL/mean": 0.2800103425979614,
      "KL/rejected_KL_mean": 0.16302871704101562,
      "KL/std": 0.757789134979248,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2339601218700409,
      "fcm_dpo/q_t": 0.4994151294231415,
      "grad_norm": 15.456948280334473,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 1.9013357162475586,
      "logits/rejected": 1.8096816539764404,
      "logps/chosen": -284.4021301269531,
      "logps/ref_chosen": -284.7991027832031,
      "logps/ref_rejected": -248.8882598876953,
      "logps/rejected": -248.7252197265625,
      "loss": 5.5359,
      "margin_dpo/margin_mean": 0.23396100103855133,
      "margin_dpo/margin_std": 1.0052895545959473,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.43169403076171875,
      "KL/mean": 0.23350346088409424,
      "KL/rejected_KL_mean": 0.03531837463378906,
      "KL/std": 0.7829483151435852,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.39637160301208496,
      "fcm_dpo/q_t": 0.49900907278060913,
      "grad_norm": 14.570927619934082,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": 1.9198373556137085,
      "logits/rejected": 1.976628065109253,
      "logps/chosen": -245.40798950195312,
      "logps/ref_chosen": -245.83966064453125,
      "logps/ref_rejected": -259.2870788574219,
      "logps/rejected": -259.25177001953125,
      "loss": 5.5295,
      "margin_dpo/margin_mean": 0.39637261629104614,
      "margin_dpo/margin_std": 1.050824522972107,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.49402618408203125,
      "KL/mean": 0.2794777750968933,
      "KL/rejected_KL_mean": 0.06492424011230469,
      "KL/std": 0.8902381062507629,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.42910754680633545,
      "fcm_dpo/q_t": 0.4989272356033325,
      "grad_norm": 15.66744327545166,
      "learning_rate": 4.375e-07,
      "logits/chosen": 1.9905283451080322,
      "logits/rejected": 1.9901080131530762,
      "logps/chosen": -307.9035339355469,
      "logps/ref_chosen": -308.39752197265625,
      "logps/ref_rejected": -270.3797912597656,
      "logps/rejected": -270.31488037109375,
      "loss": 5.5282,
      "margin_dpo/margin_mean": 0.4291083812713623,
      "margin_dpo/margin_std": 1.0799281597137451,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 0.7445163726806641,
      "KL/mean": 0.4396844208240509,
      "KL/rejected_KL_mean": 0.13485336303710938,
      "KL/std": 0.9521983861923218,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6096630096435547,
      "fcm_dpo/q_t": 0.49847590923309326,
      "grad_norm": 15.745288848876953,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": 1.8464998006820679,
      "logits/rejected": 1.8452539443969727,
      "logps/chosen": -274.6171875,
      "logps/ref_chosen": -275.36175537109375,
      "logps/ref_rejected": -279.7540588378906,
      "logps/rejected": -279.6192321777344,
      "loss": 5.521,
      "margin_dpo/margin_mean": 0.6096642017364502,
      "margin_dpo/margin_std": 1.3053245544433594,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 0.7552051544189453,
      "KL/mean": 0.47546863555908203,
      "KL/rejected_KL_mean": 0.1957225799560547,
      "KL/std": 0.9753624200820923,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5594832897186279,
      "fcm_dpo/q_t": 0.49860140681266785,
      "grad_norm": 16.921863555908203,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": 1.930234670639038,
      "logits/rejected": 1.9144927263259888,
      "logps/chosen": -314.3650207519531,
      "logps/ref_chosen": -315.1202087402344,
      "logps/ref_rejected": -290.34210205078125,
      "logps/rejected": -290.1463623046875,
      "loss": 5.523,
      "margin_dpo/margin_mean": 0.5594825744628906,
      "margin_dpo/margin_std": 1.3245391845703125,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 0.7858734130859375,
      "KL/mean": 0.45731353759765625,
      "KL/rejected_KL_mean": 0.1287555694580078,
      "KL/std": 0.970177412033081,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6571179032325745,
      "fcm_dpo/q_t": 0.4983573257923126,
      "grad_norm": 16.49712562561035,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": 1.716583490371704,
      "logits/rejected": 1.7208372354507446,
      "logps/chosen": -292.15472412109375,
      "logps/ref_chosen": -292.94061279296875,
      "logps/ref_rejected": -264.0979309082031,
      "logps/rejected": -263.96917724609375,
      "loss": 5.5191,
      "margin_dpo/margin_mean": 0.6571168899536133,
      "margin_dpo/margin_std": 1.3130911588668823,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 0.6432647705078125,
      "KL/mean": 0.46273159980773926,
      "KL/rejected_KL_mean": 0.28220367431640625,
      "KL/std": 0.912684440612793,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.36105573177337646,
      "fcm_dpo/q_t": 0.49909740686416626,
      "grad_norm": 13.546757698059082,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": 1.7095186710357666,
      "logits/rejected": 1.8087146282196045,
      "logps/chosen": -254.20941162109375,
      "logps/ref_chosen": -254.85269165039062,
      "logps/ref_rejected": -255.88380432128906,
      "logps/rejected": -255.60159301757812,
      "loss": 5.5309,
      "margin_dpo/margin_mean": 0.3610554337501526,
      "margin_dpo/margin_std": 1.2991602420806885,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 0.7809944152832031,
      "KL/mean": 0.45296037197113037,
      "KL/rejected_KL_mean": 0.12491989135742188,
      "KL/std": 1.1147668361663818,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6560739278793335,
      "fcm_dpo/q_t": 0.4983599781990051,
      "grad_norm": 14.89731502532959,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": 1.802992582321167,
      "logits/rejected": 1.7823169231414795,
      "logps/chosen": -261.7267150878906,
      "logps/ref_chosen": -262.5077209472656,
      "logps/ref_rejected": -239.25320434570312,
      "logps/rejected": -239.1282958984375,
      "loss": 5.5192,
      "margin_dpo/margin_mean": 0.6560737490653992,
      "margin_dpo/margin_std": 1.4462041854858398,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.8695182800292969,
      "KL/mean": 0.6102737784385681,
      "KL/rejected_KL_mean": 0.351043701171875,
      "KL/std": 1.1386690139770508,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5184862613677979,
      "fcm_dpo/q_t": 0.49870386719703674,
      "grad_norm": 15.232952117919922,
      "learning_rate": 5e-07,
      "logits/chosen": 1.9480514526367188,
      "logits/rejected": 2.0625219345092773,
      "logps/chosen": -248.6583709716797,
      "logps/ref_chosen": -249.5278778076172,
      "logps/ref_rejected": -254.6465301513672,
      "logps/rejected": -254.2954864501953,
      "loss": 5.5247,
      "margin_dpo/margin_mean": 0.518486738204956,
      "margin_dpo/margin_std": 1.5239760875701904,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 0.8807907104492188,
      "KL/mean": 0.698351263999939,
      "KL/rejected_KL_mean": 0.5159015655517578,
      "KL/std": 1.1846559047698975,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3648926019668579,
      "fcm_dpo/q_t": 0.4990878701210022,
      "grad_norm": 14.618494987487793,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 2.0543301105499268,
      "logits/rejected": 2.0709121227264404,
      "logps/chosen": -291.62091064453125,
      "logps/ref_chosen": -292.501708984375,
      "logps/ref_rejected": -294.6496887207031,
      "logps/rejected": -294.1337890625,
      "loss": 5.5309,
      "margin_dpo/margin_mean": 0.36489278078079224,
      "margin_dpo/margin_std": 1.5042037963867188,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 0.9076118469238281,
      "KL/mean": 0.5931621789932251,
      "KL/rejected_KL_mean": 0.2787055969238281,
      "KL/std": 1.3319183588027954,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6289072632789612,
      "fcm_dpo/q_t": 0.4984278678894043,
      "grad_norm": 14.733009338378906,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 1.6045843362808228,
      "logits/rejected": 1.7285107374191284,
      "logps/chosen": -258.95428466796875,
      "logps/ref_chosen": -259.8619079589844,
      "logps/ref_rejected": -293.1994934082031,
      "logps/rejected": -292.9207763671875,
      "loss": 5.5204,
      "margin_dpo/margin_mean": 0.628907322883606,
      "margin_dpo/margin_std": 1.7746456861495972,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 1.2445945739746094,
      "KL/mean": 0.7644231915473938,
      "KL/rejected_KL_mean": 0.28426170349121094,
      "KL/std": 1.388979196548462,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9603267312049866,
      "fcm_dpo/q_t": 0.4975995123386383,
      "grad_norm": 17.049270629882812,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": 1.9132016897201538,
      "logits/rejected": 1.8282064199447632,
      "logps/chosen": -297.18072509765625,
      "logps/ref_chosen": -298.42529296875,
      "logps/ref_rejected": -278.5128479003906,
      "logps/rejected": -278.2285461425781,
      "loss": 5.5072,
      "margin_dpo/margin_mean": 0.9603263139724731,
      "margin_dpo/margin_std": 1.839072823524475,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 1.2587718963623047,
      "KL/mean": 0.8521550297737122,
      "KL/rejected_KL_mean": 0.4455394744873047,
      "KL/std": 1.4988524913787842,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.813229501247406,
      "fcm_dpo/q_t": 0.4979671537876129,
      "grad_norm": 16.070592880249023,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": 2.019869089126587,
      "logits/rejected": 2.0961806774139404,
      "logps/chosen": -274.04974365234375,
      "logps/ref_chosen": -275.3085021972656,
      "logps/ref_rejected": -267.97381591796875,
      "logps/rejected": -267.5282897949219,
      "loss": 5.5132,
      "margin_dpo/margin_mean": 0.8132294416427612,
      "margin_dpo/margin_std": 1.9238917827606201,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 1.3440914154052734,
      "KL/mean": 0.9746026992797852,
      "KL/rejected_KL_mean": 0.6051082611083984,
      "KL/std": 1.4383636713027954,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7389852404594421,
      "fcm_dpo/q_t": 0.49815279245376587,
      "grad_norm": 14.566572189331055,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": 1.6468708515167236,
      "logits/rejected": 1.6692531108856201,
      "logps/chosen": -296.8032531738281,
      "logps/ref_chosen": -298.1473388671875,
      "logps/ref_rejected": -269.7539367675781,
      "logps/rejected": -269.1488342285156,
      "loss": 5.5161,
      "margin_dpo/margin_mean": 0.7389855980873108,
      "margin_dpo/margin_std": 1.9537945985794067,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.1441574096679688,
      "KL/mean": 0.8570212125778198,
      "KL/rejected_KL_mean": 0.56988525390625,
      "KL/std": 1.3473942279815674,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5742765665054321,
      "fcm_dpo/q_t": 0.4985644817352295,
      "grad_norm": 12.822190284729004,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": 1.8121758699417114,
      "logits/rejected": 1.8737857341766357,
      "logps/chosen": -215.60580444335938,
      "logps/ref_chosen": -216.74998474121094,
      "logps/ref_rejected": -202.7234649658203,
      "logps/rejected": -202.15357971191406,
      "loss": 5.5226,
      "margin_dpo/margin_mean": 0.5742762088775635,
      "margin_dpo/margin_std": 1.8632638454437256,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 1.2345409393310547,
      "KL/mean": 0.7433028817176819,
      "KL/rejected_KL_mean": 0.2520618438720703,
      "KL/std": 1.853980541229248,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9824758172035217,
      "fcm_dpo/q_t": 0.4975442588329315,
      "grad_norm": 14.349050521850586,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": 1.4617174863815308,
      "logits/rejected": 1.541257381439209,
      "logps/chosen": -242.4568634033203,
      "logps/ref_chosen": -243.69140625,
      "logps/ref_rejected": -234.970947265625,
      "logps/rejected": -234.71890258789062,
      "loss": 5.5066,
      "margin_dpo/margin_mean": 0.9824749231338501,
      "margin_dpo/margin_std": 2.3652021884918213,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 1.6827316284179688,
      "KL/mean": 1.0160224437713623,
      "KL/rejected_KL_mean": 0.3493175506591797,
      "KL/std": 1.6125212907791138,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3333920240402222,
      "fcm_dpo/q_t": 0.4966672956943512,
      "grad_norm": 17.681446075439453,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": 2.0503153800964355,
      "logits/rejected": 1.949772834777832,
      "logps/chosen": -309.0635986328125,
      "logps/ref_chosen": -310.746337890625,
      "logps/ref_rejected": -270.6407165527344,
      "logps/rejected": -270.2913818359375,
      "loss": 5.4925,
      "margin_dpo/margin_mean": 1.3333923816680908,
      "margin_dpo/margin_std": 1.9355008602142334,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 1.3630256652832031,
      "KL/mean": 0.8502845764160156,
      "KL/rejected_KL_mean": 0.337554931640625,
      "KL/std": 1.5676177740097046,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0254621505737305,
      "fcm_dpo/q_t": 0.49743643403053284,
      "grad_norm": 14.814376831054688,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": 1.743753433227539,
      "logits/rejected": 1.7290273904800415,
      "logps/chosen": -257.4499816894531,
      "logps/ref_chosen": -258.8130187988281,
      "logps/ref_rejected": -268.2101745605469,
      "logps/rejected": -267.87261962890625,
      "loss": 5.5047,
      "margin_dpo/margin_mean": 1.0254621505737305,
      "margin_dpo/margin_std": 1.871786117553711,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 1.507598876953125,
      "KL/mean": 1.0538583993911743,
      "KL/rejected_KL_mean": 0.6001186370849609,
      "KL/std": 1.9316776990890503,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9074810147285461,
      "fcm_dpo/q_t": 0.49773189425468445,
      "grad_norm": 15.281980514526367,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": 1.7941328287124634,
      "logits/rejected": 1.7350484132766724,
      "logps/chosen": -275.7338562011719,
      "logps/ref_chosen": -277.2414245605469,
      "logps/ref_rejected": -259.13519287109375,
      "logps/rejected": -258.5350646972656,
      "loss": 5.5096,
      "margin_dpo/margin_mean": 0.9074807167053223,
      "margin_dpo/margin_std": 2.465378761291504,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 1.5077857971191406,
      "KL/mean": 0.8772637844085693,
      "KL/rejected_KL_mean": 0.2467479705810547,
      "KL/std": 1.6710270643234253,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2610355615615845,
      "fcm_dpo/q_t": 0.49684783816337585,
      "grad_norm": 14.677238464355469,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.947865605354309,
      "logits/rejected": 1.997636079788208,
      "logps/chosen": -249.83807373046875,
      "logps/ref_chosen": -251.34585571289062,
      "logps/ref_rejected": -251.4120635986328,
      "logps/rejected": -251.16531372070312,
      "loss": 5.4954,
      "margin_dpo/margin_mean": 1.2610344886779785,
      "margin_dpo/margin_std": 2.177244186401367,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 1.6120624542236328,
      "KL/mean": 1.0927889347076416,
      "KL/rejected_KL_mean": 0.5735206604003906,
      "KL/std": 1.7748773097991943,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.038541555404663,
      "fcm_dpo/q_t": 0.4974041283130646,
      "grad_norm": 14.993729591369629,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 1.933851957321167,
      "logits/rejected": 1.9019213914871216,
      "logps/chosen": -267.37054443359375,
      "logps/ref_chosen": -268.98260498046875,
      "logps/ref_rejected": -249.34474182128906,
      "logps/rejected": -248.77122497558594,
      "loss": 5.5044,
      "margin_dpo/margin_mean": 1.0385409593582153,
      "margin_dpo/margin_std": 2.4141643047332764,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 2.024576187133789,
      "KL/mean": 1.4300038814544678,
      "KL/rejected_KL_mean": 0.8354225158691406,
      "KL/std": 1.9847910404205322,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1891523599624634,
      "fcm_dpo/q_t": 0.49702784419059753,
      "grad_norm": 14.668143272399902,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 1.9193660020828247,
      "logits/rejected": 1.8926868438720703,
      "logps/chosen": -262.1789245605469,
      "logps/ref_chosen": -264.2034912109375,
      "logps/ref_rejected": -260.2911376953125,
      "logps/rejected": -259.45574951171875,
      "loss": 5.4984,
      "margin_dpo/margin_mean": 1.1891520023345947,
      "margin_dpo/margin_std": 2.5558016300201416,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 2.0036964416503906,
      "KL/mean": 1.294305682182312,
      "KL/rejected_KL_mean": 0.5849056243896484,
      "KL/std": 2.1132876873016357,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4187986850738525,
      "fcm_dpo/q_t": 0.4964539110660553,
      "grad_norm": 14.727814674377441,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": 1.755799651145935,
      "logits/rejected": 1.7799817323684692,
      "logps/chosen": -269.464599609375,
      "logps/ref_chosen": -271.4682922363281,
      "logps/ref_rejected": -273.0804748535156,
      "logps/rejected": -272.4955749511719,
      "loss": 5.4895,
      "margin_dpo/margin_mean": 1.4187986850738525,
      "margin_dpo/margin_std": 2.764784336090088,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 2.358095169067383,
      "KL/mean": 1.6028822660446167,
      "KL/rejected_KL_mean": 0.8476657867431641,
      "KL/std": 2.3603696823120117,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5104295015335083,
      "fcm_dpo/q_t": 0.4962249994277954,
      "grad_norm": 15.721498489379883,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": 2.0388362407684326,
      "logits/rejected": 2.0650696754455566,
      "logps/chosen": -271.24530029296875,
      "logps/ref_chosen": -273.6033935546875,
      "logps/ref_rejected": -259.54107666015625,
      "logps/rejected": -258.6933898925781,
      "loss": 5.4859,
      "margin_dpo/margin_mean": 1.5104296207427979,
      "margin_dpo/margin_std": 2.9949536323547363,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 2.345813751220703,
      "KL/mean": 1.752155065536499,
      "KL/rejected_KL_mean": 1.1584949493408203,
      "KL/std": 2.083946704864502,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1873126029968262,
      "fcm_dpo/q_t": 0.49703219532966614,
      "grad_norm": 14.50374984741211,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": 1.7700073719024658,
      "logits/rejected": 1.870046854019165,
      "logps/chosen": -256.1718444824219,
      "logps/ref_chosen": -258.5176696777344,
      "logps/ref_rejected": -264.9944763183594,
      "logps/rejected": -263.8359375,
      "loss": 5.4986,
      "margin_dpo/margin_mean": 1.187312126159668,
      "margin_dpo/margin_std": 2.6377952098846436,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 2.217815399169922,
      "KL/mean": 1.5300347805023193,
      "KL/rejected_KL_mean": 0.842254638671875,
      "KL/std": 2.7106220722198486,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.375559687614441,
      "fcm_dpo/q_t": 0.49656224250793457,
      "grad_norm": 14.87004280090332,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": 1.9038126468658447,
      "logits/rejected": 2.0036251544952393,
      "logps/chosen": -265.2915954589844,
      "logps/ref_chosen": -267.5094299316406,
      "logps/ref_rejected": -277.03228759765625,
      "logps/rejected": -276.19000244140625,
      "loss": 5.4917,
      "margin_dpo/margin_mean": 1.3755590915679932,
      "margin_dpo/margin_std": 3.5484609603881836,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 2.571012496948242,
      "KL/mean": 1.9833064079284668,
      "KL/rejected_KL_mean": 1.3956069946289062,
      "KL/std": 2.6359827518463135,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1753946542739868,
      "fcm_dpo/q_t": 0.49706214666366577,
      "grad_norm": 14.235478401184082,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": 2.0069072246551514,
      "logits/rejected": 2.0557613372802734,
      "logps/chosen": -254.8939208984375,
      "logps/ref_chosen": -257.4649353027344,
      "logps/ref_rejected": -247.29161071777344,
      "logps/rejected": -245.89599609375,
      "loss": 5.4995,
      "margin_dpo/margin_mean": 1.1753945350646973,
      "margin_dpo/margin_std": 3.3209216594696045,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 2.4951705932617188,
      "KL/mean": 1.6340646743774414,
      "KL/rejected_KL_mean": 0.7729721069335938,
      "KL/std": 2.9358785152435303,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7221958637237549,
      "fcm_dpo/q_t": 0.49569690227508545,
      "grad_norm": 14.7554292678833,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": 1.8177176713943481,
      "logits/rejected": 1.775955319404602,
      "logps/chosen": -287.989013671875,
      "logps/ref_chosen": -290.4841613769531,
      "logps/ref_rejected": -260.8398132324219,
      "logps/rejected": -260.06683349609375,
      "loss": 5.4782,
      "margin_dpo/margin_mean": 1.722196340560913,
      "margin_dpo/margin_std": 3.892500877380371,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 2.8399925231933594,
      "KL/mean": 1.7768393754959106,
      "KL/rejected_KL_mean": 0.7136898040771484,
      "KL/std": 2.962188243865967,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.126312732696533,
      "fcm_dpo/q_t": 0.4946866035461426,
      "grad_norm": 14.992707252502441,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": 1.871492624282837,
      "logits/rejected": 1.9134466648101807,
      "logps/chosen": -254.79759216308594,
      "logps/ref_chosen": -257.6375732421875,
      "logps/ref_rejected": -235.6666717529297,
      "logps/rejected": -234.9529571533203,
      "loss": 5.4621,
      "margin_dpo/margin_mean": 2.1263129711151123,
      "margin_dpo/margin_std": 3.730478286743164,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 3.4998607635498047,
      "KL/mean": 2.476236343383789,
      "KL/rejected_KL_mean": 1.4526119232177734,
      "KL/std": 2.9685850143432617,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.047244071960449,
      "fcm_dpo/q_t": 0.4948844015598297,
      "grad_norm": 15.591229438781738,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.6660312414169312,
      "logits/rejected": 1.6143622398376465,
      "logps/chosen": -283.96533203125,
      "logps/ref_chosen": -287.4651794433594,
      "logps/ref_rejected": -253.4770965576172,
      "logps/rejected": -252.02447509765625,
      "loss": 5.4652,
      "margin_dpo/margin_mean": 2.0472443103790283,
      "margin_dpo/margin_std": 3.7040200233459473,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 3.5522403717041016,
      "KL/mean": 2.571442127227783,
      "KL/rejected_KL_mean": 1.5906448364257812,
      "KL/std": 3.334211826324463,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9616049528121948,
      "fcm_dpo/q_t": 0.49509790539741516,
      "grad_norm": 15.035438537597656,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 2.096977472305298,
      "logits/rejected": 2.117103099822998,
      "logps/chosen": -277.3831481933594,
      "logps/ref_chosen": -280.9353942871094,
      "logps/ref_rejected": -275.998779296875,
      "logps/rejected": -274.40814208984375,
      "loss": 5.4693,
      "margin_dpo/margin_mean": 1.9616044759750366,
      "margin_dpo/margin_std": 4.359718322753906,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 3.114969253540039,
      "KL/mean": 2.1368677616119385,
      "KL/rejected_KL_mean": 1.158761978149414,
      "KL/std": 3.0561962127685547,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.956218957901001,
      "fcm_dpo/q_t": 0.495111882686615,
      "grad_norm": 14.539937019348145,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": 1.689402461051941,
      "logits/rejected": 1.8292930126190186,
      "logps/chosen": -227.0305633544922,
      "logps/ref_chosen": -230.14553833007812,
      "logps/ref_rejected": -253.37661743164062,
      "logps/rejected": -252.2178497314453,
      "loss": 5.4691,
      "margin_dpo/margin_mean": 1.9562186002731323,
      "margin_dpo/margin_std": 3.97003436088562,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 3.8152542114257812,
      "KL/mean": 2.8659894466400146,
      "KL/rejected_KL_mean": 1.9167251586914062,
      "KL/std": 3.1900479793548584,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8985309600830078,
      "fcm_dpo/q_t": 0.49525532126426697,
      "grad_norm": 14.805742263793945,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": 1.7801011800765991,
      "logits/rejected": 1.8064188957214355,
      "logps/chosen": -261.051025390625,
      "logps/ref_chosen": -264.86627197265625,
      "logps/ref_rejected": -245.21304321289062,
      "logps/rejected": -243.29629516601562,
      "loss": 5.4718,
      "margin_dpo/margin_mean": 1.898528814315796,
      "margin_dpo/margin_std": 4.477626323699951,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.8101558685302734,
      "KL/mean": 2.8084115982055664,
      "KL/rejected_KL_mean": 1.8066883087158203,
      "KL/std": 3.5353262424468994,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0034737586975098,
      "fcm_dpo/q_t": 0.4949941635131836,
      "grad_norm": 15.688196182250977,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": 1.9111007452011108,
      "logits/rejected": 1.968408465385437,
      "logps/chosen": -330.74334716796875,
      "logps/ref_chosen": -334.55352783203125,
      "logps/ref_rejected": -325.8403015136719,
      "logps/rejected": -324.0335998535156,
      "loss": 5.4678,
      "margin_dpo/margin_mean": 2.003473997116089,
      "margin_dpo/margin_std": 4.66332483291626,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.709741592407227,
      "KL/mean": 3.327197551727295,
      "KL/rejected_KL_mean": 1.9446601867675781,
      "KL/std": 3.642332077026367,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.765089273452759,
      "fcm_dpo/q_t": 0.49309101700782776,
      "grad_norm": 15.30482006072998,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 1.7007465362548828,
      "logits/rejected": 1.7792117595672607,
      "logps/chosen": -255.71212768554688,
      "logps/ref_chosen": -260.421875,
      "logps/ref_rejected": -273.0882263183594,
      "logps/rejected": -271.1435546875,
      "loss": 5.4373,
      "margin_dpo/margin_mean": 2.7650890350341797,
      "margin_dpo/margin_std": 4.180358409881592,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.594623565673828,
      "KL/mean": 3.079987049102783,
      "KL/rejected_KL_mean": 1.565359115600586,
      "KL/std": 3.858675718307495,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0292677879333496,
      "fcm_dpo/q_t": 0.49243438243865967,
      "grad_norm": 16.762033462524414,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": 1.8418477773666382,
      "logits/rejected": 1.8303914070129395,
      "logps/chosen": -267.1408386230469,
      "logps/ref_chosen": -271.7354431152344,
      "logps/ref_rejected": -262.64111328125,
      "logps/rejected": -261.0757751464844,
      "loss": 5.4275,
      "margin_dpo/margin_mean": 3.029268264770508,
      "margin_dpo/margin_std": 4.843067646026611,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 4.901239395141602,
      "KL/mean": 3.7020132541656494,
      "KL/rejected_KL_mean": 2.5027923583984375,
      "KL/std": 4.1426167488098145,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.3984453678131104,
      "fcm_dpo/q_t": 0.49400976300239563,
      "grad_norm": 14.852890014648438,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": 1.7040801048278809,
      "logits/rejected": 1.5405802726745605,
      "logps/chosen": -289.7693786621094,
      "logps/ref_chosen": -294.6706237792969,
      "logps/ref_rejected": -247.6900177001953,
      "logps/rejected": -245.18724060058594,
      "loss": 5.4527,
      "margin_dpo/margin_mean": 2.3984451293945312,
      "margin_dpo/margin_std": 5.141509056091309,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 5.213813781738281,
      "KL/mean": 3.7372069358825684,
      "KL/rejected_KL_mean": 2.2605953216552734,
      "KL/std": 3.7120089530944824,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9532151222229004,
      "fcm_dpo/q_t": 0.49262458086013794,
      "grad_norm": 14.660135269165039,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": 1.5285110473632812,
      "logits/rejected": 1.4464118480682373,
      "logps/chosen": -242.9384765625,
      "logps/ref_chosen": -248.1522979736328,
      "logps/ref_rejected": -209.12808227539062,
      "logps/rejected": -206.86746215820312,
      "loss": 5.4308,
      "margin_dpo/margin_mean": 2.953214645385742,
      "margin_dpo/margin_std": 5.300847053527832,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 5.060859680175781,
      "KL/mean": 3.544020891189575,
      "KL/rejected_KL_mean": 2.0271778106689453,
      "KL/std": 4.7413787841796875,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0336763858795166,
      "fcm_dpo/q_t": 0.4924156069755554,
      "grad_norm": 16.732967376708984,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": 1.7439801692962646,
      "logits/rejected": 1.8042641878128052,
      "logps/chosen": -254.41351318359375,
      "logps/ref_chosen": -259.4743957519531,
      "logps/ref_rejected": -246.9627227783203,
      "logps/rejected": -244.935546875,
      "loss": 5.4293,
      "margin_dpo/margin_mean": 3.0336766242980957,
      "margin_dpo/margin_std": 6.18861198425293,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 5.379819869995117,
      "KL/mean": 4.1106181144714355,
      "KL/rejected_KL_mean": 2.841419219970703,
      "KL/std": 4.762851238250732,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.538395404815674,
      "fcm_dpo/q_t": 0.49366065859794617,
      "grad_norm": 15.637837409973145,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.6808106899261475,
      "logits/rejected": 1.67099130153656,
      "logps/chosen": -280.5467224121094,
      "logps/ref_chosen": -285.926513671875,
      "logps/ref_rejected": -253.77996826171875,
      "logps/rejected": -250.93853759765625,
      "loss": 5.4482,
      "margin_dpo/margin_mean": 2.538395881652832,
      "margin_dpo/margin_std": 5.732854843139648,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 6.135602951049805,
      "KL/mean": 4.615637302398682,
      "KL/rejected_KL_mean": 3.0956687927246094,
      "KL/std": 4.523761749267578,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0399281978607178,
      "fcm_dpo/q_t": 0.4924063980579376,
      "grad_norm": 14.624700546264648,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 1.8814154863357544,
      "logits/rejected": 1.8342338800430298,
      "logps/chosen": -256.4585876464844,
      "logps/ref_chosen": -262.5942077636719,
      "logps/ref_rejected": -236.687744140625,
      "logps/rejected": -233.59205627441406,
      "loss": 5.4289,
      "margin_dpo/margin_mean": 3.0399296283721924,
      "margin_dpo/margin_std": 6.298126697540283,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 6.595163345336914,
      "KL/mean": 5.005348205566406,
      "KL/rejected_KL_mean": 3.4155235290527344,
      "KL/std": 4.428990364074707,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.179635524749756,
      "fcm_dpo/q_t": 0.49206018447875977,
      "grad_norm": 23.096643447875977,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": 1.6704881191253662,
      "logits/rejected": 1.6059318780899048,
      "logps/chosen": -264.4480285644531,
      "logps/ref_chosen": -271.04315185546875,
      "logps/ref_rejected": -234.03359985351562,
      "logps/rejected": -230.61805725097656,
      "loss": 5.4227,
      "margin_dpo/margin_mean": 3.179635763168335,
      "margin_dpo/margin_std": 5.662868976593018,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 6.678592681884766,
      "KL/mean": 4.903674125671387,
      "KL/rejected_KL_mean": 3.1287574768066406,
      "KL/std": 4.919029235839844,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.549840211868286,
      "fcm_dpo/q_t": 0.49114057421684265,
      "grad_norm": 15.477609634399414,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": 1.7081586122512817,
      "logits/rejected": 1.7834362983703613,
      "logps/chosen": -259.606689453125,
      "logps/ref_chosen": -266.2852783203125,
      "logps/ref_rejected": -274.32501220703125,
      "logps/rejected": -271.1962585449219,
      "loss": 5.4093,
      "margin_dpo/margin_mean": 3.5498406887054443,
      "margin_dpo/margin_std": 6.594154357910156,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 6.769054412841797,
      "KL/mean": 5.367814064025879,
      "KL/rejected_KL_mean": 3.966571807861328,
      "KL/std": 5.099339008331299,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.8024837970733643,
      "fcm_dpo/q_t": 0.49299943447113037,
      "grad_norm": 15.240982055664062,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": 1.8319710493087769,
      "logits/rejected": 1.817328929901123,
      "logps/chosen": -280.01153564453125,
      "logps/ref_chosen": -286.7806091308594,
      "logps/ref_rejected": -276.1176452636719,
      "logps/rejected": -272.1510925292969,
      "loss": 5.4387,
      "margin_dpo/margin_mean": 2.8024826049804688,
      "margin_dpo/margin_std": 6.560458183288574,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": 6.43632698059082,
      "KL/mean": 5.406087398529053,
      "KL/rejected_KL_mean": 4.375846862792969,
      "KL/std": 4.390769958496094,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0604827404022217,
      "fcm_dpo/q_t": 0.4948526620864868,
      "grad_norm": 13.862303733825684,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": 2.0082762241363525,
      "logits/rejected": 2.1088311672210693,
      "logps/chosen": -228.5935516357422,
      "logps/ref_chosen": -235.02987670898438,
      "logps/ref_rejected": -259.80999755859375,
      "logps/rejected": -255.43414306640625,
      "loss": 5.4665,
      "margin_dpo/margin_mean": 2.0604825019836426,
      "margin_dpo/margin_std": 5.3985161781311035,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 8.020017623901367,
      "KL/mean": 6.192572593688965,
      "KL/rejected_KL_mean": 4.365131378173828,
      "KL/std": 5.513957500457764,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6548943519592285,
      "fcm_dpo/q_t": 0.49087703227996826,
      "grad_norm": 16.413917541503906,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": 1.8734852075576782,
      "logits/rejected": 1.9172875881195068,
      "logps/chosen": -293.292724609375,
      "logps/ref_chosen": -301.312744140625,
      "logps/ref_rejected": -288.8368225097656,
      "logps/rejected": -284.4717102050781,
      "loss": 5.4059,
      "margin_dpo/margin_mean": 3.654893398284912,
      "margin_dpo/margin_std": 7.294094085693359,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": 8.108610153198242,
      "KL/mean": 6.100605010986328,
      "KL/rejected_KL_mean": 4.09259033203125,
      "KL/std": 5.477041721343994,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.016026973724365,
      "fcm_dpo/q_t": 0.48997408151626587,
      "grad_norm": 15.631966590881348,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": 1.7246675491333008,
      "logits/rejected": 1.7117444276809692,
      "logps/chosen": -257.557373046875,
      "logps/ref_chosen": -265.6659851074219,
      "logps/ref_rejected": -242.0697479248047,
      "logps/rejected": -237.9771728515625,
      "loss": 5.3906,
      "margin_dpo/margin_mean": 4.016026020050049,
      "margin_dpo/margin_std": 6.420953750610352,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": 7.46299934387207,
      "KL/mean": 5.795078277587891,
      "KL/rejected_KL_mean": 4.1271514892578125,
      "KL/std": 5.567847728729248,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.335850477218628,
      "fcm_dpo/q_t": 0.4916718304157257,
      "grad_norm": 14.06159782409668,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 1.8771694898605347,
      "logits/rejected": 1.8978253602981567,
      "logps/chosen": -253.2584228515625,
      "logps/ref_chosen": -260.721435546875,
      "logps/ref_rejected": -262.6593322753906,
      "logps/rejected": -258.53216552734375,
      "loss": 5.4183,
      "margin_dpo/margin_mean": 3.3358514308929443,
      "margin_dpo/margin_std": 7.126031398773193,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": 9.026639938354492,
      "KL/mean": 7.1607818603515625,
      "KL/rejected_KL_mean": 5.294918060302734,
      "KL/std": 5.2332305908203125,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.7317252159118652,
      "fcm_dpo/q_t": 0.49068379402160645,
      "grad_norm": 15.804368019104004,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": 1.7459875345230103,
      "logits/rejected": 1.7041118144989014,
      "logps/chosen": -247.97616577148438,
      "logps/ref_chosen": -257.0028076171875,
      "logps/ref_rejected": -254.66232299804688,
      "logps/rejected": -249.36740112304688,
      "loss": 5.4019,
      "margin_dpo/margin_mean": 3.73172664642334,
      "margin_dpo/margin_std": 6.3766326904296875,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": 8.992788314819336,
      "KL/mean": 7.286672115325928,
      "KL/rejected_KL_mean": 5.580558776855469,
      "KL/std": 5.608501434326172,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.4122276306152344,
      "fcm_dpo/q_t": 0.4914812743663788,
      "grad_norm": 15.299113273620605,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 1.9598567485809326,
      "logits/rejected": 1.8996613025665283,
      "logps/chosen": -289.4176025390625,
      "logps/ref_chosen": -298.4103698730469,
      "logps/ref_rejected": -262.8761901855469,
      "logps/rejected": -257.2956237792969,
      "loss": 5.4153,
      "margin_dpo/margin_mean": 3.4122273921966553,
      "margin_dpo/margin_std": 6.826263904571533,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": 8.806732177734375,
      "KL/mean": 7.108377456665039,
      "KL/rejected_KL_mean": 5.410028457641602,
      "KL/std": 5.4552764892578125,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.396704912185669,
      "fcm_dpo/q_t": 0.4915269613265991,
      "grad_norm": 14.700748443603516,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 1.7251824140548706,
      "logits/rejected": 1.775238037109375,
      "logps/chosen": -218.7171630859375,
      "logps/ref_chosen": -227.5238800048828,
      "logps/ref_rejected": -227.11428833007812,
      "logps/rejected": -221.70428466796875,
      "loss": 5.4164,
      "margin_dpo/margin_mean": 3.39670467376709,
      "margin_dpo/margin_std": 7.423587799072266,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": 9.019119262695312,
      "KL/mean": 7.11264705657959,
      "KL/rejected_KL_mean": 5.206157684326172,
      "KL/std": 6.641966819763184,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.8129661083221436,
      "fcm_dpo/q_t": 0.49047887325286865,
      "grad_norm": 15.664932250976562,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": 1.7011375427246094,
      "logits/rejected": 1.5961568355560303,
      "logps/chosen": -266.36309814453125,
      "logps/ref_chosen": -275.3822326660156,
      "logps/ref_rejected": -269.8356018066406,
      "logps/rejected": -264.6294250488281,
      "loss": 5.4005,
      "margin_dpo/margin_mean": 3.8129656314849854,
      "margin_dpo/margin_std": 7.670473098754883,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": 9.188753128051758,
      "KL/mean": 7.471936225891113,
      "KL/rejected_KL_mean": 5.755125045776367,
      "KL/std": 5.749417781829834,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.4336280822753906,
      "fcm_dpo/q_t": 0.49143195152282715,
      "grad_norm": 14.223604202270508,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": 1.7469688653945923,
      "logits/rejected": 1.7304763793945312,
      "logps/chosen": -218.92762756347656,
      "logps/ref_chosen": -228.1163787841797,
      "logps/ref_rejected": -228.26266479492188,
      "logps/rejected": -222.5075225830078,
      "loss": 5.4147,
      "margin_dpo/margin_mean": 3.4336280822753906,
      "margin_dpo/margin_std": 7.219976425170898,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": 10.055526733398438,
      "KL/mean": 8.01603889465332,
      "KL/rejected_KL_mean": 5.976541519165039,
      "KL/std": 6.612032413482666,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.07898473739624,
      "fcm_dpo/q_t": 0.4898287057876587,
      "grad_norm": 15.77200698852539,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": 1.6795659065246582,
      "logits/rejected": 1.7661185264587402,
      "logps/chosen": -273.9717102050781,
      "logps/ref_chosen": -284.0272521972656,
      "logps/ref_rejected": -282.1720275878906,
      "logps/rejected": -276.19549560546875,
      "loss": 5.3918,
      "margin_dpo/margin_mean": 4.078984260559082,
      "margin_dpo/margin_std": 8.588692665100098,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": 9.84528923034668,
      "KL/mean": 7.606139183044434,
      "KL/rejected_KL_mean": 5.36700439453125,
      "KL/std": 6.857881546020508,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.47829532623291,
      "fcm_dpo/q_t": 0.48883333802223206,
      "grad_norm": 16.487751007080078,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": 1.9858918190002441,
      "logits/rejected": 1.9323406219482422,
      "logps/chosen": -293.20452880859375,
      "logps/ref_chosen": -303.0498352050781,
      "logps/ref_rejected": -259.8980712890625,
      "logps/rejected": -254.53106689453125,
      "loss": 5.3763,
      "margin_dpo/margin_mean": 4.4782938957214355,
      "margin_dpo/margin_std": 8.843856811523438,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": 9.963457107543945,
      "KL/mean": 7.810983180999756,
      "KL/rejected_KL_mean": 5.658514022827148,
      "KL/std": 7.03044319152832,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.3049421310424805,
      "fcm_dpo/q_t": 0.48926281929016113,
      "grad_norm": 15.241086959838867,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": 1.8710383176803589,
      "logits/rejected": 1.8339159488677979,
      "logps/chosen": -260.62420654296875,
      "logps/ref_chosen": -270.58770751953125,
      "logps/ref_rejected": -238.90069580078125,
      "logps/rejected": -233.24217224121094,
      "loss": 5.3817,
      "margin_dpo/margin_mean": 4.3049421310424805,
      "margin_dpo/margin_std": 7.829098701477051,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": 9.859138488769531,
      "KL/mean": 7.532392501831055,
      "KL/rejected_KL_mean": 5.2056427001953125,
      "KL/std": 6.977463245391846,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.6534953117370605,
      "fcm_dpo/q_t": 0.48840954899787903,
      "grad_norm": 15.30223560333252,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": 1.8220527172088623,
      "logits/rejected": 1.704922080039978,
      "logps/chosen": -286.326416015625,
      "logps/ref_chosen": -296.1855773925781,
      "logps/ref_rejected": -260.35333251953125,
      "logps/rejected": -255.14768981933594,
      "loss": 5.3712,
      "margin_dpo/margin_mean": 4.653496742248535,
      "margin_dpo/margin_std": 9.727502822875977,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": 9.971080780029297,
      "KL/mean": 7.620296478271484,
      "KL/rejected_KL_mean": 5.269496917724609,
      "KL/std": 7.350746154785156,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.7015838623046875,
      "fcm_dpo/q_t": 0.48828279972076416,
      "grad_norm": 15.79086971282959,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": 1.9368873834609985,
      "logits/rejected": 1.921276330947876,
      "logps/chosen": -267.7442626953125,
      "logps/ref_chosen": -277.7153625488281,
      "logps/ref_rejected": -272.9094543457031,
      "logps/rejected": -267.63995361328125,
      "loss": 5.3697,
      "margin_dpo/margin_mean": 4.7015838623046875,
      "margin_dpo/margin_std": 9.355917930603027,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": 10.231681823730469,
      "KL/mean": 7.705739974975586,
      "KL/rejected_KL_mean": 5.17979621887207,
      "KL/std": 7.573766231536865,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.051895618438721,
      "fcm_dpo/q_t": 0.4873978793621063,
      "grad_norm": 15.055027961730957,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": 1.7941193580627441,
      "logits/rejected": 1.769832730293274,
      "logps/chosen": -262.30615234375,
      "logps/ref_chosen": -272.5378112792969,
      "logps/ref_rejected": -244.6329345703125,
      "logps/rejected": -239.453125,
      "loss": 5.3568,
      "margin_dpo/margin_mean": 5.0518951416015625,
      "margin_dpo/margin_std": 9.940389633178711,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": 9.139177322387695,
      "KL/mean": 6.967242240905762,
      "KL/rejected_KL_mean": 4.795314788818359,
      "KL/std": 7.800963401794434,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.343876361846924,
      "fcm_dpo/q_t": 0.4891747832298279,
      "grad_norm": 15.533707618713379,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.6988040208816528,
      "logits/rejected": 1.620392918586731,
      "logps/chosen": -280.6731872558594,
      "logps/ref_chosen": -289.8123474121094,
      "logps/ref_rejected": -274.29071044921875,
      "logps/rejected": -269.495361328125,
      "loss": 5.3825,
      "margin_dpo/margin_mean": 4.343875408172607,
      "margin_dpo/margin_std": 9.019431114196777,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": 9.824851989746094,
      "KL/mean": 7.59401273727417,
      "KL/rejected_KL_mean": 5.3631744384765625,
      "KL/std": 8.047196388244629,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.461677074432373,
      "fcm_dpo/q_t": 0.4888765215873718,
      "grad_norm": 15.399192810058594,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 1.9640792608261108,
      "logits/rejected": 2.141399621963501,
      "logps/chosen": -235.88827514648438,
      "logps/ref_chosen": -245.71311950683594,
      "logps/ref_rejected": -272.08935546875,
      "logps/rejected": -266.72613525390625,
      "loss": 5.3794,
      "margin_dpo/margin_mean": 4.461676597595215,
      "margin_dpo/margin_std": 9.704748153686523,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": 10.271535873413086,
      "KL/mean": 7.072397708892822,
      "KL/rejected_KL_mean": 3.8732681274414062,
      "KL/std": 7.590062141418457,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.39826774597168,
      "fcm_dpo/q_t": 0.4840470254421234,
      "grad_norm": 15.14384651184082,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": 1.4451887607574463,
      "logits/rejected": 1.5297844409942627,
      "logps/chosen": -199.67086791992188,
      "logps/ref_chosen": -209.94239807128906,
      "logps/ref_rejected": -230.66897583007812,
      "logps/rejected": -226.7957000732422,
      "loss": 5.3025,
      "margin_dpo/margin_mean": 6.39826774597168,
      "margin_dpo/margin_std": 9.019288063049316,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": 9.280250549316406,
      "KL/mean": 6.859302520751953,
      "KL/rejected_KL_mean": 4.438350677490234,
      "KL/std": 8.272806167602539,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.841897487640381,
      "fcm_dpo/q_t": 0.48794078826904297,
      "grad_norm": 16.991683959960938,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": 1.725380539894104,
      "logits/rejected": 1.8218398094177246,
      "logps/chosen": -242.01937866210938,
      "logps/ref_chosen": -251.2996368408203,
      "logps/ref_rejected": -277.4847412109375,
      "logps/rejected": -273.04638671875,
      "loss": 5.3653,
      "margin_dpo/margin_mean": 4.841897010803223,
      "margin_dpo/margin_std": 10.320094108581543,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": 8.160594940185547,
      "KL/mean": 5.960206985473633,
      "KL/rejected_KL_mean": 3.7598190307617188,
      "KL/std": 8.57548999786377,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.400773048400879,
      "fcm_dpo/q_t": 0.48904260993003845,
      "grad_norm": 14.266998291015625,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": 1.7321412563323975,
      "logits/rejected": 1.767950415611267,
      "logps/chosen": -249.61883544921875,
      "logps/ref_chosen": -257.7794189453125,
      "logps/ref_rejected": -227.91546630859375,
      "logps/rejected": -224.15562438964844,
      "loss": 5.3843,
      "margin_dpo/margin_mean": 4.400773048400879,
      "margin_dpo/margin_std": 11.072494506835938,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": 10.045347213745117,
      "KL/mean": 6.001959323883057,
      "KL/rejected_KL_mean": 1.9585838317871094,
      "KL/std": 9.596831321716309,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.086767196655273,
      "fcm_dpo/q_t": 0.4798516035079956,
      "grad_norm": 17.808961868286133,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": 1.8605296611785889,
      "logits/rejected": 1.8361819982528687,
      "logps/chosen": -302.0486755371094,
      "logps/ref_chosen": -312.09405517578125,
      "logps/ref_rejected": -277.97332763671875,
      "logps/rejected": -276.0147399902344,
      "loss": 5.2408,
      "margin_dpo/margin_mean": 8.086766242980957,
      "margin_dpo/margin_std": 10.653196334838867,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": 6.672573089599609,
      "KL/mean": 3.9637837409973145,
      "KL/rejected_KL_mean": 1.2549915313720703,
      "KL/std": 8.845263481140137,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.417587757110596,
      "fcm_dpo/q_t": 0.48651206493377686,
      "grad_norm": 16.4842586517334,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": 1.9164067506790161,
      "logits/rejected": 1.9461092948913574,
      "logps/chosen": -278.6937561035156,
      "logps/ref_chosen": -285.3663330078125,
      "logps/ref_rejected": -262.2779541015625,
      "logps/rejected": -261.0229797363281,
      "loss": 5.3447,
      "margin_dpo/margin_mean": 5.417588233947754,
      "margin_dpo/margin_std": 11.044485092163086,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": 9.305768966674805,
      "KL/mean": 5.678952217102051,
      "KL/rejected_KL_mean": 2.052143096923828,
      "KL/std": 9.531133651733398,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.253628730773926,
      "fcm_dpo/q_t": 0.48192140460014343,
      "grad_norm": 16.524959564208984,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": 1.778128743171692,
      "logits/rejected": 1.845292568206787,
      "logps/chosen": -286.9878234863281,
      "logps/ref_chosen": -296.2935791015625,
      "logps/ref_rejected": -291.5162353515625,
      "logps/rejected": -289.4640808105469,
      "loss": 5.2754,
      "margin_dpo/margin_mean": 7.253628730773926,
      "margin_dpo/margin_std": 11.589075088500977,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": 9.363056182861328,
      "KL/mean": 5.334782123565674,
      "KL/rejected_KL_mean": 1.3065013885498047,
      "KL/std": 8.97293472290039,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.056559562683105,
      "fcm_dpo/q_t": 0.4799349009990692,
      "grad_norm": 16.403430938720703,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": 1.8729196786880493,
      "logits/rejected": 1.7378572225570679,
      "logps/chosen": -278.76129150390625,
      "logps/ref_chosen": -288.12432861328125,
      "logps/ref_rejected": -232.35720825195312,
      "logps/rejected": -231.0507049560547,
      "loss": 5.2414,
      "margin_dpo/margin_mean": 8.056561470031738,
      "margin_dpo/margin_std": 10.58915901184082,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": 8.138481140136719,
      "KL/mean": 3.93196439743042,
      "KL/rejected_KL_mean": -0.27454185485839844,
      "KL/std": 10.138080596923828,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.413019180297852,
      "fcm_dpo/q_t": 0.47910526394844055,
      "grad_norm": 17.586462020874023,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": 1.651432991027832,
      "logits/rejected": 1.5929906368255615,
      "logps/chosen": -309.0630798339844,
      "logps/ref_chosen": -317.2015686035156,
      "logps/ref_rejected": -278.0653381347656,
      "logps/rejected": -278.3398742675781,
      "loss": 5.2339,
      "margin_dpo/margin_mean": 8.413018226623535,
      "margin_dpo/margin_std": 12.87901782989502,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": 8.643138885498047,
      "KL/mean": 4.566476821899414,
      "KL/rejected_KL_mean": 0.4898262023925781,
      "KL/std": 10.238203048706055,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.153326034545898,
      "fcm_dpo/q_t": 0.4797130525112152,
      "grad_norm": 15.910147666931152,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.752532958984375,
      "logits/rejected": 1.7450892925262451,
      "logps/chosen": -254.29632568359375,
      "logps/ref_chosen": -262.939453125,
      "logps/ref_rejected": -257.82275390625,
      "logps/rejected": -257.33294677734375,
      "loss": 5.2443,
      "margin_dpo/margin_mean": 8.153325080871582,
      "margin_dpo/margin_std": 13.266733169555664,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": 9.03776741027832,
      "KL/mean": 4.698492050170898,
      "KL/rejected_KL_mean": 0.3592243194580078,
      "KL/std": 11.267974853515625,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.678547859191895,
      "fcm_dpo/q_t": 0.4784431457519531,
      "grad_norm": 22.158884048461914,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 1.878027081489563,
      "logits/rejected": 1.7822016477584839,
      "logps/chosen": -310.1160583496094,
      "logps/ref_chosen": -319.15380859375,
      "logps/ref_rejected": -281.7733154296875,
      "logps/rejected": -281.4140930175781,
      "loss": 5.2243,
      "margin_dpo/margin_mean": 8.678547859191895,
      "margin_dpo/margin_std": 12.926002502441406,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": 5.553424835205078,
      "KL/mean": 3.3843395709991455,
      "KL/rejected_KL_mean": 1.2152595520019531,
      "KL/std": 10.540499687194824,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.338167190551758,
      "fcm_dpo/q_t": 0.48920130729675293,
      "grad_norm": 14.623066902160645,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": 1.8860262632369995,
      "logits/rejected": 1.9955151081085205,
      "logps/chosen": -244.94699096679688,
      "logps/ref_chosen": -250.50042724609375,
      "logps/ref_rejected": -268.32635498046875,
      "logps/rejected": -267.1111145019531,
      "loss": 5.3967,
      "margin_dpo/margin_mean": 4.338167667388916,
      "margin_dpo/margin_std": 14.122950553894043,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": 7.117481231689453,
      "KL/mean": 1.966789722442627,
      "KL/rejected_KL_mean": -3.183897018432617,
      "KL/std": 11.726834297180176,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.301370620727539,
      "fcm_dpo/q_t": 0.47443023324012756,
      "grad_norm": 16.72608757019043,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": 1.6522220373153687,
      "logits/rejected": 1.6938014030456543,
      "logps/chosen": -272.2840881347656,
      "logps/ref_chosen": -279.4015808105469,
      "logps/ref_rejected": -266.6392822265625,
      "logps/rejected": -269.82318115234375,
      "loss": 5.166,
      "margin_dpo/margin_mean": 10.301371574401855,
      "margin_dpo/margin_std": 14.282323837280273,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": 5.630266189575195,
      "KL/mean": 3.0384325981140137,
      "KL/rejected_KL_mean": 0.44660186767578125,
      "KL/std": 10.77761173248291,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.183665752410889,
      "fcm_dpo/q_t": 0.48711520433425903,
      "grad_norm": 16.24945831298828,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 1.751715898513794,
      "logits/rejected": 1.739471197128296,
      "logps/chosen": -248.63021850585938,
      "logps/ref_chosen": -254.26048278808594,
      "logps/ref_rejected": -241.8769989013672,
      "logps/rejected": -241.43038940429688,
      "loss": 5.3594,
      "margin_dpo/margin_mean": 5.183666706085205,
      "margin_dpo/margin_std": 13.078446388244629,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": 5.760259628295898,
      "KL/mean": 2.913747549057007,
      "KL/rejected_KL_mean": 0.06722831726074219,
      "KL/std": 11.830049514770508,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.693040370941162,
      "fcm_dpo/q_t": 0.4858887493610382,
      "grad_norm": 15.864185333251953,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": 1.7434039115905762,
      "logits/rejected": 1.7795276641845703,
      "logps/chosen": -283.04425048828125,
      "logps/ref_chosen": -288.80450439453125,
      "logps/ref_rejected": -266.8573913574219,
      "logps/rejected": -266.7901611328125,
      "loss": 5.3433,
      "margin_dpo/margin_mean": 5.693039894104004,
      "margin_dpo/margin_std": 14.277177810668945,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": 7.341268539428711,
      "KL/mean": 3.1930267810821533,
      "KL/rejected_KL_mean": -0.9552173614501953,
      "KL/std": 11.180510520935059,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.296485900878906,
      "fcm_dpo/q_t": 0.47938302159309387,
      "grad_norm": 16.39175796508789,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": 1.8342329263687134,
      "logits/rejected": 1.8243844509124756,
      "logps/chosen": -264.84869384765625,
      "logps/ref_chosen": -272.18994140625,
      "logps/ref_rejected": -253.14334106445312,
      "logps/rejected": -254.0985565185547,
      "loss": 5.2412,
      "margin_dpo/margin_mean": 8.296486854553223,
      "margin_dpo/margin_std": 14.03947639465332,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": 5.858909606933594,
      "KL/mean": 1.6260355710983276,
      "KL/rejected_KL_mean": -2.606830596923828,
      "KL/std": 11.082222938537598,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.465743064880371,
      "fcm_dpo/q_t": 0.47896048426628113,
      "grad_norm": 15.620043754577637,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": 1.5449641942977905,
      "logits/rejected": 1.5270919799804688,
      "logps/chosen": -244.54373168945312,
      "logps/ref_chosen": -250.4026336669922,
      "logps/ref_rejected": -229.63853454589844,
      "logps/rejected": -232.245361328125,
      "loss": 5.2328,
      "margin_dpo/margin_mean": 8.465743064880371,
      "margin_dpo/margin_std": 12.538251876831055,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": 5.659423828125,
      "KL/mean": 1.1056039333343506,
      "KL/rejected_KL_mean": -3.4482250213623047,
      "KL/std": 12.776625633239746,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.107654571533203,
      "fcm_dpo/q_t": 0.4773828387260437,
      "grad_norm": 20.122955322265625,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": 1.870786428451538,
      "logits/rejected": 1.926609992980957,
      "logps/chosen": -252.73646545410156,
      "logps/ref_chosen": -258.3959045410156,
      "logps/ref_rejected": -268.84228515625,
      "logps/rejected": -272.29052734375,
      "loss": 5.2123,
      "margin_dpo/margin_mean": 9.107654571533203,
      "margin_dpo/margin_std": 14.583890914916992,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": 6.027412414550781,
      "KL/mean": 2.161963701248169,
      "KL/rejected_KL_mean": -1.7034797668457031,
      "KL/std": 14.188530921936035,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.730893611907959,
      "fcm_dpo/q_t": 0.48090851306915283,
      "grad_norm": 16.109636306762695,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": 1.9719607830047607,
      "logits/rejected": 1.953966498374939,
      "logps/chosen": -311.275146484375,
      "logps/ref_chosen": -317.30255126953125,
      "logps/ref_rejected": -300.47552490234375,
      "logps/rejected": -302.17901611328125,
      "loss": 5.2753,
      "margin_dpo/margin_mean": 7.730894088745117,
      "margin_dpo/margin_std": 16.836528778076172,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": 6.968473434448242,
      "KL/mean": 2.421053409576416,
      "KL/rejected_KL_mean": -2.126375198364258,
      "KL/std": 13.444896697998047,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.094856262207031,
      "fcm_dpo/q_t": 0.47745728492736816,
      "grad_norm": 16.302446365356445,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.7116224765777588,
      "logits/rejected": 1.6488630771636963,
      "logps/chosen": -298.28594970703125,
      "logps/ref_chosen": -305.2544250488281,
      "logps/ref_rejected": -259.40716552734375,
      "logps/rejected": -261.53350830078125,
      "loss": 5.2193,
      "margin_dpo/margin_mean": 9.094856262207031,
      "margin_dpo/margin_std": 16.476913452148438,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": 3.3260116577148438,
      "KL/mean": 0.6451948285102844,
      "KL/rejected_KL_mean": -2.0356197357177734,
      "KL/std": 11.869659423828125,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.361635684967041,
      "fcm_dpo/q_t": 0.4866945445537567,
      "grad_norm": 14.756305694580078,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 1.7257933616638184,
      "logits/rejected": 1.7493102550506592,
      "logps/chosen": -213.31471252441406,
      "logps/ref_chosen": -216.64073181152344,
      "logps/ref_rejected": -224.36422729492188,
      "logps/rejected": -226.3998260498047,
      "loss": 5.357,
      "margin_dpo/margin_mean": 5.361634731292725,
      "margin_dpo/margin_std": 14.490612030029297,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": 5.52252197265625,
      "KL/mean": 1.8193567991256714,
      "KL/rejected_KL_mean": -1.8838062286376953,
      "KL/std": 13.971630096435547,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.406329154968262,
      "fcm_dpo/q_t": 0.481655478477478,
      "grad_norm": 15.87812614440918,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": 1.8213753700256348,
      "logits/rejected": 1.813562035560608,
      "logps/chosen": -263.2962646484375,
      "logps/ref_chosen": -268.8188171386719,
      "logps/ref_rejected": -272.55853271484375,
      "logps/rejected": -274.44232177734375,
      "loss": 5.2875,
      "margin_dpo/margin_mean": 7.406330108642578,
      "margin_dpo/margin_std": 17.444034576416016,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": 8.357036590576172,
      "KL/mean": 3.0691592693328857,
      "KL/rejected_KL_mean": -2.218709945678711,
      "KL/std": 15.100142478942871,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.575753211975098,
      "fcm_dpo/q_t": 0.473831444978714,
      "grad_norm": 18.272907257080078,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": 1.6535494327545166,
      "logits/rejected": 1.6237690448760986,
      "logps/chosen": -313.3346862792969,
      "logps/ref_chosen": -321.6917419433594,
      "logps/ref_rejected": -279.1517333984375,
      "logps/rejected": -281.37042236328125,
      "loss": 5.1685,
      "margin_dpo/margin_mean": 10.575752258300781,
      "margin_dpo/margin_std": 18.36194610595703,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": 5.898250579833984,
      "KL/mean": 1.1165496110916138,
      "KL/rejected_KL_mean": -3.6651601791381836,
      "KL/std": 14.463729858398438,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.563407897949219,
      "fcm_dpo/q_t": 0.4762840270996094,
      "grad_norm": 17.110431671142578,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": 1.7093058824539185,
      "logits/rejected": 1.5980933904647827,
      "logps/chosen": -265.7769470214844,
      "logps/ref_chosen": -271.6752014160156,
      "logps/ref_rejected": -221.85577392578125,
      "logps/rejected": -225.5209197998047,
      "loss": 5.2018,
      "margin_dpo/margin_mean": 9.563409805297852,
      "margin_dpo/margin_std": 15.997897148132324,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": 5.240959167480469,
      "KL/mean": 0.558391809463501,
      "KL/rejected_KL_mean": -4.124174118041992,
      "KL/std": 14.12118148803711,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.365134239196777,
      "fcm_dpo/q_t": 0.47686082124710083,
      "grad_norm": 16.643329620361328,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": 1.65989351272583,
      "logits/rejected": 1.6648253202438354,
      "logps/chosen": -265.1581115722656,
      "logps/ref_chosen": -270.3990478515625,
      "logps/ref_rejected": -264.16925048828125,
      "logps/rejected": -268.2934265136719,
      "loss": 5.2154,
      "margin_dpo/margin_mean": 9.365134239196777,
      "margin_dpo/margin_std": 18.52533531188965,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": 6.661823272705078,
      "KL/mean": 1.308668851852417,
      "KL/rejected_KL_mean": -4.044473648071289,
      "KL/std": 15.620298385620117,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.010251020081341267,
      "fcm_dpo/delta": 0.03535527363419533,
      "fcm_dpo/margin": 10.706303596496582,
      "fcm_dpo/q_t": 0.47302713990211487,
      "grad_norm": 17.13081932067871,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": 1.8886208534240723,
      "logits/rejected": 1.8825353384017944,
      "logps/chosen": -306.5894775390625,
      "logps/ref_chosen": -313.25128173828125,
      "logps/ref_rejected": -283.0433349609375,
      "logps/rejected": -287.0877990722656,
      "loss": 5.1564,
      "margin_dpo/margin_mean": 10.706303596496582,
      "margin_dpo/margin_std": 18.218219757080078,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": 7.900579452514648,
      "KL/mean": 1.7361433506011963,
      "KL/rejected_KL_mean": -4.428289413452148,
      "KL/std": 16.04119873046875,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.010472334921360016,
      "fcm_dpo/delta": 0.06991249322891235,
      "fcm_dpo/margin": 12.32887077331543,
      "fcm_dpo/q_t": 0.4681806266307831,
      "grad_norm": 17.500600814819336,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 1.536065936088562,
      "logits/rejected": 1.4557569026947021,
      "logps/chosen": -290.6634216308594,
      "logps/ref_chosen": -298.5639953613281,
      "logps/ref_rejected": -274.021728515625,
      "logps/rejected": -278.4500427246094,
      "loss": 5.0813,
      "margin_dpo/margin_mean": 12.32887077331543,
      "margin_dpo/margin_std": 18.11617660522461,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": 8.637063980102539,
      "KL/mean": 1.8465750217437744,
      "KL/rejected_KL_mean": -4.943904876708984,
      "KL/std": 16.910566329956055,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.011113150976598263,
      "fcm_dpo/delta": 0.03542490303516388,
      "fcm_dpo/margin": 13.580964088439941,
      "fcm_dpo/q_t": 0.4629313051700592,
      "grad_norm": 17.703550338745117,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": 1.600803017616272,
      "logits/rejected": 1.6278514862060547,
      "logps/chosen": -248.4938201904297,
      "logps/ref_chosen": -257.130859375,
      "logps/ref_rejected": -247.71636962890625,
      "logps/rejected": -252.66026306152344,
      "loss": 5.0155,
      "margin_dpo/margin_mean": 13.580965995788574,
      "margin_dpo/margin_std": 20.05784797668457,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": 8.987005233764648,
      "KL/mean": 2.725919485092163,
      "KL/rejected_KL_mean": -3.535158157348633,
      "KL/std": 17.008638381958008,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.01150730811059475,
      "fcm_dpo/delta": 0.07001318037509918,
      "fcm_dpo/margin": 12.522164344787598,
      "fcm_dpo/q_t": 0.4648369550704956,
      "grad_norm": 18.61189842224121,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": 1.7158299684524536,
      "logits/rejected": 1.76116943359375,
      "logps/chosen": -285.9735412597656,
      "logps/ref_chosen": -294.9605712890625,
      "logps/ref_rejected": -272.6249694824219,
      "logps/rejected": -276.16009521484375,
      "loss": 5.0475,
      "margin_dpo/margin_mean": 12.522164344787598,
      "margin_dpo/margin_std": 20.175939559936523,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": 7.618045806884766,
      "KL/mean": 1.7802190780639648,
      "KL/rejected_KL_mean": -4.057613372802734,
      "KL/std": 15.515937805175781,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.011836046352982521,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.6756591796875,
      "fcm_dpo/q_t": 0.4659108519554138,
      "grad_norm": 23.802391052246094,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 1.7260011434555054,
      "logits/rejected": 1.8043789863586426,
      "logps/chosen": -268.82415771484375,
      "logps/ref_chosen": -276.4422302246094,
      "logps/ref_rejected": -290.241455078125,
      "logps/rejected": -294.299072265625,
      "loss": 5.0654,
      "margin_dpo/margin_mean": 11.675660133361816,
      "margin_dpo/margin_std": 19.317903518676758,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": 5.080375671386719,
      "KL/mean": 0.2767190635204315,
      "KL/rejected_KL_mean": -4.526937484741211,
      "KL/std": 16.416540145874023,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.01203650701791048,
      "fcm_dpo/delta": 0.033421795815229416,
      "fcm_dpo/margin": 9.607320785522461,
      "fcm_dpo/q_t": 0.4718432128429413,
      "grad_norm": 21.017688751220703,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 1.6537426710128784,
      "logits/rejected": 1.651482105255127,
      "logps/chosen": -299.74432373046875,
      "logps/ref_chosen": -304.82470703125,
      "logps/ref_rejected": -289.9726867675781,
      "logps/rejected": -294.4996032714844,
      "loss": 5.1647,
      "margin_dpo/margin_mean": 9.607321739196777,
      "margin_dpo/margin_std": 21.135969161987305,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": 7.366617202758789,
      "KL/mean": 0.8674134016036987,
      "KL/rejected_KL_mean": -5.631786346435547,
      "KL/std": 18.43401336669922,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.012294160202145576,
      "fcm_dpo/delta": 0.05249980837106705,
      "fcm_dpo/margin": 12.998409271240234,
      "fcm_dpo/q_t": 0.4614144563674927,
      "grad_norm": 21.39548683166504,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": 1.6357831954956055,
      "logits/rejected": 1.746757984161377,
      "logps/chosen": -252.9481201171875,
      "logps/ref_chosen": -260.31475830078125,
      "logps/ref_rejected": -280.93756103515625,
      "logps/rejected": -286.5693359375,
      "loss": 5.0193,
      "margin_dpo/margin_mean": 12.998411178588867,
      "margin_dpo/margin_std": 23.298921585083008,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": 7.972652435302734,
      "KL/mean": 1.2752999067306519,
      "KL/rejected_KL_mean": -5.4220428466796875,
      "KL/std": 17.183929443359375,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.012874508276581764,
      "fcm_dpo/delta": 0.058184750378131866,
      "fcm_dpo/margin": 13.394699096679688,
      "fcm_dpo/q_t": 0.4578048884868622,
      "grad_norm": 20.56849479675293,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": 1.6824402809143066,
      "logits/rejected": 1.6613080501556396,
      "logps/chosen": -271.14794921875,
      "logps/ref_chosen": -279.12060546875,
      "logps/ref_rejected": -247.22982788085938,
      "logps/rejected": -252.65187072753906,
      "loss": 4.963,
      "margin_dpo/margin_mean": 13.394698143005371,
      "margin_dpo/margin_std": 20.926292419433594,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": 1.9481639862060547,
      "KL/mean": -2.798513412475586,
      "KL/rejected_KL_mean": -7.545194625854492,
      "KL/std": 17.501066207885742,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.013685951009392738,
      "fcm_dpo/delta": 0.06518751382827759,
      "fcm_dpo/margin": 9.493376731872559,
      "fcm_dpo/q_t": 0.46848607063293457,
      "grad_norm": 27.851367950439453,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": 1.5203371047973633,
      "logits/rejected": 1.53583824634552,
      "logps/chosen": -313.65594482421875,
      "logps/ref_chosen": -315.6040954589844,
      "logps/ref_rejected": -291.45635986328125,
      "logps/rejected": -299.00152587890625,
      "loss": 5.1214,
      "margin_dpo/margin_mean": 9.493375778198242,
      "margin_dpo/margin_std": 20.15289878845215,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": 5.479560852050781,
      "KL/mean": -1.124311923980713,
      "KL/rejected_KL_mean": -7.728189468383789,
      "KL/std": 17.948070526123047,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.014848006889224052,
      "fcm_dpo/delta": 0.11190246790647507,
      "fcm_dpo/margin": 13.207749366760254,
      "fcm_dpo/q_t": 0.4529009759426117,
      "grad_norm": 23.821683883666992,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": 1.5347298383712769,
      "logits/rejected": 1.527937412261963,
      "logps/chosen": -277.3934326171875,
      "logps/ref_chosen": -282.87298583984375,
      "logps/ref_rejected": -265.0682373046875,
      "logps/rejected": -272.7964172363281,
      "loss": 4.8921,
      "margin_dpo/margin_mean": 13.207748413085938,
      "margin_dpo/margin_std": 19.194381713867188,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": 4.679023742675781,
      "KL/mean": -1.5391241312026978,
      "KL/rejected_KL_mean": -7.75727653503418,
      "KL/std": 17.520055770874023,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.01642434485256672,
      "fcm_dpo/delta": 0.11292678117752075,
      "fcm_dpo/margin": 12.43630599975586,
      "fcm_dpo/q_t": 0.4515303671360016,
      "grad_norm": 30.452056884765625,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": 1.4680678844451904,
      "logits/rejected": 1.6315441131591797,
      "logps/chosen": -258.76287841796875,
      "logps/ref_chosen": -263.4419250488281,
      "logps/ref_rejected": -251.43150329589844,
      "logps/rejected": -259.18878173828125,
      "loss": 4.9083,
      "margin_dpo/margin_mean": 12.436306953430176,
      "margin_dpo/margin_std": 21.772756576538086,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": 2.791048049926758,
      "KL/mean": -2.849806070327759,
      "KL/rejected_KL_mean": -8.49066162109375,
      "KL/std": 19.127317428588867,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.017353549599647522,
      "fcm_dpo/delta": 0.10800304263830185,
      "fcm_dpo/margin": 11.281705856323242,
      "fcm_dpo/q_t": 0.45332586765289307,
      "grad_norm": 29.059709548950195,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": 1.7510448694229126,
      "logits/rejected": 1.746030569076538,
      "logps/chosen": -292.4797058105469,
      "logps/ref_chosen": -295.2707214355469,
      "logps/ref_rejected": -283.9851989746094,
      "logps/rejected": -292.47589111328125,
      "loss": 4.9368,
      "margin_dpo/margin_mean": 11.281705856323242,
      "margin_dpo/margin_std": 20.73297882080078,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": 1.0942840576171875,
      "KL/mean": -4.434977054595947,
      "KL/rejected_KL_mean": -9.964242935180664,
      "KL/std": 18.45460319519043,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.018433703109622,
      "fcm_dpo/delta": 0.039801668375730515,
      "fcm_dpo/margin": 11.058513641357422,
      "fcm_dpo/q_t": 0.4522782564163208,
      "grad_norm": 29.17181968688965,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": 1.5447523593902588,
      "logits/rejected": 1.5383518934249878,
      "logps/chosen": -281.4635925292969,
      "logps/ref_chosen": -282.5578918457031,
      "logps/ref_rejected": -262.4772644042969,
      "logps/rejected": -272.4415283203125,
      "loss": 4.9691,
      "margin_dpo/margin_mean": 11.058513641357422,
      "margin_dpo/margin_std": 23.558799743652344,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": 2.1814727783203125,
      "KL/mean": -1.8737328052520752,
      "KL/rejected_KL_mean": -5.928947448730469,
      "KL/std": 19.142053604125977,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.01949838176369667,
      "fcm_dpo/delta": 0.05981515347957611,
      "fcm_dpo/margin": 8.110417366027832,
      "fcm_dpo/q_t": 0.46302950382232666,
      "grad_norm": 32.37385559082031,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": 1.5196315050125122,
      "logits/rejected": 1.5927009582519531,
      "logps/chosen": -261.4693298339844,
      "logps/ref_chosen": -263.65081787109375,
      "logps/ref_rejected": -266.2420959472656,
      "logps/rejected": -272.1710205078125,
      "loss": 5.1531,
      "margin_dpo/margin_mean": 8.110418319702148,
      "margin_dpo/margin_std": 23.66728973388672,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": 5.656614303588867,
      "KL/mean": -1.8066785335540771,
      "KL/rejected_KL_mean": -9.26995849609375,
      "KL/std": 19.017704010009766,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.020524360239505768,
      "fcm_dpo/delta": 0.07615679502487183,
      "fcm_dpo/margin": 14.926575660705566,
      "fcm_dpo/q_t": 0.428432822227478,
      "grad_norm": 31.257823944091797,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 1.6631237268447876,
      "logits/rejected": 1.622153639793396,
      "logps/chosen": -266.725830078125,
      "logps/ref_chosen": -272.3824462890625,
      "logps/ref_rejected": -248.96444702148438,
      "logps/rejected": -258.2344055175781,
      "loss": 4.6147,
      "margin_dpo/margin_mean": 14.92657470703125,
      "margin_dpo/margin_std": 21.37627601623535,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": 3.488697052001953,
      "KL/mean": -3.6387805938720703,
      "KL/rejected_KL_mean": -10.766241073608398,
      "KL/std": 20.087749481201172,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.022133847698569298,
      "fcm_dpo/delta": 0.14451265335083008,
      "fcm_dpo/margin": 14.254928588867188,
      "fcm_dpo/q_t": 0.4275742769241333,
      "grad_norm": 47.806304931640625,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 1.7083895206451416,
      "logits/rejected": 1.7630856037139893,
      "logps/chosen": -284.58233642578125,
      "logps/ref_chosen": -288.071044921875,
      "logps/ref_rejected": -274.83477783203125,
      "logps/rejected": -285.6009826660156,
      "loss": 4.6784,
      "margin_dpo/margin_mean": 14.254927635192871,
      "margin_dpo/margin_std": 24.25179100036621,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": 6.639408111572266,
      "KL/mean": -0.36042773723602295,
      "KL/rejected_KL_mean": -7.360269546508789,
      "KL/std": 19.901702880859375,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.024510251358151436,
      "fcm_dpo/delta": 0.07126586139202118,
      "fcm_dpo/margin": 13.999675750732422,
      "fcm_dpo/q_t": 0.42289331555366516,
      "grad_norm": 39.50135040283203,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": 1.5882548093795776,
      "logits/rejected": 1.4828299283981323,
      "logps/chosen": -282.3524169921875,
      "logps/ref_chosen": -288.9918518066406,
      "logps/ref_rejected": -228.26071166992188,
      "logps/rejected": -235.6209716796875,
      "loss": 4.6456,
      "margin_dpo/margin_mean": 13.999675750732422,
      "margin_dpo/margin_std": 24.138038635253906,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": 6.256631851196289,
      "KL/mean": -1.7551326751708984,
      "KL/rejected_KL_mean": -9.766897201538086,
      "KL/std": 20.88290023803711,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.025608275085687637,
      "fcm_dpo/delta": 0.054042670875787735,
      "fcm_dpo/margin": 16.023521423339844,
      "fcm_dpo/q_t": 0.40881800651550293,
      "grad_norm": 43.72981262207031,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": 1.47073495388031,
      "logits/rejected": 1.4986305236816406,
      "logps/chosen": -283.2178039550781,
      "logps/ref_chosen": -289.4744873046875,
      "logps/ref_rejected": -265.130615234375,
      "logps/rejected": -274.89752197265625,
      "loss": 4.5061,
      "margin_dpo/margin_mean": 16.023523330688477,
      "margin_dpo/margin_std": 26.285497665405273,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": 7.702434539794922,
      "KL/mean": 1.0905137062072754,
      "KL/rejected_KL_mean": -5.521400451660156,
      "KL/std": 19.991565704345703,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.026206927374005318,
      "fcm_dpo/delta": 0.05294700711965561,
      "fcm_dpo/margin": 13.223828315734863,
      "fcm_dpo/q_t": 0.4233609735965729,
      "grad_norm": 41.953453063964844,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": 1.790850043296814,
      "logits/rejected": 1.7804946899414062,
      "logps/chosen": -282.8839111328125,
      "logps/ref_chosen": -290.5863342285156,
      "logps/ref_rejected": -263.41046142578125,
      "logps/rejected": -268.9318542480469,
      "loss": 4.6659,
      "margin_dpo/margin_mean": 13.223828315734863,
      "margin_dpo/margin_std": 23.631793975830078,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": 7.966377258300781,
      "KL/mean": -0.5003594160079956,
      "KL/rejected_KL_mean": -8.967098236083984,
      "KL/std": 18.930419921875,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.026692412793636322,
      "fcm_dpo/delta": -0.008457301184535027,
      "fcm_dpo/margin": 16.933467864990234,
      "fcm_dpo/q_t": 0.39985617995262146,
      "grad_norm": 39.23404312133789,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": 1.6561901569366455,
      "logits/rejected": 1.5715723037719727,
      "logps/chosen": -256.5738830566406,
      "logps/ref_chosen": -264.5402526855469,
      "logps/ref_rejected": -214.7958221435547,
      "logps/rejected": -223.76290893554688,
      "loss": 4.3607,
      "margin_dpo/margin_mean": 16.933469772338867,
      "margin_dpo/margin_std": 23.714200973510742,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": 3.9144630432128906,
      "KL/mean": -2.7743875980377197,
      "KL/rejected_KL_mean": -9.463239669799805,
      "KL/std": 19.140737533569336,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.027529913932085037,
      "fcm_dpo/delta": 0.03858964517712593,
      "fcm_dpo/margin": 13.377704620361328,
      "fcm_dpo/q_t": 0.41803863644599915,
      "grad_norm": 66.58966827392578,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": 1.502076506614685,
      "logits/rejected": 1.6161510944366455,
      "logps/chosen": -266.2652893066406,
      "logps/ref_chosen": -270.17974853515625,
      "logps/ref_rejected": -261.2323913574219,
      "logps/rejected": -270.69561767578125,
      "loss": 4.6323,
      "margin_dpo/margin_mean": 13.377705574035645,
      "margin_dpo/margin_std": 23.45267105102539,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": 2.802431106567383,
      "KL/mean": -2.913335084915161,
      "KL/rejected_KL_mean": -8.629104614257812,
      "KL/std": 21.3914852142334,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.028712399303913116,
      "fcm_dpo/delta": 0.048692040145397186,
      "fcm_dpo/margin": 11.431533813476562,
      "fcm_dpo/q_t": 0.42882218956947327,
      "grad_norm": 42.825218200683594,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": 1.4525302648544312,
      "logits/rejected": 1.421783208847046,
      "logps/chosen": -279.9781494140625,
      "logps/ref_chosen": -282.7806091308594,
      "logps/ref_rejected": -249.23399353027344,
      "logps/rejected": -257.86309814453125,
      "loss": 4.916,
      "margin_dpo/margin_mean": 11.431534767150879,
      "margin_dpo/margin_std": 25.809860229492188,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": 3.649202346801758,
      "KL/mean": -2.3682734966278076,
      "KL/rejected_KL_mean": -8.385749816894531,
      "KL/std": 19.873151779174805,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.029432816430926323,
      "fcm_dpo/delta": 0.06814030557870865,
      "fcm_dpo/margin": 12.034950256347656,
      "fcm_dpo/q_t": 0.4239949584007263,
      "grad_norm": 50.020389556884766,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": 1.6711677312850952,
      "logits/rejected": 1.691810131072998,
      "logps/chosen": -225.2681121826172,
      "logps/ref_chosen": -228.9173126220703,
      "logps/ref_rejected": -226.81222534179688,
      "logps/rejected": -235.197998046875,
      "loss": 4.7706,
      "margin_dpo/margin_mean": 12.034950256347656,
      "margin_dpo/margin_std": 24.651840209960938,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": 5.663875579833984,
      "KL/mean": -1.7436025142669678,
      "KL/rejected_KL_mean": -9.151079177856445,
      "KL/std": 20.42293930053711,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.03072037175297737,
      "fcm_dpo/delta": 0.04683511704206467,
      "fcm_dpo/margin": 14.814956665039062,
      "fcm_dpo/q_t": 0.4021509885787964,
      "grad_norm": 48.73683547973633,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": 1.741321086883545,
      "logits/rejected": 1.7186156511306763,
      "logps/chosen": -305.42205810546875,
      "logps/ref_chosen": -311.0859375,
      "logps/ref_rejected": -303.225341796875,
      "logps/rejected": -312.3764343261719,
      "loss": 4.4524,
      "margin_dpo/margin_mean": 14.814956665039062,
      "margin_dpo/margin_std": 24.091129302978516,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": 3.7308406829833984,
      "KL/mean": -3.3869247436523438,
      "KL/rejected_KL_mean": -10.504692077636719,
      "KL/std": 19.5787353515625,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.03231970965862274,
      "fcm_dpo/delta": 0.04112008959054947,
      "fcm_dpo/margin": 14.235535621643066,
      "fcm_dpo/q_t": 0.40098837018013,
      "grad_norm": 48.99260711669922,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 1.5193488597869873,
      "logits/rejected": 1.5322024822235107,
      "logps/chosen": -266.7525939941406,
      "logps/ref_chosen": -270.48345947265625,
      "logps/ref_rejected": -269.0491638183594,
      "logps/rejected": -279.5538330078125,
      "loss": 4.4525,
      "margin_dpo/margin_mean": 14.235536575317383,
      "margin_dpo/margin_std": 23.275096893310547,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": 7.274282455444336,
      "KL/mean": -3.062441110610962,
      "KL/rejected_KL_mean": -13.399164199829102,
      "KL/std": 21.21367645263672,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.03051789104938507,
      "fcm_dpo/delta": -0.14118832349777222,
      "fcm_dpo/margin": 20.67343521118164,
      "fcm_dpo/q_t": 0.36476796865463257,
      "grad_norm": 68.81394958496094,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.5092480182647705,
      "logits/rejected": 1.5257569551467896,
      "logps/chosen": -280.89141845703125,
      "logps/ref_chosen": -288.16571044921875,
      "logps/ref_rejected": -264.7412414550781,
      "logps/rejected": -278.14044189453125,
      "loss": 4.0184,
      "margin_dpo/margin_mean": 20.67343521118164,
      "margin_dpo/margin_std": 26.500263214111328,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": 5.646274566650391,
      "KL/mean": -2.20851993560791,
      "KL/rejected_KL_mean": -10.063314437866211,
      "KL/std": 21.50983428955078,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.02872726134955883,
      "fcm_dpo/delta": -0.01910620927810669,
      "fcm_dpo/margin": 15.709596633911133,
      "fcm_dpo/q_t": 0.4020068049430847,
      "grad_norm": 57.363563537597656,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": 1.290924072265625,
      "logits/rejected": 1.3611884117126465,
      "logps/chosen": -254.29225158691406,
      "logps/ref_chosen": -259.9385070800781,
      "logps/ref_rejected": -266.2453308105469,
      "logps/rejected": -276.30865478515625,
      "loss": 4.4851,
      "margin_dpo/margin_mean": 15.7095947265625,
      "margin_dpo/margin_std": 25.74575424194336,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": 4.2243804931640625,
      "KL/mean": -7.445926666259766,
      "KL/rejected_KL_mean": -19.11623191833496,
      "KL/std": 23.005102157592773,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.02672061324119568,
      "fcm_dpo/delta": -0.13447779417037964,
      "fcm_dpo/margin": 23.34060287475586,
      "fcm_dpo/q_t": 0.3611125946044922,
      "grad_norm": 45.75851058959961,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 1.7758398056030273,
      "logits/rejected": 1.845663070678711,
      "logps/chosen": -261.3526306152344,
      "logps/ref_chosen": -265.5769958496094,
      "logps/ref_rejected": -277.6246032714844,
      "logps/rejected": -296.7408447265625,
      "loss": 3.8421,
      "margin_dpo/margin_mean": 23.34060287475586,
      "margin_dpo/margin_std": 25.104759216308594,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": 1.9519157409667969,
      "KL/mean": -6.44109582901001,
      "KL/rejected_KL_mean": -14.834100723266602,
      "KL/std": 23.45318603515625,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.02619553543627262,
      "fcm_dpo/delta": 0.01858358085155487,
      "fcm_dpo/margin": 16.78602409362793,
      "fcm_dpo/q_t": 0.40260452032089233,
      "grad_norm": 46.73292541503906,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": 1.5383821725845337,
      "logits/rejected": 1.5164201259613037,
      "logps/chosen": -280.32147216796875,
      "logps/ref_chosen": -282.2734069824219,
      "logps/ref_rejected": -261.5135498046875,
      "logps/rejected": -276.34765625,
      "loss": 4.5355,
      "margin_dpo/margin_mean": 16.78602409362793,
      "margin_dpo/margin_std": 28.57425308227539,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": 0.2925281524658203,
      "KL/mean": -5.842662334442139,
      "KL/rejected_KL_mean": -11.977855682373047,
      "KL/std": 21.249221801757812,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.026629850268363953,
      "fcm_dpo/delta": 0.030585970729589462,
      "fcm_dpo/margin": 12.2703857421875,
      "fcm_dpo/q_t": 0.42817360162734985,
      "grad_norm": 50.04084777832031,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": 1.538283348083496,
      "logits/rejected": 1.6303126811981201,
      "logps/chosen": -278.4190368652344,
      "logps/ref_chosen": -278.7115783691406,
      "logps/ref_rejected": -298.0945739746094,
      "logps/rejected": -310.07244873046875,
      "loss": 4.8374,
      "margin_dpo/margin_mean": 12.2703857421875,
      "margin_dpo/margin_std": 26.04026222229004,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": 1.603414535522461,
      "KL/mean": -8.759516716003418,
      "KL/rejected_KL_mean": -19.122451782226562,
      "KL/std": 23.371320724487305,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.02651548571884632,
      "fcm_dpo/delta": -0.05285944417119026,
      "fcm_dpo/margin": 20.725866317749023,
      "fcm_dpo/q_t": 0.380948007106781,
      "grad_norm": 40.32730484008789,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": 1.6658281087875366,
      "logits/rejected": 1.6967509984970093,
      "logps/chosen": -266.7477111816406,
      "logps/ref_chosen": -268.35113525390625,
      "logps/ref_rejected": -241.9660186767578,
      "logps/rejected": -261.0884704589844,
      "loss": 4.2583,
      "margin_dpo/margin_mean": 20.725866317749023,
      "margin_dpo/margin_std": 30.86381721496582,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": 0.24628639221191406,
      "KL/mean": -6.909778118133545,
      "KL/rejected_KL_mean": -14.065845489501953,
      "KL/std": 22.713218688964844,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.025924205780029297,
      "fcm_dpo/delta": 0.019311202690005302,
      "fcm_dpo/margin": 14.312119483947754,
      "fcm_dpo/q_t": 0.41846010088920593,
      "grad_norm": 52.89689254760742,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": 1.4378918409347534,
      "logits/rejected": 1.497484803199768,
      "logps/chosen": -265.1573486328125,
      "logps/ref_chosen": -265.40362548828125,
      "logps/ref_rejected": -257.82440185546875,
      "logps/rejected": -271.8902282714844,
      "loss": 4.6782,
      "margin_dpo/margin_mean": 14.31212043762207,
      "margin_dpo/margin_std": 26.42030906677246,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": 0.6660804748535156,
      "KL/mean": -7.208062171936035,
      "KL/rejected_KL_mean": -15.082206726074219,
      "KL/std": 22.223081588745117,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.02702466957271099,
      "fcm_dpo/delta": 0.02079038880765438,
      "fcm_dpo/margin": 15.748287200927734,
      "fcm_dpo/q_t": 0.40584322810173035,
      "grad_norm": 43.66158676147461,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": 1.442277193069458,
      "logits/rejected": 1.481935739517212,
      "logps/chosen": -280.4961853027344,
      "logps/ref_chosen": -281.1622619628906,
      "logps/ref_rejected": -268.2899475097656,
      "logps/rejected": -283.37213134765625,
      "loss": 4.5306,
      "margin_dpo/margin_mean": 15.748286247253418,
      "margin_dpo/margin_std": 26.41692352294922,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -0.07575035095214844,
      "KL/mean": -6.425474643707275,
      "KL/rejected_KL_mean": -12.775196075439453,
      "KL/std": 23.34676742553711,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.026434268802404404,
      "fcm_dpo/delta": -0.011298663914203644,
      "fcm_dpo/margin": 12.699445724487305,
      "fcm_dpo/q_t": 0.42928484082221985,
      "grad_norm": 48.772464752197266,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": 1.4944663047790527,
      "logits/rejected": 1.374680995941162,
      "logps/chosen": -311.4007873535156,
      "logps/ref_chosen": -311.32501220703125,
      "logps/ref_rejected": -248.85350036621094,
      "logps/rejected": -261.6286926269531,
      "loss": 4.9853,
      "margin_dpo/margin_mean": 12.699446678161621,
      "margin_dpo/margin_std": 30.73833656311035,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": 3.274965286254883,
      "KL/mean": -6.159825325012207,
      "KL/rejected_KL_mean": -15.594615936279297,
      "KL/std": 22.935775756835938,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.02592385560274124,
      "fcm_dpo/delta": -0.029726708307862282,
      "fcm_dpo/margin": 18.869577407836914,
      "fcm_dpo/q_t": 0.39332666993141174,
      "grad_norm": 42.669986724853516,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": 1.6007617712020874,
      "logits/rejected": 1.6152366399765015,
      "logps/chosen": -263.833984375,
      "logps/ref_chosen": -267.10894775390625,
      "logps/ref_rejected": -258.66162109375,
      "logps/rejected": -274.2562561035156,
      "loss": 4.2925,
      "margin_dpo/margin_mean": 18.86957550048828,
      "margin_dpo/margin_std": 26.28278350830078,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": 3.1456527709960938,
      "KL/mean": -6.680301666259766,
      "KL/rejected_KL_mean": -16.506244659423828,
      "KL/std": 23.677452087402344,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.026107612997293472,
      "fcm_dpo/delta": -0.014074288308620453,
      "fcm_dpo/margin": 19.651905059814453,
      "fcm_dpo/q_t": 0.38919302821159363,
      "grad_norm": 49.53634262084961,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 1.6677488088607788,
      "logits/rejected": 1.5528379678726196,
      "logps/chosen": -298.7467346191406,
      "logps/ref_chosen": -301.89239501953125,
      "logps/ref_rejected": -247.645263671875,
      "logps/rejected": -264.1515197753906,
      "loss": 4.3136,
      "margin_dpo/margin_mean": 19.651905059814453,
      "margin_dpo/margin_std": 29.583837509155273,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": 4.455509185791016,
      "KL/mean": -4.34804105758667,
      "KL/rejected_KL_mean": -13.15158462524414,
      "KL/std": 22.941532135009766,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.025072362273931503,
      "fcm_dpo/delta": 0.013698695227503777,
      "fcm_dpo/margin": 17.607091903686523,
      "fcm_dpo/q_t": 0.40170612931251526,
      "grad_norm": 51.15333938598633,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": 1.5508891344070435,
      "logits/rejected": 1.5267348289489746,
      "logps/chosen": -283.7775573730469,
      "logps/ref_chosen": -288.2330627441406,
      "logps/ref_rejected": -252.5148162841797,
      "logps/rejected": -265.6664123535156,
      "loss": 4.4607,
      "margin_dpo/margin_mean": 17.607093811035156,
      "margin_dpo/margin_std": 27.489389419555664,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": 2.6031875610351562,
      "KL/mean": -7.513082027435303,
      "KL/rejected_KL_mean": -17.62936019897461,
      "KL/std": 21.983333587646484,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.025759361684322357,
      "fcm_dpo/delta": -0.023204365745186806,
      "fcm_dpo/margin": 20.232545852661133,
      "fcm_dpo/q_t": 0.3827674388885498,
      "grad_norm": 54.89320755004883,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": 1.6922770738601685,
      "logits/rejected": 1.6776084899902344,
      "logps/chosen": -284.16064453125,
      "logps/ref_chosen": -286.7638244628906,
      "logps/ref_rejected": -262.8302917480469,
      "logps/rejected": -280.4596252441406,
      "loss": 4.1154,
      "margin_dpo/margin_mean": 20.232545852661133,
      "margin_dpo/margin_std": 24.382301330566406,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -3.6851539611816406,
      "KL/mean": -12.542126655578613,
      "KL/rejected_KL_mean": -21.399099349975586,
      "KL/std": 23.380245208740234,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.025355199351906776,
      "fcm_dpo/delta": -0.029355496168136597,
      "fcm_dpo/margin": 17.71393394470215,
      "fcm_dpo/q_t": 0.40047743916511536,
      "grad_norm": 41.639671325683594,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": 1.5446739196777344,
      "logits/rejected": 1.4719704389572144,
      "logps/chosen": -263.963623046875,
      "logps/ref_chosen": -260.2784729003906,
      "logps/ref_rejected": -217.10556030273438,
      "logps/rejected": -238.50466918945312,
      "loss": 4.397,
      "margin_dpo/margin_mean": 17.71393394470215,
      "margin_dpo/margin_std": 26.544307708740234,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": 0.34374427795410156,
      "KL/mean": -6.93380880355835,
      "KL/rejected_KL_mean": -14.211360931396484,
      "KL/std": 21.831451416015625,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.02588883601129055,
      "fcm_dpo/delta": 0.059048302471637726,
      "fcm_dpo/margin": 14.555102348327637,
      "fcm_dpo/q_t": 0.416742742061615,
      "grad_norm": 50.27622604370117,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": 1.6374256610870361,
      "logits/rejected": 1.6821448802947998,
      "logps/chosen": -270.2105712890625,
      "logps/ref_chosen": -270.5543212890625,
      "logps/ref_rejected": -259.80218505859375,
      "logps/rejected": -274.0135192871094,
      "loss": 4.6099,
      "margin_dpo/margin_mean": 14.555103302001953,
      "margin_dpo/margin_std": 25.18033790588379,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -1.3818683624267578,
      "KL/mean": -8.198911666870117,
      "KL/rejected_KL_mean": -15.01595687866211,
      "KL/std": 23.97893524169922,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.026147983968257904,
      "fcm_dpo/delta": 0.03660065680742264,
      "fcm_dpo/margin": 13.63409423828125,
      "fcm_dpo/q_t": 0.4252607226371765,
      "grad_norm": 44.71527862548828,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 1.599522590637207,
      "logits/rejected": 1.581952691078186,
      "logps/chosen": -271.6763000488281,
      "logps/ref_chosen": -270.29443359375,
      "logps/ref_rejected": -230.40151977539062,
      "logps/rejected": -245.41746520996094,
      "loss": 4.875,
      "margin_dpo/margin_mean": 13.634096145629883,
      "margin_dpo/margin_std": 30.878925323486328,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -1.5077762603759766,
      "KL/mean": -9.402682304382324,
      "KL/rejected_KL_mean": -17.297578811645508,
      "KL/std": 23.177616119384766,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.02756008692085743,
      "fcm_dpo/delta": 0.02275221422314644,
      "fcm_dpo/margin": 15.789810180664062,
      "fcm_dpo/q_t": 0.4059685468673706,
      "grad_norm": 43.212459564208984,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": 1.5967707633972168,
      "logits/rejected": 1.5488660335540771,
      "logps/chosen": -274.9683837890625,
      "logps/ref_chosen": -273.4605712890625,
      "logps/ref_rejected": -248.86997985839844,
      "logps/rejected": -266.1675720214844,
      "loss": 4.5541,
      "margin_dpo/margin_mean": 15.789812088012695,
      "margin_dpo/margin_std": 27.575830459594727,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": 2.0213470458984375,
      "KL/mean": -8.388031005859375,
      "KL/rejected_KL_mean": -18.79742431640625,
      "KL/std": 25.19881820678711,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.025799905881285667,
      "fcm_dpo/delta": -0.09897828847169876,
      "fcm_dpo/margin": 20.81877326965332,
      "fcm_dpo/q_t": 0.3803689777851105,
      "grad_norm": 45.29140090942383,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": 1.64922297000885,
      "logits/rejected": 1.649074912071228,
      "logps/chosen": -303.703369140625,
      "logps/ref_chosen": -305.7247009277344,
      "logps/ref_rejected": -290.16650390625,
      "logps/rejected": -308.96392822265625,
      "loss": 4.1799,
      "margin_dpo/margin_mean": 20.81877326965332,
      "margin_dpo/margin_std": 27.261934280395508,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": 1.2973346710205078,
      "KL/mean": -10.634824752807617,
      "KL/rejected_KL_mean": -22.566974639892578,
      "KL/std": 24.601818084716797,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.0240301676094532,
      "fcm_dpo/delta": -0.12079715728759766,
      "fcm_dpo/margin": 23.864307403564453,
      "fcm_dpo/q_t": 0.37416937947273254,
      "grad_norm": 35.743412017822266,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": 1.2677626609802246,
      "logits/rejected": 1.2811099290847778,
      "logps/chosen": -250.8729248046875,
      "logps/ref_chosen": -252.1702880859375,
      "logps/ref_rejected": -233.34750366210938,
      "logps/rejected": -255.91448974609375,
      "loss": 4.0317,
      "margin_dpo/margin_mean": 23.864307403564453,
      "margin_dpo/margin_std": 27.65340805053711,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": 1.7891082763671875,
      "KL/mean": -8.375896453857422,
      "KL/rejected_KL_mean": -18.540904998779297,
      "KL/std": 23.212343215942383,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.022961996495723724,
      "fcm_dpo/delta": 0.034975565969944,
      "fcm_dpo/margin": 20.330013275146484,
      "fcm_dpo/q_t": 0.39598602056503296,
      "grad_norm": 38.662376403808594,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": 1.4924213886260986,
      "logits/rejected": 1.5297690629959106,
      "logps/chosen": -300.6893005371094,
      "logps/ref_chosen": -302.4783935546875,
      "logps/ref_rejected": -259.40777587890625,
      "logps/rejected": -277.9486999511719,
      "loss": 4.2673,
      "margin_dpo/margin_mean": 20.330015182495117,
      "margin_dpo/margin_std": 27.5391845703125,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": 1.4222793579101562,
      "KL/mean": -8.995136260986328,
      "KL/rejected_KL_mean": -19.412551879882812,
      "KL/std": 23.74144744873047,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.022861752659082413,
      "fcm_dpo/delta": -0.03283146023750305,
      "fcm_dpo/margin": 20.834836959838867,
      "fcm_dpo/q_t": 0.3936239182949066,
      "grad_norm": 39.593353271484375,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 1.4385496377944946,
      "logits/rejected": 1.5473015308380127,
      "logps/chosen": -249.2244415283203,
      "logps/ref_chosen": -250.646728515625,
      "logps/ref_rejected": -259.1659851074219,
      "logps/rejected": -278.57855224609375,
      "loss": 4.2466,
      "margin_dpo/margin_mean": 20.834836959838867,
      "margin_dpo/margin_std": 26.651826858520508,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": 2.358030319213867,
      "KL/mean": -4.608431816101074,
      "KL/rejected_KL_mean": -11.574897766113281,
      "KL/std": 24.682079315185547,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.024221152067184448,
      "fcm_dpo/delta": 0.11910364031791687,
      "fcm_dpo/margin": 13.932912826538086,
      "fcm_dpo/q_t": 0.4236924350261688,
      "grad_norm": 43.578365325927734,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": 1.6260100603103638,
      "logits/rejected": 1.6227073669433594,
      "logps/chosen": -326.6669006347656,
      "logps/ref_chosen": -329.02490234375,
      "logps/ref_rejected": -293.0215759277344,
      "logps/rejected": -304.5964660644531,
      "loss": 4.7759,
      "margin_dpo/margin_mean": 13.932912826538086,
      "margin_dpo/margin_std": 28.11588478088379,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": 1.9350433349609375,
      "KL/mean": -6.779400825500488,
      "KL/rejected_KL_mean": -15.493844985961914,
      "KL/std": 24.32616424560547,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.025564322248101234,
      "fcm_dpo/delta": 0.00413338840007782,
      "fcm_dpo/margin": 17.42889404296875,
      "fcm_dpo/q_t": 0.4018845558166504,
      "grad_norm": 41.68101501464844,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": 1.5467591285705566,
      "logits/rejected": 1.5178754329681396,
      "logps/chosen": -288.20556640625,
      "logps/ref_chosen": -290.140625,
      "logps/ref_rejected": -271.474853515625,
      "logps/rejected": -286.96868896484375,
      "loss": 4.4301,
      "margin_dpo/margin_mean": 17.42889404296875,
      "margin_dpo/margin_std": 26.840534210205078,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": 2.7770633697509766,
      "KL/mean": -8.041800498962402,
      "KL/rejected_KL_mean": -18.86064910888672,
      "KL/std": 23.58777618408203,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.0248112715780735,
      "fcm_dpo/delta": -0.04028956592082977,
      "fcm_dpo/margin": 21.63772964477539,
      "fcm_dpo/q_t": 0.38250553607940674,
      "grad_norm": 43.188106536865234,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": 1.5079617500305176,
      "logits/rejected": 1.4176305532455444,
      "logps/chosen": -326.53021240234375,
      "logps/ref_chosen": -329.3072509765625,
      "logps/ref_rejected": -256.6839599609375,
      "logps/rejected": -275.54461669921875,
      "loss": 4.1158,
      "margin_dpo/margin_mean": 21.637727737426758,
      "margin_dpo/margin_std": 26.846721649169922,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": 0.8732376098632812,
      "KL/mean": -7.898525714874268,
      "KL/rejected_KL_mean": -16.6702880859375,
      "KL/std": 24.455541610717773,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.024972641840577126,
      "fcm_dpo/delta": 0.026717210188508034,
      "fcm_dpo/margin": 17.54352378845215,
      "fcm_dpo/q_t": 0.4024149179458618,
      "grad_norm": 42.03804397583008,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": 1.4684417247772217,
      "logits/rejected": 1.4539307355880737,
      "logps/chosen": -275.029052734375,
      "logps/ref_chosen": -275.9023132324219,
      "logps/ref_rejected": -281.4163513183594,
      "logps/rejected": -298.0866394042969,
      "loss": 4.4979,
      "margin_dpo/margin_mean": 17.543521881103516,
      "margin_dpo/margin_std": 28.98847198486328,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -1.055673599243164,
      "KL/mean": -7.483036994934082,
      "KL/rejected_KL_mean": -13.910390853881836,
      "KL/std": 23.990861892700195,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.02472308836877346,
      "fcm_dpo/delta": -0.04210592061281204,
      "fcm_dpo/margin": 12.854723930358887,
      "fcm_dpo/q_t": 0.4282280206680298,
      "grad_norm": 46.58727264404297,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": 1.503767967224121,
      "logits/rejected": 1.5692119598388672,
      "logps/chosen": -244.619873046875,
      "logps/ref_chosen": -243.56423950195312,
      "logps/ref_rejected": -239.12905883789062,
      "logps/rejected": -253.0394287109375,
      "loss": 4.9286,
      "margin_dpo/margin_mean": 12.85472297668457,
      "margin_dpo/margin_std": 29.223665237426758,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -0.8472118377685547,
      "KL/mean": -7.318711757659912,
      "KL/rejected_KL_mean": -13.790216445922852,
      "KL/std": 24.5457763671875,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.0246879979968071,
      "fcm_dpo/delta": 0.03442464396357536,
      "fcm_dpo/margin": 12.943016052246094,
      "fcm_dpo/q_t": 0.42907729744911194,
      "grad_norm": 77.37248229980469,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": 1.56449556350708,
      "logits/rejected": 1.485506296157837,
      "logps/chosen": -294.15087890625,
      "logps/ref_chosen": -293.30364990234375,
      "logps/ref_rejected": -292.293701171875,
      "logps/rejected": -306.08392333984375,
      "loss": 4.9778,
      "margin_dpo/margin_mean": 12.943016052246094,
      "margin_dpo/margin_std": 31.13312339782715,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": 2.980937957763672,
      "KL/mean": -7.231280326843262,
      "KL/rejected_KL_mean": -17.4434871673584,
      "KL/std": 25.86884307861328,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.02423681877553463,
      "fcm_dpo/delta": -0.039432741701602936,
      "fcm_dpo/margin": 20.42443084716797,
      "fcm_dpo/q_t": 0.3927445709705353,
      "grad_norm": 54.50771713256836,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": 1.5508716106414795,
      "logits/rejected": 1.5678563117980957,
      "logps/chosen": -273.1452941894531,
      "logps/ref_chosen": -276.126220703125,
      "logps/ref_rejected": -240.31179809570312,
      "logps/rejected": -257.7552795410156,
      "loss": 4.3259,
      "margin_dpo/margin_mean": 20.42443084716797,
      "margin_dpo/margin_std": 30.567272186279297,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": 2.247842788696289,
      "KL/mean": -4.884121417999268,
      "KL/rejected_KL_mean": -12.016094207763672,
      "KL/std": 24.851221084594727,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.02437719702720642,
      "fcm_dpo/delta": 0.040706340223550797,
      "fcm_dpo/margin": 14.263941764831543,
      "fcm_dpo/q_t": 0.42409244179725647,
      "grad_norm": 54.71653366088867,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 1.435565710067749,
      "logits/rejected": 1.5478944778442383,
      "logps/chosen": -247.61460876464844,
      "logps/ref_chosen": -249.8624267578125,
      "logps/ref_rejected": -271.18603515625,
      "logps/rejected": -283.2021484375,
      "loss": 4.7847,
      "margin_dpo/margin_mean": 14.263941764831543,
      "margin_dpo/margin_std": 28.746904373168945,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": 3.8838138580322266,
      "KL/mean": -5.684909820556641,
      "KL/rejected_KL_mean": -15.253644943237305,
      "KL/std": 23.913930892944336,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.024857033044099808,
      "fcm_dpo/delta": -0.01464785635471344,
      "fcm_dpo/margin": 19.137466430664062,
      "fcm_dpo/q_t": 0.3962242603302002,
      "grad_norm": 45.110286712646484,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": 1.5326040983200073,
      "logits/rejected": 1.5642106533050537,
      "logps/chosen": -256.7852783203125,
      "logps/ref_chosen": -260.66912841796875,
      "logps/ref_rejected": -229.25491333007812,
      "logps/rejected": -244.508544921875,
      "loss": 4.3375,
      "margin_dpo/margin_mean": 19.137466430664062,
      "margin_dpo/margin_std": 27.92254638671875,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": 5.767498016357422,
      "KL/mean": -3.1858558654785156,
      "KL/rejected_KL_mean": -12.139205932617188,
      "KL/std": 23.656536102294922,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.024952903389930725,
      "fcm_dpo/delta": 0.05532096326351166,
      "fcm_dpo/margin": 17.90673065185547,
      "fcm_dpo/q_t": 0.4024082124233246,
      "grad_norm": 44.794097900390625,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.5375219583511353,
      "logits/rejected": 1.504660964012146,
      "logps/chosen": -285.12109375,
      "logps/ref_chosen": -290.8885803222656,
      "logps/ref_rejected": -275.1922302246094,
      "logps/rejected": -287.3314514160156,
      "loss": 4.4934,
      "margin_dpo/margin_mean": 17.90673065185547,
      "margin_dpo/margin_std": 29.494245529174805,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": 3.6624698638916016,
      "KL/mean": -7.385015964508057,
      "KL/rejected_KL_mean": -18.432504653930664,
      "KL/std": 24.2274169921875,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.025384465232491493,
      "fcm_dpo/delta": -0.06446253508329391,
      "fcm_dpo/margin": 22.094989776611328,
      "fcm_dpo/q_t": 0.3744257390499115,
      "grad_norm": 56.71232986450195,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": 1.3929295539855957,
      "logits/rejected": 1.3956677913665771,
      "logps/chosen": -266.90435791015625,
      "logps/ref_chosen": -270.56683349609375,
      "logps/ref_rejected": -260.3321533203125,
      "logps/rejected": -278.7646484375,
      "loss": 4.03,
      "margin_dpo/margin_mean": 22.094989776611328,
      "margin_dpo/margin_std": 25.669845581054688,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": 4.434354782104492,
      "KL/mean": -4.752945899963379,
      "KL/rejected_KL_mean": -13.940252304077148,
      "KL/std": 23.97296714782715,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.024733351543545723,
      "fcm_dpo/delta": 0.047651004046201706,
      "fcm_dpo/margin": 18.37460708618164,
      "fcm_dpo/q_t": 0.4006389379501343,
      "grad_norm": 42.538780212402344,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": 1.6731057167053223,
      "logits/rejected": 1.7261345386505127,
      "logps/chosen": -256.5936279296875,
      "logps/ref_chosen": -261.0279846191406,
      "logps/ref_rejected": -240.72361755371094,
      "logps/rejected": -254.66387939453125,
      "loss": 4.4125,
      "margin_dpo/margin_mean": 18.37460708618164,
      "margin_dpo/margin_std": 28.902524948120117,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": 8.194587707519531,
      "KL/mean": -4.8760480880737305,
      "KL/rejected_KL_mean": -17.946685791015625,
      "KL/std": 26.839893341064453,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.023441120982170105,
      "fcm_dpo/delta": -0.1236531138420105,
      "fcm_dpo/margin": 26.141281127929688,
      "fcm_dpo/q_t": 0.36602476239204407,
      "grad_norm": 44.20436096191406,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": 1.5843298435211182,
      "logits/rejected": 1.5618892908096313,
      "logps/chosen": -300.8892822265625,
      "logps/ref_chosen": -309.0838317871094,
      "logps/ref_rejected": -263.6435241699219,
      "logps/rejected": -281.5902099609375,
      "loss": 3.9651,
      "margin_dpo/margin_mean": 26.141277313232422,
      "margin_dpo/margin_std": 31.76325225830078,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": 2.7854766845703125,
      "KL/mean": -6.864591121673584,
      "KL/rejected_KL_mean": -16.51465606689453,
      "KL/std": 25.785049438476562,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.022976692765951157,
      "fcm_dpo/delta": 0.015215151011943817,
      "fcm_dpo/margin": 19.300125122070312,
      "fcm_dpo/q_t": 0.4036623537540436,
      "grad_norm": 41.961204528808594,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": 1.5531084537506104,
      "logits/rejected": 1.627820611000061,
      "logps/chosen": -262.916259765625,
      "logps/ref_chosen": -265.70172119140625,
      "logps/ref_rejected": -257.5074157714844,
      "logps/rejected": -274.0220642089844,
      "loss": 4.4682,
      "margin_dpo/margin_mean": 19.300125122070312,
      "margin_dpo/margin_std": 30.809659957885742,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": 1.27532958984375,
      "KL/mean": -8.178308486938477,
      "KL/rejected_KL_mean": -17.631946563720703,
      "KL/std": 24.246339797973633,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.0233543012291193,
      "fcm_dpo/delta": 0.019014297053217888,
      "fcm_dpo/margin": 18.907276153564453,
      "fcm_dpo/q_t": 0.4018472135066986,
      "grad_norm": 50.05070495605469,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": 1.5760433673858643,
      "logits/rejected": 1.600659728050232,
      "logps/chosen": -290.44964599609375,
      "logps/ref_chosen": -291.7249755859375,
      "logps/ref_rejected": -273.54010009765625,
      "logps/rejected": -291.17205810546875,
      "loss": 4.386,
      "margin_dpo/margin_mean": 18.907278060913086,
      "margin_dpo/margin_std": 27.391435623168945,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": 1.8294219970703125,
      "KL/mean": -8.008243560791016,
      "KL/rejected_KL_mean": -17.845905303955078,
      "KL/std": 26.46088409423828,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.023605627939105034,
      "fcm_dpo/delta": 0.036869123578071594,
      "fcm_dpo/margin": 19.67531967163086,
      "fcm_dpo/q_t": 0.39982885122299194,
      "grad_norm": 47.505287170410156,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": 1.6718838214874268,
      "logits/rejected": 1.713102102279663,
      "logps/chosen": -319.8724365234375,
      "logps/ref_chosen": -321.70184326171875,
      "logps/ref_rejected": -261.8386535644531,
      "logps/rejected": -279.6846008300781,
      "loss": 4.487,
      "margin_dpo/margin_mean": 19.675321578979492,
      "margin_dpo/margin_std": 32.837528228759766,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -0.5541267395019531,
      "KL/mean": -10.43777084350586,
      "KL/rejected_KL_mean": -20.321428298950195,
      "KL/std": 26.528581619262695,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.023997029289603233,
      "fcm_dpo/delta": -0.011217843741178513,
      "fcm_dpo/margin": 19.767301559448242,
      "fcm_dpo/q_t": 0.3972713351249695,
      "grad_norm": 41.705345153808594,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": 1.4828836917877197,
      "logits/rejected": 1.4586613178253174,
      "logps/chosen": -256.4695739746094,
      "logps/ref_chosen": -255.91542053222656,
      "logps/ref_rejected": -245.1119842529297,
      "logps/rejected": -265.43341064453125,
      "loss": 4.4651,
      "margin_dpo/margin_mean": 19.767301559448242,
      "margin_dpo/margin_std": 32.669166564941406,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": 5.20756721496582,
      "KL/mean": -6.365396022796631,
      "KL/rejected_KL_mean": -17.93836784362793,
      "KL/std": 25.688934326171875,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.023018017411231995,
      "fcm_dpo/delta": -0.08934271335601807,
      "fcm_dpo/margin": 23.14593505859375,
      "fcm_dpo/q_t": 0.3802693486213684,
      "grad_norm": 36.42985153198242,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": 1.543867588043213,
      "logits/rejected": 1.5728968381881714,
      "logps/chosen": -260.81097412109375,
      "logps/ref_chosen": -266.0185546875,
      "logps/ref_rejected": -272.862548828125,
      "logps/rejected": -290.8009033203125,
      "loss": 4.0607,
      "margin_dpo/margin_mean": 23.14593505859375,
      "margin_dpo/margin_std": 26.373544692993164,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": 3.6681461334228516,
      "KL/mean": -7.496296405792236,
      "KL/rejected_KL_mean": -18.660734176635742,
      "KL/std": 25.23130989074707,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.02243884466588497,
      "fcm_dpo/delta": -0.0016482900828123093,
      "fcm_dpo/margin": 22.328887939453125,
      "fcm_dpo/q_t": 0.3886590600013733,
      "grad_norm": 37.307064056396484,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 1.48799467086792,
      "logits/rejected": 1.4256293773651123,
      "logps/chosen": -246.4828338623047,
      "logps/ref_chosen": -250.15098571777344,
      "logps/ref_rejected": -223.34393310546875,
      "logps/rejected": -242.00465393066406,
      "loss": 4.254,
      "margin_dpo/margin_mean": 22.328886032104492,
      "margin_dpo/margin_std": 31.1062068939209,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": 3.2643470764160156,
      "KL/mean": -5.635519027709961,
      "KL/rejected_KL_mean": -14.53538703918457,
      "KL/std": 26.182960510253906,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.022745633497834206,
      "fcm_dpo/delta": 0.04591470584273338,
      "fcm_dpo/margin": 17.799728393554688,
      "fcm_dpo/q_t": 0.41206619143486023,
      "grad_norm": 46.92563247680664,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 1.4245549440383911,
      "logits/rejected": 1.4872061014175415,
      "logps/chosen": -235.12744140625,
      "logps/ref_chosen": -238.3917999267578,
      "logps/ref_rejected": -243.70272827148438,
      "logps/rejected": -258.2381286621094,
      "loss": 4.6185,
      "margin_dpo/margin_mean": 17.799726486206055,
      "margin_dpo/margin_std": 32.47420120239258,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": 0.34075164794921875,
      "KL/mean": -8.390233039855957,
      "KL/rejected_KL_mean": -17.1212158203125,
      "KL/std": 25.484851837158203,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.02295929193496704,
      "fcm_dpo/delta": -0.0067469896748661995,
      "fcm_dpo/margin": 17.461965560913086,
      "fcm_dpo/q_t": 0.4111282229423523,
      "grad_norm": 50.50203323364258,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.4544838666915894,
      "logits/rejected": 1.5228523015975952,
      "logps/chosen": -256.2962341308594,
      "logps/ref_chosen": -256.6369934082031,
      "logps/ref_rejected": -256.8894348144531,
      "logps/rejected": -274.0106506347656,
      "loss": 4.6583,
      "margin_dpo/margin_mean": 17.46196746826172,
      "margin_dpo/margin_std": 31.88494873046875,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -2.2097415924072266,
      "KL/mean": -11.374972343444824,
      "KL/rejected_KL_mean": -20.54020881652832,
      "KL/std": 27.71904754638672,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.023490339517593384,
      "fcm_dpo/delta": 0.03423144295811653,
      "fcm_dpo/margin": 18.330469131469727,
      "fcm_dpo/q_t": 0.40598395466804504,
      "grad_norm": 43.9073371887207,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": 1.522355079650879,
      "logits/rejected": 1.6164439916610718,
      "logps/chosen": -275.0093994140625,
      "logps/ref_chosen": -272.7996826171875,
      "logps/ref_rejected": -289.2391052246094,
      "logps/rejected": -309.7793273925781,
      "loss": 4.5457,
      "margin_dpo/margin_mean": 18.330469131469727,
      "margin_dpo/margin_std": 31.89092254638672,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -1.6300125122070312,
      "KL/mean": -8.57393741607666,
      "KL/rejected_KL_mean": -15.517864227294922,
      "KL/std": 27.664112091064453,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.024363718926906586,
      "fcm_dpo/delta": 0.06770587712526321,
      "fcm_dpo/margin": 13.88785171508789,
      "fcm_dpo/q_t": 0.42903196811676025,
      "grad_norm": 44.02293395996094,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": 1.1707743406295776,
      "logits/rejected": 1.196197271347046,
      "logps/chosen": -282.9390563964844,
      "logps/ref_chosen": -281.30902099609375,
      "logps/ref_rejected": -246.3243408203125,
      "logps/rejected": -261.84222412109375,
      "loss": 4.8372,
      "margin_dpo/margin_mean": 13.88785171508789,
      "margin_dpo/margin_std": 30.244003295898438,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -2.197690963745117,
      "KL/mean": -9.119394302368164,
      "KL/rejected_KL_mean": -16.041091918945312,
      "KL/std": 27.29082489013672,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.02520233578979969,
      "fcm_dpo/delta": 0.02569194883108139,
      "fcm_dpo/margin": 13.84339714050293,
      "fcm_dpo/q_t": 0.42584899067878723,
      "grad_norm": 46.18429946899414,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": 1.5356361865997314,
      "logits/rejected": 1.6347060203552246,
      "logps/chosen": -269.0558166503906,
      "logps/ref_chosen": -266.8581237792969,
      "logps/ref_rejected": -276.0855712890625,
      "logps/rejected": -292.12664794921875,
      "loss": 4.8411,
      "margin_dpo/margin_mean": 13.84339714050293,
      "margin_dpo/margin_std": 30.448915481567383,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": 3.003704071044922,
      "KL/mean": -7.371312618255615,
      "KL/rejected_KL_mean": -17.7463436126709,
      "KL/std": 25.614606857299805,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.025004370138049126,
      "fcm_dpo/delta": -0.06539537012577057,
      "fcm_dpo/margin": 20.750041961669922,
      "fcm_dpo/q_t": 0.3875337243080139,
      "grad_norm": 63.23878479003906,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": 1.4738712310791016,
      "logits/rejected": 1.5855073928833008,
      "logps/chosen": -263.9912414550781,
      "logps/ref_chosen": -266.99493408203125,
      "logps/ref_rejected": -248.60513305664062,
      "logps/rejected": -266.35150146484375,
      "loss": 4.238,
      "margin_dpo/margin_mean": 20.750041961669922,
      "margin_dpo/margin_std": 28.239017486572266,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": 5.285785675048828,
      "KL/mean": -8.126077651977539,
      "KL/rejected_KL_mean": -21.537939071655273,
      "KL/std": 28.251937866210938,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.02301020734012127,
      "fcm_dpo/delta": -0.16285070776939392,
      "fcm_dpo/margin": 26.82371711730957,
      "fcm_dpo/q_t": 0.3639894425868988,
      "grad_norm": 51.60329055786133,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": 1.3349766731262207,
      "logits/rejected": 1.2927849292755127,
      "logps/chosen": -297.7039794921875,
      "logps/ref_chosen": -302.98980712890625,
      "logps/ref_rejected": -262.8002624511719,
      "logps/rejected": -284.33819580078125,
      "loss": 3.9493,
      "margin_dpo/margin_mean": 26.82371711730957,
      "margin_dpo/margin_std": 30.413665771484375,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": 2.428445816040039,
      "KL/mean": -7.782029151916504,
      "KL/rejected_KL_mean": -17.992509841918945,
      "KL/std": 28.32909393310547,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.02127678692340851,
      "fcm_dpo/delta": 0.025381118059158325,
      "fcm_dpo/margin": 20.420949935913086,
      "fcm_dpo/q_t": 0.40452954173088074,
      "grad_norm": 42.62589645385742,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": 1.462496042251587,
      "logits/rejected": 1.543704867362976,
      "logps/chosen": -293.7833557128906,
      "logps/ref_chosen": -296.2117614746094,
      "logps/ref_rejected": -289.8357238769531,
      "logps/rejected": -307.8282165527344,
      "loss": 4.566,
      "margin_dpo/margin_mean": 20.420949935913086,
      "margin_dpo/margin_std": 35.906837463378906,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": 1.9507732391357422,
      "KL/mean": -8.66640853881836,
      "KL/rejected_KL_mean": -19.283580780029297,
      "KL/std": 26.672941207885742,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.021931443363428116,
      "fcm_dpo/delta": -0.00755532830953598,
      "fcm_dpo/margin": 21.234352111816406,
      "fcm_dpo/q_t": 0.3977181911468506,
      "grad_norm": 40.99057388305664,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": 1.326876163482666,
      "logits/rejected": 1.2523499727249146,
      "logps/chosen": -253.34185791015625,
      "logps/ref_chosen": -255.29263305664062,
      "logps/ref_rejected": -212.69761657714844,
      "logps/rejected": -231.981201171875,
      "loss": 4.3586,
      "margin_dpo/margin_mean": 21.23435401916504,
      "margin_dpo/margin_std": 31.7078800201416,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": 2.9443702697753906,
      "KL/mean": -6.034752368927002,
      "KL/rejected_KL_mean": -15.013877868652344,
      "KL/std": 27.890384674072266,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.02188418060541153,
      "fcm_dpo/delta": 0.03435240313410759,
      "fcm_dpo/margin": 17.9582576751709,
      "fcm_dpo/q_t": 0.41707083582878113,
      "grad_norm": 48.510459899902344,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 1.6478283405303955,
      "logits/rejected": 1.635554552078247,
      "logps/chosen": -310.5920715332031,
      "logps/ref_chosen": -313.5364074707031,
      "logps/ref_rejected": -274.349609375,
      "logps/rejected": -289.36346435546875,
      "loss": 4.8044,
      "margin_dpo/margin_mean": 17.9582576751709,
      "margin_dpo/margin_std": 38.35268783569336,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": 0.840576171875,
      "KL/mean": -10.404754638671875,
      "KL/rejected_KL_mean": -21.65007972717285,
      "KL/std": 28.011777877807617,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.022028401494026184,
      "fcm_dpo/delta": -0.04470596835017204,
      "fcm_dpo/margin": 22.49065399169922,
      "fcm_dpo/q_t": 0.3901330828666687,
      "grad_norm": 36.16594696044922,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 1.4603097438812256,
      "logits/rejected": 1.501871109008789,
      "logps/chosen": -261.8666076660156,
      "logps/ref_chosen": -262.7071838378906,
      "logps/ref_rejected": -243.85879516601562,
      "logps/rejected": -265.50885009765625,
      "loss": 4.2748,
      "margin_dpo/margin_mean": 22.49065399169922,
      "margin_dpo/margin_std": 30.90816879272461,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -1.3401546478271484,
      "KL/mean": -11.959707260131836,
      "KL/rejected_KL_mean": -22.57926368713379,
      "KL/std": 28.19306182861328,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.021505191922187805,
      "fcm_dpo/delta": -0.0004792753607034683,
      "fcm_dpo/margin": 21.239118576049805,
      "fcm_dpo/q_t": 0.40040379762649536,
      "grad_norm": 49.95712661743164,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": 1.5171266794204712,
      "logits/rejected": 1.489497184753418,
      "logps/chosen": -265.6269226074219,
      "logps/ref_chosen": -264.2867431640625,
      "logps/ref_rejected": -231.6894989013672,
      "logps/rejected": -254.26876831054688,
      "loss": 4.5124,
      "margin_dpo/margin_mean": 21.239116668701172,
      "margin_dpo/margin_std": 35.21827697753906,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -1.3336162567138672,
      "KL/mean": -9.789764404296875,
      "KL/rejected_KL_mean": -18.245914459228516,
      "KL/std": 27.07691764831543,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.02224450372159481,
      "fcm_dpo/delta": 0.08800016343593597,
      "fcm_dpo/margin": 16.912290573120117,
      "fcm_dpo/q_t": 0.41584834456443787,
      "grad_norm": 48.258338928222656,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": 1.8049485683441162,
      "logits/rejected": 1.7915972471237183,
      "logps/chosen": -272.6178283691406,
      "logps/ref_chosen": -271.2842102050781,
      "logps/ref_rejected": -247.95462036132812,
      "logps/rejected": -266.2005310058594,
      "loss": 4.5622,
      "margin_dpo/margin_mean": 16.912288665771484,
      "margin_dpo/margin_std": 27.876766204833984,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": 0.971527099609375,
      "KL/mean": -11.224570274353027,
      "KL/rejected_KL_mean": -23.4206600189209,
      "KL/std": 28.182703018188477,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.022280005738139153,
      "fcm_dpo/delta": -0.04676084592938423,
      "fcm_dpo/margin": 24.39219856262207,
      "fcm_dpo/q_t": 0.38198307156562805,
      "grad_norm": 37.581417083740234,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": 1.4821715354919434,
      "logits/rejected": 1.440575361251831,
      "logps/chosen": -309.1589050292969,
      "logps/ref_chosen": -310.13043212890625,
      "logps/ref_rejected": -267.3905334472656,
      "logps/rejected": -290.8111572265625,
      "loss": 4.155,
      "margin_dpo/margin_mean": 24.39219856262207,
      "margin_dpo/margin_std": 32.97306823730469,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": 2.4572830200195312,
      "KL/mean": -8.832206726074219,
      "KL/rejected_KL_mean": -20.121694564819336,
      "KL/std": 27.345314025878906,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.02254023775458336,
      "fcm_dpo/delta": -0.010645265690982342,
      "fcm_dpo/margin": 22.578983306884766,
      "fcm_dpo/q_t": 0.3893302083015442,
      "grad_norm": 44.06562042236328,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 1.4200186729431152,
      "logits/rejected": 1.4167134761810303,
      "logps/chosen": -274.7033996582031,
      "logps/ref_chosen": -277.16064453125,
      "logps/ref_rejected": -292.91815185546875,
      "logps/rejected": -313.03985595703125,
      "loss": 4.2988,
      "margin_dpo/margin_mean": 22.578981399536133,
      "margin_dpo/margin_std": 32.404842376708984,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": 2.781412124633789,
      "KL/mean": -7.461120128631592,
      "KL/rejected_KL_mean": -17.703649520874023,
      "KL/std": 27.237247467041016,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.021682605147361755,
      "fcm_dpo/delta": 0.020622391253709793,
      "fcm_dpo/margin": 20.485063552856445,
      "fcm_dpo/q_t": 0.4013519585132599,
      "grad_norm": 47.30685806274414,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": 1.6048787832260132,
      "logits/rejected": 1.7129998207092285,
      "logps/chosen": -284.00115966796875,
      "logps/ref_chosen": -286.7825927734375,
      "logps/ref_rejected": -289.08270263671875,
      "logps/rejected": -306.786376953125,
      "loss": 4.4112,
      "margin_dpo/margin_mean": 20.485063552856445,
      "margin_dpo/margin_std": 30.264453887939453,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": 0.11418724060058594,
      "KL/mean": -8.702489852905273,
      "KL/rejected_KL_mean": -17.519180297851562,
      "KL/std": 28.943588256835938,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.02274220436811447,
      "fcm_dpo/delta": 0.048416417092084885,
      "fcm_dpo/margin": 17.633371353149414,
      "fcm_dpo/q_t": 0.41429704427719116,
      "grad_norm": 46.518367767333984,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": 1.4280017614364624,
      "logits/rejected": 1.5316479206085205,
      "logps/chosen": -266.4180908203125,
      "logps/ref_chosen": -266.5322570800781,
      "logps/ref_rejected": -275.20672607421875,
      "logps/rejected": -292.7259216308594,
      "loss": 4.79,
      "margin_dpo/margin_mean": 17.633371353149414,
      "margin_dpo/margin_std": 36.966800689697266,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": 0.8615703582763672,
      "KL/mean": -9.850393295288086,
      "KL/rejected_KL_mean": -20.562368392944336,
      "KL/std": 28.32248878479004,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.02300262451171875,
      "fcm_dpo/delta": -0.04212130233645439,
      "fcm_dpo/margin": 21.42392921447754,
      "fcm_dpo/q_t": 0.39385998249053955,
      "grad_norm": 43.10112380981445,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": 1.6348551511764526,
      "logits/rejected": 1.6414332389831543,
      "logps/chosen": -271.93975830078125,
      "logps/ref_chosen": -272.80133056640625,
      "logps/ref_rejected": -288.4167785644531,
      "logps/rejected": -308.9791259765625,
      "loss": 4.3691,
      "margin_dpo/margin_mean": 21.42392921447754,
      "margin_dpo/margin_std": 32.972293853759766,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -1.1993846893310547,
      "KL/mean": -10.556899070739746,
      "KL/rejected_KL_mean": -19.914424896240234,
      "KL/std": 28.324739456176758,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.022694623097777367,
      "fcm_dpo/delta": 0.02240050584077835,
      "fcm_dpo/margin": 18.715045928955078,
      "fcm_dpo/q_t": 0.40847334265708923,
      "grad_norm": 52.353267669677734,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": 1.6725133657455444,
      "logits/rejected": 1.540366291999817,
      "logps/chosen": -311.8788146972656,
      "logps/ref_chosen": -310.679443359375,
      "logps/ref_rejected": -259.3792419433594,
      "logps/rejected": -279.2936706542969,
      "loss": 4.5807,
      "margin_dpo/margin_mean": 18.715049743652344,
      "margin_dpo/margin_std": 32.24475860595703,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": 1.6015377044677734,
      "KL/mean": -8.520307540893555,
      "KL/rejected_KL_mean": -18.64215660095215,
      "KL/std": 28.909852981567383,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.022692572325468063,
      "fcm_dpo/delta": 0.005227616056799889,
      "fcm_dpo/margin": 20.243694305419922,
      "fcm_dpo/q_t": 0.39966094493865967,
      "grad_norm": 40.27193832397461,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 1.5310298204421997,
      "logits/rejected": 1.5707497596740723,
      "logps/chosen": -273.3666076660156,
      "logps/ref_chosen": -274.9681701660156,
      "logps/ref_rejected": -283.346923828125,
      "logps/rejected": -301.9891052246094,
      "loss": 4.4574,
      "margin_dpo/margin_mean": 20.243694305419922,
      "margin_dpo/margin_std": 33.12035369873047,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": 4.763813018798828,
      "KL/mean": -9.132356643676758,
      "KL/rejected_KL_mean": -23.028522491455078,
      "KL/std": 28.717905044555664,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.02187255211174488,
      "fcm_dpo/delta": -0.11644446849822998,
      "fcm_dpo/margin": 27.79233169555664,
      "fcm_dpo/q_t": 0.36605536937713623,
      "grad_norm": 35.258445739746094,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.3903056383132935,
      "logits/rejected": 1.3732010126113892,
      "logps/chosen": -261.51580810546875,
      "logps/ref_chosen": -266.2795715332031,
      "logps/ref_rejected": -249.7120819091797,
      "logps/rejected": -272.7406005859375,
      "loss": 3.9254,
      "margin_dpo/margin_mean": 27.79233169555664,
      "margin_dpo/margin_std": 30.68857765197754,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": 3.364156723022461,
      "KL/mean": -6.377314567565918,
      "KL/rejected_KL_mean": -16.118783950805664,
      "KL/std": 26.801441192626953,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.021113742142915726,
      "fcm_dpo/delta": 0.047198764979839325,
      "fcm_dpo/margin": 19.482938766479492,
      "fcm_dpo/q_t": 0.4082508683204651,
      "grad_norm": 36.517967224121094,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": 1.6997841596603394,
      "logits/rejected": 1.7547862529754639,
      "logps/chosen": -261.388671875,
      "logps/ref_chosen": -264.7528076171875,
      "logps/ref_rejected": -241.02842712402344,
      "logps/rejected": -257.147216796875,
      "loss": 4.4992,
      "margin_dpo/margin_mean": 19.482940673828125,
      "margin_dpo/margin_std": 31.462257385253906,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": 0.3742504119873047,
      "KL/mean": -8.925004005432129,
      "KL/rejected_KL_mean": -18.224267959594727,
      "KL/std": 27.909217834472656,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.021887026727199554,
      "fcm_dpo/delta": 0.01787290722131729,
      "fcm_dpo/margin": 18.59851837158203,
      "fcm_dpo/q_t": 0.4133617579936981,
      "grad_norm": 40.55683898925781,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": 1.4586365222930908,
      "logits/rejected": 1.5749484300613403,
      "logps/chosen": -293.7643737792969,
      "logps/ref_chosen": -294.13861083984375,
      "logps/ref_rejected": -303.11669921875,
      "logps/rejected": -321.3409729003906,
      "loss": 4.5701,
      "margin_dpo/margin_mean": 18.59851837158203,
      "margin_dpo/margin_std": 32.48381423950195,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -4.653156280517578,
      "KL/mean": -14.195435523986816,
      "KL/rejected_KL_mean": -23.737716674804688,
      "KL/std": 28.072280883789062,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.022465461865067482,
      "fcm_dpo/delta": 0.0357758104801178,
      "fcm_dpo/margin": 19.084556579589844,
      "fcm_dpo/q_t": 0.4073125422000885,
      "grad_norm": 59.27125930786133,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": 1.6325297355651855,
      "logits/rejected": 1.7843742370605469,
      "logps/chosen": -271.2298889160156,
      "logps/ref_chosen": -266.5767517089844,
      "logps/ref_rejected": -291.2264709472656,
      "logps/rejected": -314.9642028808594,
      "loss": 4.5319,
      "margin_dpo/margin_mean": 19.084556579589844,
      "margin_dpo/margin_std": 32.8261604309082,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -1.8562164306640625,
      "KL/mean": -11.612351417541504,
      "KL/rejected_KL_mean": -21.368486404418945,
      "KL/std": 30.644540786743164,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.022230125963687897,
      "fcm_dpo/delta": -0.02628779597580433,
      "fcm_dpo/margin": 19.512264251708984,
      "fcm_dpo/q_t": 0.40889352560043335,
      "grad_norm": 41.068756103515625,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": 1.4935755729675293,
      "logits/rejected": 1.489006757736206,
      "logps/chosen": -286.89697265625,
      "logps/ref_chosen": -285.040771484375,
      "logps/ref_rejected": -258.0254821777344,
      "logps/rejected": -279.3939514160156,
      "loss": 4.6046,
      "margin_dpo/margin_mean": 19.512264251708984,
      "margin_dpo/margin_std": 35.52467346191406,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": 0.8715686798095703,
      "KL/mean": -9.905322074890137,
      "KL/rejected_KL_mean": -20.682209014892578,
      "KL/std": 29.54822540283203,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.021859154105186462,
      "fcm_dpo/delta": -0.01129305548965931,
      "fcm_dpo/margin": 21.553781509399414,
      "fcm_dpo/q_t": 0.3964061737060547,
      "grad_norm": 44.43794250488281,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": 1.5941317081451416,
      "logits/rejected": 1.6050429344177246,
      "logps/chosen": -331.87982177734375,
      "logps/ref_chosen": -332.7513732910156,
      "logps/ref_rejected": -302.9167785644531,
      "logps/rejected": -323.5989685058594,
      "loss": 4.4567,
      "margin_dpo/margin_mean": 21.553781509399414,
      "margin_dpo/margin_std": 34.7470703125,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -3.1434879302978516,
      "KL/mean": -17.414602279663086,
      "KL/rejected_KL_mean": -31.685718536376953,
      "KL/std": 31.645814895629883,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.021169180050492287,
      "fcm_dpo/delta": -0.10975827276706696,
      "fcm_dpo/margin": 28.54221534729004,
      "fcm_dpo/q_t": 0.37226736545562744,
      "grad_norm": 33.05510711669922,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": 1.5253796577453613,
      "logits/rejected": 1.5187439918518066,
      "logps/chosen": -289.6114501953125,
      "logps/ref_chosen": -286.46795654296875,
      "logps/ref_rejected": -253.99298095703125,
      "logps/rejected": -285.6787109375,
      "loss": 4.0914,
      "margin_dpo/margin_mean": 28.542217254638672,
      "margin_dpo/margin_std": 38.55399703979492,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": 0.6197795867919922,
      "KL/mean": -10.862521171569824,
      "KL/rejected_KL_mean": -22.344820022583008,
      "KL/std": 30.270959854125977,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.0205703042447567,
      "fcm_dpo/delta": 0.02819104678928852,
      "fcm_dpo/margin": 22.9645938873291,
      "fcm_dpo/q_t": 0.3966803252696991,
      "grad_norm": 37.832130432128906,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 1.541850209236145,
      "logits/rejected": 1.5834317207336426,
      "logps/chosen": -310.7587585449219,
      "logps/ref_chosen": -311.3785400390625,
      "logps/ref_rejected": -290.3092956542969,
      "logps/rejected": -312.6541442871094,
      "loss": 4.3472,
      "margin_dpo/margin_mean": 22.96459197998047,
      "margin_dpo/margin_std": 34.33341598510742,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": 0.7409744262695312,
      "KL/mean": -11.34870719909668,
      "KL/rejected_KL_mean": -23.43838119506836,
      "KL/std": 28.20771598815918,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.0203812625259161,
      "fcm_dpo/delta": -0.03206367418169975,
      "fcm_dpo/margin": 24.179359436035156,
      "fcm_dpo/q_t": 0.3905361294746399,
      "grad_norm": 34.3472900390625,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": 1.3319720029830933,
      "logits/rejected": 1.3850853443145752,
      "logps/chosen": -258.59197998046875,
      "logps/ref_chosen": -259.33294677734375,
      "logps/ref_rejected": -261.7661437988281,
      "logps/rejected": -285.2044982910156,
      "loss": 4.2622,
      "margin_dpo/margin_mean": 24.179359436035156,
      "margin_dpo/margin_std": 32.85729217529297,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -0.0023136138916015625,
      "KL/mean": -12.74544620513916,
      "KL/rejected_KL_mean": -25.48859405517578,
      "KL/std": 30.33628273010254,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.01997969299554825,
      "fcm_dpo/delta": -0.009944531135261059,
      "fcm_dpo/margin": 25.486284255981445,
      "fcm_dpo/q_t": 0.3874198794364929,
      "grad_norm": 36.58561706542969,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 1.889756202697754,
      "logits/rejected": 1.8604758977890015,
      "logps/chosen": -306.2066955566406,
      "logps/ref_chosen": -306.2043762207031,
      "logps/ref_rejected": -256.7926330566406,
      "logps/rejected": -282.28125,
      "loss": 4.2432,
      "margin_dpo/margin_mean": 25.486282348632812,
      "margin_dpo/margin_std": 35.85205078125,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -2.416675567626953,
      "KL/mean": -12.026041030883789,
      "KL/rejected_KL_mean": -21.635406494140625,
      "KL/std": 30.353633880615234,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.02092001773416996,
      "fcm_dpo/delta": 0.05995943397283554,
      "fcm_dpo/margin": 19.21872329711914,
      "fcm_dpo/q_t": 0.4116639494895935,
      "grad_norm": 42.17182922363281,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 1.2371704578399658,
      "logits/rejected": 1.3477809429168701,
      "logps/chosen": -295.7486267089844,
      "logps/ref_chosen": -293.3319396972656,
      "logps/ref_rejected": -282.7599182128906,
      "logps/rejected": -304.39532470703125,
      "loss": 4.6002,
      "margin_dpo/margin_mean": 19.218725204467773,
      "margin_dpo/margin_std": 34.31272888183594,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -0.9813690185546875,
      "KL/mean": -12.987902641296387,
      "KL/rejected_KL_mean": -24.994434356689453,
      "KL/std": 28.90921974182129,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.020513873547315598,
      "fcm_dpo/delta": 0.0059272367507219315,
      "fcm_dpo/margin": 24.013065338134766,
      "fcm_dpo/q_t": 0.39112353324890137,
      "grad_norm": 44.63298034667969,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": 1.485053300857544,
      "logits/rejected": 1.3782469034194946,
      "logps/chosen": -287.9875183105469,
      "logps/ref_chosen": -287.0061340332031,
      "logps/ref_rejected": -233.23130798339844,
      "logps/rejected": -258.2257385253906,
      "loss": 4.2617,
      "margin_dpo/margin_mean": 24.013065338134766,
      "margin_dpo/margin_std": 32.481842041015625,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -1.7717342376708984,
      "KL/mean": -15.359490394592285,
      "KL/rejected_KL_mean": -28.94725227355957,
      "KL/std": 32.312740325927734,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.02028743177652359,
      "fcm_dpo/delta": -0.0566394105553627,
      "fcm_dpo/margin": 27.17552375793457,
      "fcm_dpo/q_t": 0.3789609670639038,
      "grad_norm": 35.602508544921875,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": 1.5037739276885986,
      "logits/rejected": 1.4040201902389526,
      "logps/chosen": -277.5870361328125,
      "logps/ref_chosen": -275.8153076171875,
      "logps/ref_rejected": -238.54286193847656,
      "logps/rejected": -267.4900817871094,
      "loss": 4.1539,
      "margin_dpo/margin_mean": 27.175521850585938,
      "margin_dpo/margin_std": 35.91019058227539,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -2.9217300415039062,
      "KL/mean": -16.16417694091797,
      "KL/rejected_KL_mean": -29.406620025634766,
      "KL/std": 30.802074432373047,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.019911842420697212,
      "fcm_dpo/delta": -0.06768704950809479,
      "fcm_dpo/margin": 26.48488998413086,
      "fcm_dpo/q_t": 0.3825586438179016,
      "grad_norm": 34.64250946044922,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": 1.4521839618682861,
      "logits/rejected": 1.512455940246582,
      "logps/chosen": -242.02212524414062,
      "logps/ref_chosen": -239.1003875732422,
      "logps/ref_rejected": -230.31900024414062,
      "logps/rejected": -259.7256164550781,
      "loss": 4.1421,
      "margin_dpo/margin_mean": 26.48488998413086,
      "margin_dpo/margin_std": 32.86759948730469,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -7.436559677124023,
      "KL/mean": -17.687332153320312,
      "KL/rejected_KL_mean": -27.938108444213867,
      "KL/std": 32.354705810546875,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.019064106047153473,
      "fcm_dpo/delta": 0.056184522807598114,
      "fcm_dpo/margin": 20.501535415649414,
      "fcm_dpo/q_t": 0.4175623953342438,
      "grad_norm": 38.31582260131836,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": 1.4967445135116577,
      "logits/rejected": 1.5152662992477417,
      "logps/chosen": -307.76593017578125,
      "logps/ref_chosen": -300.3293762207031,
      "logps/ref_rejected": -332.132568359375,
      "logps/rejected": -360.0706787109375,
      "loss": 4.7334,
      "margin_dpo/margin_mean": 20.501537322998047,
      "margin_dpo/margin_std": 40.918792724609375,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -5.192844390869141,
      "KL/mean": -14.435035705566406,
      "KL/rejected_KL_mean": -23.67721939086914,
      "KL/std": 29.87112808227539,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.020574389025568962,
      "fcm_dpo/delta": 0.08295334875583649,
      "fcm_dpo/margin": 18.484375,
      "fcm_dpo/q_t": 0.4177274703979492,
      "grad_norm": 38.172874450683594,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": 1.5231850147247314,
      "logits/rejected": 1.5881251096725464,
      "logps/chosen": -250.60498046875,
      "logps/ref_chosen": -245.41213989257812,
      "logps/ref_rejected": -251.72999572753906,
      "logps/rejected": -275.4071960449219,
      "loss": 4.6879,
      "margin_dpo/margin_mean": 18.484375,
      "margin_dpo/margin_std": 35.6612663269043,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -5.299543380737305,
      "KL/mean": -18.413965225219727,
      "KL/rejected_KL_mean": -31.528379440307617,
      "KL/std": 31.8794002532959,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.020868917927145958,
      "fcm_dpo/delta": -0.05092344433069229,
      "fcm_dpo/margin": 26.22884750366211,
      "fcm_dpo/q_t": 0.38042202591896057,
      "grad_norm": 39.33684539794922,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": 1.4402408599853516,
      "logits/rejected": 1.4821916818618774,
      "logps/chosen": -282.17535400390625,
      "logps/ref_chosen": -276.87579345703125,
      "logps/ref_rejected": -259.0455017089844,
      "logps/rejected": -290.5738830566406,
      "loss": 4.2002,
      "margin_dpo/margin_mean": 26.22884750366211,
      "margin_dpo/margin_std": 36.26725769042969,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -3.904726028442383,
      "KL/mean": -17.158384323120117,
      "KL/rejected_KL_mean": -30.41204261779785,
      "KL/std": 31.250690460205078,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.020314836874604225,
      "fcm_dpo/delta": -0.04121715947985649,
      "fcm_dpo/margin": 26.50731658935547,
      "fcm_dpo/q_t": 0.38272884488105774,
      "grad_norm": 38.05610275268555,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": 1.696607232093811,
      "logits/rejected": 1.704059362411499,
      "logps/chosen": -307.8757629394531,
      "logps/ref_chosen": -303.9710693359375,
      "logps/ref_rejected": -255.62033081054688,
      "logps/rejected": -286.0323791503906,
      "loss": 4.1755,
      "margin_dpo/margin_mean": 26.50731658935547,
      "margin_dpo/margin_std": 35.5152473449707,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -4.289339065551758,
      "KL/mean": -21.608654022216797,
      "KL/rejected_KL_mean": -38.92796325683594,
      "KL/std": 33.19681167602539,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.01839957945048809,
      "fcm_dpo/delta": -0.14846664667129517,
      "fcm_dpo/margin": 34.63862228393555,
      "fcm_dpo/q_t": 0.35887980461120605,
      "grad_norm": 46.876583099365234,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": 1.3077651262283325,
      "logits/rejected": 1.2705870866775513,
      "logps/chosen": -285.9852600097656,
      "logps/ref_chosen": -281.6959533691406,
      "logps/ref_rejected": -258.93353271484375,
      "logps/rejected": -297.8615417480469,
      "loss": 3.859,
      "margin_dpo/margin_mean": 34.63862228393555,
      "margin_dpo/margin_std": 37.348854064941406,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -5.96107292175293,
      "KL/mean": -20.0012264251709,
      "KL/rejected_KL_mean": -34.04137420654297,
      "KL/std": 31.98023223876953,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.017677079886198044,
      "fcm_dpo/delta": 0.0023499801754951477,
      "fcm_dpo/margin": 28.080291748046875,
      "fcm_dpo/q_t": 0.38893598318099976,
      "grad_norm": 45.7639274597168,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 1.1456023454666138,
      "logits/rejected": 1.2253587245941162,
      "logps/chosen": -229.91761779785156,
      "logps/ref_chosen": -223.95655822753906,
      "logps/ref_rejected": -238.95187377929688,
      "logps/rejected": -272.9932556152344,
      "loss": 4.2586,
      "margin_dpo/margin_mean": 28.080291748046875,
      "margin_dpo/margin_std": 38.07732009887695,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -9.972625732421875,
      "KL/mean": -21.185670852661133,
      "KL/rejected_KL_mean": -32.39871597290039,
      "KL/std": 31.618499755859375,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.01787586137652397,
      "fcm_dpo/delta": 0.06127004325389862,
      "fcm_dpo/margin": 22.426097869873047,
      "fcm_dpo/q_t": 0.40958744287490845,
      "grad_norm": 37.27577590942383,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 1.4258936643600464,
      "logits/rejected": 1.4934577941894531,
      "logps/chosen": -324.8972473144531,
      "logps/ref_chosen": -314.92462158203125,
      "logps/ref_rejected": -279.6347351074219,
      "logps/rejected": -312.033447265625,
      "loss": 4.4752,
      "margin_dpo/margin_mean": 22.426101684570312,
      "margin_dpo/margin_std": 34.230979919433594,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -5.795236587524414,
      "KL/mean": -18.390058517456055,
      "KL/rejected_KL_mean": -30.984880447387695,
      "KL/std": 33.70909881591797,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.01820511184632778,
      "fcm_dpo/delta": -0.015445715747773647,
      "fcm_dpo/margin": 25.189655303955078,
      "fcm_dpo/q_t": 0.39917218685150146,
      "grad_norm": 42.97810745239258,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": 1.5419259071350098,
      "logits/rejected": 1.6084210872650146,
      "logps/chosen": -269.1148376464844,
      "logps/ref_chosen": -263.3196105957031,
      "logps/ref_rejected": -247.13612365722656,
      "logps/rejected": -278.1209716796875,
      "loss": 4.3666,
      "margin_dpo/margin_mean": 25.189655303955078,
      "margin_dpo/margin_std": 36.80788040161133,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -6.007509231567383,
      "KL/mean": -19.5338077545166,
      "KL/rejected_KL_mean": -33.06010055541992,
      "KL/std": 32.344608306884766,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.01839297078549862,
      "fcm_dpo/delta": 0.0013584308326244354,
      "fcm_dpo/margin": 27.052587509155273,
      "fcm_dpo/q_t": 0.3907015919685364,
      "grad_norm": 36.22296142578125,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": 1.4175597429275513,
      "logits/rejected": 1.4860700368881226,
      "logps/chosen": -285.3697509765625,
      "logps/ref_chosen": -279.36224365234375,
      "logps/ref_rejected": -264.46112060546875,
      "logps/rejected": -297.5212097167969,
      "loss": 4.2582,
      "margin_dpo/margin_mean": 27.05259132385254,
      "margin_dpo/margin_std": 37.36870574951172,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -5.216760635375977,
      "KL/mean": -19.192344665527344,
      "KL/rejected_KL_mean": -33.167930603027344,
      "KL/std": 34.29956817626953,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.01832735911011696,
      "fcm_dpo/delta": -0.013369518332183361,
      "fcm_dpo/margin": 27.95116424560547,
      "fcm_dpo/q_t": 0.3890242576599121,
      "grad_norm": 37.798709869384766,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": 1.1806925535202026,
      "logits/rejected": 1.244687795639038,
      "logps/chosen": -291.6658935546875,
      "logps/ref_chosen": -286.4491271972656,
      "logps/ref_rejected": -294.5469665527344,
      "logps/rejected": -327.71490478515625,
      "loss": 4.3372,
      "margin_dpo/margin_mean": 27.95116424560547,
      "margin_dpo/margin_std": 42.28049850463867,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -7.992887496948242,
      "KL/mean": -20.413177490234375,
      "KL/rejected_KL_mean": -32.833465576171875,
      "KL/std": 35.54878234863281,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.018449630588293076,
      "fcm_dpo/delta": -0.002752946689724922,
      "fcm_dpo/margin": 24.840574264526367,
      "fcm_dpo/q_t": 0.3994058072566986,
      "grad_norm": 58.96935272216797,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": 1.512418270111084,
      "logits/rejected": 1.5140960216522217,
      "logps/chosen": -290.3529357910156,
      "logps/ref_chosen": -282.36004638671875,
      "logps/ref_rejected": -243.79052734375,
      "logps/rejected": -276.62396240234375,
      "loss": 4.4759,
      "margin_dpo/margin_mean": 24.840576171875,
      "margin_dpo/margin_std": 40.77588653564453,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -3.8103466033935547,
      "KL/mean": -18.237276077270508,
      "KL/rejected_KL_mean": -32.66420364379883,
      "KL/std": 33.48912811279297,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.01817731373012066,
      "fcm_dpo/delta": -0.026607418432831764,
      "fcm_dpo/margin": 28.853857040405273,
      "fcm_dpo/q_t": 0.3827803432941437,
      "grad_norm": 31.963726043701172,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": 1.4454190731048584,
      "logits/rejected": 1.4540629386901855,
      "logps/chosen": -282.94976806640625,
      "logps/ref_chosen": -279.139404296875,
      "logps/ref_rejected": -240.5850067138672,
      "logps/rejected": -273.24920654296875,
      "loss": 4.2108,
      "margin_dpo/margin_mean": 28.853857040405273,
      "margin_dpo/margin_std": 38.7138671875,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -7.628204345703125,
      "KL/mean": -19.977664947509766,
      "KL/rejected_KL_mean": -32.32711410522461,
      "KL/std": 30.607446670532227,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.017916545271873474,
      "fcm_dpo/delta": 0.003640567883849144,
      "fcm_dpo/margin": 24.698915481567383,
      "fcm_dpo/q_t": 0.40237748622894287,
      "grad_norm": 40.15890884399414,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": 1.5768283605575562,
      "logits/rejected": 1.6519980430603027,
      "logps/chosen": -266.4397277832031,
      "logps/ref_chosen": -258.8114929199219,
      "logps/ref_rejected": -266.3898010253906,
      "logps/rejected": -298.7168884277344,
      "loss": 4.4314,
      "margin_dpo/margin_mean": 24.698915481567383,
      "margin_dpo/margin_std": 38.160221099853516,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -7.732395172119141,
      "KL/mean": -21.818706512451172,
      "KL/rejected_KL_mean": -35.9050178527832,
      "KL/std": 34.95335006713867,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.01711191050708294,
      "fcm_dpo/delta": -0.04651428759098053,
      "fcm_dpo/margin": 28.172616958618164,
      "fcm_dpo/q_t": 0.39424046874046326,
      "grad_norm": 36.986289978027344,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": 1.2698062658309937,
      "logits/rejected": 1.1756470203399658,
      "logps/chosen": -309.798583984375,
      "logps/ref_chosen": -302.0661926269531,
      "logps/ref_rejected": -265.5518798828125,
      "logps/rejected": -301.4568786621094,
      "loss": 4.309,
      "margin_dpo/margin_mean": 28.17261505126953,
      "margin_dpo/margin_std": 39.96277618408203,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -4.363134384155273,
      "KL/mean": -16.113767623901367,
      "KL/rejected_KL_mean": -27.864402770996094,
      "KL/std": 34.068790435791016,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.017539935186505318,
      "fcm_dpo/delta": 0.09222282469272614,
      "fcm_dpo/margin": 23.50127601623535,
      "fcm_dpo/q_t": 0.4108741283416748,
      "grad_norm": 41.15702438354492,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": 1.5170950889587402,
      "logits/rejected": 1.6136685609817505,
      "logps/chosen": -263.36151123046875,
      "logps/ref_chosen": -258.99835205078125,
      "logps/ref_rejected": -273.7834777832031,
      "logps/rejected": -301.64788818359375,
      "loss": 4.5958,
      "margin_dpo/margin_mean": 23.501277923583984,
      "margin_dpo/margin_std": 42.582908630371094,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -2.7504005432128906,
      "KL/mean": -17.797470092773438,
      "KL/rejected_KL_mean": -32.84454345703125,
      "KL/std": 34.05438995361328,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.017592858523130417,
      "fcm_dpo/delta": -0.07368596643209457,
      "fcm_dpo/margin": 30.094144821166992,
      "fcm_dpo/q_t": 0.38161399960517883,
      "grad_norm": 40.40775680541992,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 1.3076533079147339,
      "logits/rejected": 1.381419062614441,
      "logps/chosen": -278.2734680175781,
      "logps/ref_chosen": -275.5230407714844,
      "logps/ref_rejected": -255.50784301757812,
      "logps/rejected": -288.35235595703125,
      "loss": 4.1229,
      "margin_dpo/margin_mean": 30.09414291381836,
      "margin_dpo/margin_std": 37.56632995605469,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -6.403270721435547,
      "KL/mean": -17.97686767578125,
      "KL/rejected_KL_mean": -29.550466537475586,
      "KL/std": 34.77226638793945,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.017447544261813164,
      "fcm_dpo/delta": 0.05021177977323532,
      "fcm_dpo/margin": 23.147197723388672,
      "fcm_dpo/q_t": 0.4126003086566925,
      "grad_norm": 33.244625091552734,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.494274377822876,
      "logits/rejected": 1.5359807014465332,
      "logps/chosen": -259.7667236328125,
      "logps/ref_chosen": -253.36349487304688,
      "logps/ref_rejected": -247.1916961669922,
      "logps/rejected": -276.7421569824219,
      "loss": 4.6078,
      "margin_dpo/margin_mean": 23.147199630737305,
      "margin_dpo/margin_std": 41.4663200378418,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -4.768028259277344,
      "KL/mean": -18.703519821166992,
      "KL/rejected_KL_mean": -32.639007568359375,
      "KL/std": 34.00221252441406,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.017938071861863136,
      "fcm_dpo/delta": -0.0017030963208526373,
      "fcm_dpo/margin": 27.870983123779297,
      "fcm_dpo/q_t": 0.3888096809387207,
      "grad_norm": 33.028045654296875,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": 1.6613554954528809,
      "logits/rejected": 1.6181492805480957,
      "logps/chosen": -269.756103515625,
      "logps/ref_chosen": -264.9880676269531,
      "logps/ref_rejected": -243.07838439941406,
      "logps/rejected": -275.7174072265625,
      "loss": 4.2351,
      "margin_dpo/margin_mean": 27.870983123779297,
      "margin_dpo/margin_std": 37.08135223388672,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -0.25167274475097656,
      "KL/mean": -16.53740119934082,
      "KL/rejected_KL_mean": -32.82312774658203,
      "KL/std": 33.70093536376953,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.0175673458725214,
      "fcm_dpo/delta": -0.0785430520772934,
      "fcm_dpo/margin": 32.571449279785156,
      "fcm_dpo/q_t": 0.37332215905189514,
      "grad_norm": 51.408931732177734,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": 1.4026705026626587,
      "logits/rejected": 1.4513009786605835,
      "logps/chosen": -316.1223449707031,
      "logps/ref_chosen": -315.8706970214844,
      "logps/ref_rejected": -304.3729248046875,
      "logps/rejected": -337.196044921875,
      "loss": 4.0258,
      "margin_dpo/margin_mean": 32.57144546508789,
      "margin_dpo/margin_std": 36.59093475341797,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -2.2900142669677734,
      "KL/mean": -18.87673568725586,
      "KL/rejected_KL_mean": -35.463470458984375,
      "KL/std": 35.366600036621094,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.016178620979189873,
      "fcm_dpo/delta": -0.08305059373378754,
      "fcm_dpo/margin": 33.173458099365234,
      "fcm_dpo/q_t": 0.37978070974349976,
      "grad_norm": 30.617616653442383,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 1.4732431173324585,
      "logits/rejected": 1.4332544803619385,
      "logps/chosen": -313.3871765136719,
      "logps/ref_chosen": -311.09716796875,
      "logps/ref_rejected": -266.01959228515625,
      "logps/rejected": -301.4830322265625,
      "loss": 4.1132,
      "margin_dpo/margin_mean": 33.173458099365234,
      "margin_dpo/margin_std": 40.455406188964844,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -1.3574085235595703,
      "KL/mean": -17.776784896850586,
      "KL/rejected_KL_mean": -34.19615173339844,
      "KL/std": 35.01076889038086,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.015465817414224148,
      "fcm_dpo/delta": -0.06945983320474625,
      "fcm_dpo/margin": 32.8387565612793,
      "fcm_dpo/q_t": 0.38557150959968567,
      "grad_norm": 45.441001892089844,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": 1.6924965381622314,
      "logits/rejected": 1.5911293029785156,
      "logps/chosen": -286.097900390625,
      "logps/ref_chosen": -284.740478515625,
      "logps/ref_rejected": -237.64117431640625,
      "logps/rejected": -271.8373107910156,
      "loss": 4.1693,
      "margin_dpo/margin_mean": 32.8387565612793,
      "margin_dpo/margin_std": 40.00797653198242,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -9.453258514404297,
      "KL/mean": -20.27108383178711,
      "KL/rejected_KL_mean": -31.08891487121582,
      "KL/std": 34.29841613769531,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.01496600266546011,
      "fcm_dpo/delta": 0.02908795326948166,
      "fcm_dpo/margin": 21.635650634765625,
      "fcm_dpo/q_t": 0.42855706810951233,
      "grad_norm": 38.83654022216797,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": 1.6364089250564575,
      "logits/rejected": 1.6656599044799805,
      "logps/chosen": -282.0661315917969,
      "logps/ref_chosen": -272.6128845214844,
      "logps/ref_rejected": -249.99725341796875,
      "logps/rejected": -281.0861511230469,
      "loss": 4.7714,
      "margin_dpo/margin_mean": 21.63565444946289,
      "margin_dpo/margin_std": 42.220359802246094,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -3.471952438354492,
      "KL/mean": -17.18140411376953,
      "KL/rejected_KL_mean": -30.890859603881836,
      "KL/std": 33.61980056762695,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.015404448844492435,
      "fcm_dpo/delta": 0.005553007125854492,
      "fcm_dpo/margin": 27.41891860961914,
      "fcm_dpo/q_t": 0.4044422507286072,
      "grad_norm": 33.08731460571289,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": 1.2327741384506226,
      "logits/rejected": 1.2023051977157593,
      "logps/chosen": -277.9077453613281,
      "logps/ref_chosen": -274.435791015625,
      "logps/ref_rejected": -257.8125305175781,
      "logps/rejected": -288.7033996582031,
      "loss": 4.3915,
      "margin_dpo/margin_mean": 27.41891860961914,
      "margin_dpo/margin_std": 38.73517608642578,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -5.218334197998047,
      "KL/mean": -17.218442916870117,
      "KL/rejected_KL_mean": -29.21855354309082,
      "KL/std": 33.58876037597656,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.016162222251296043,
      "fcm_dpo/delta": 0.11570519208908081,
      "fcm_dpo/margin": 24.000215530395508,
      "fcm_dpo/q_t": 0.41404134035110474,
      "grad_norm": 39.777565002441406,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": 1.3204677104949951,
      "logits/rejected": 1.3489099740982056,
      "logps/chosen": -266.5296325683594,
      "logps/ref_chosen": -261.3113098144531,
      "logps/ref_rejected": -231.152587890625,
      "logps/rejected": -260.3711242675781,
      "loss": 4.5646,
      "margin_dpo/margin_mean": 24.00021743774414,
      "margin_dpo/margin_std": 40.29976272583008,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -7.040529251098633,
      "KL/mean": -20.12958526611328,
      "KL/rejected_KL_mean": -33.218650817871094,
      "KL/std": 35.52104568481445,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.01697562076151371,
      "fcm_dpo/delta": 0.01420489139854908,
      "fcm_dpo/margin": 26.178115844726562,
      "fcm_dpo/q_t": 0.4018368124961853,
      "grad_norm": 40.875267028808594,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": 1.5299392938613892,
      "logits/rejected": 1.4864516258239746,
      "logps/chosen": -275.50775146484375,
      "logps/ref_chosen": -268.46722412109375,
      "logps/ref_rejected": -240.27597045898438,
      "logps/rejected": -273.4945983886719,
      "loss": 4.4093,
      "margin_dpo/margin_mean": 26.178115844726562,
      "margin_dpo/margin_std": 39.69072341918945,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -7.475971221923828,
      "KL/mean": -20.887737274169922,
      "KL/rejected_KL_mean": -34.29948425292969,
      "KL/std": 36.88992691040039,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.01687629148364067,
      "fcm_dpo/delta": 0.008671794086694717,
      "fcm_dpo/margin": 26.82352638244629,
      "fcm_dpo/q_t": 0.4005794823169708,
      "grad_norm": 34.96428298950195,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 1.7428663969039917,
      "logits/rejected": 1.8781514167785645,
      "logps/chosen": -292.5811767578125,
      "logps/ref_chosen": -285.1051940917969,
      "logps/ref_rejected": -281.2331848144531,
      "logps/rejected": -315.5326843261719,
      "loss": 4.4823,
      "margin_dpo/margin_mean": 26.82352638244629,
      "margin_dpo/margin_std": 44.07164764404297,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -4.343530654907227,
      "KL/mean": -17.984128952026367,
      "KL/rejected_KL_mean": -31.624727249145508,
      "KL/std": 34.537696838378906,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.017417775467038155,
      "fcm_dpo/delta": -0.01743229292333126,
      "fcm_dpo/margin": 27.28118896484375,
      "fcm_dpo/q_t": 0.3959423899650574,
      "grad_norm": 38.99803924560547,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 1.4180138111114502,
      "logits/rejected": 1.3818966150283813,
      "logps/chosen": -278.4561462402344,
      "logps/ref_chosen": -274.1126403808594,
      "logps/ref_rejected": -256.7730407714844,
      "logps/rejected": -288.3977966308594,
      "loss": 4.3295,
      "margin_dpo/margin_mean": 27.281192779541016,
      "margin_dpo/margin_std": 38.777313232421875,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -10.650005340576172,
      "KL/mean": -24.929529190063477,
      "KL/rejected_KL_mean": -39.20906448364258,
      "KL/std": 36.48032760620117,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.016790900379419327,
      "fcm_dpo/delta": 0.01933576725423336,
      "fcm_dpo/margin": 28.559059143066406,
      "fcm_dpo/q_t": 0.39507779479026794,
      "grad_norm": 40.71644973754883,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": 1.6661081314086914,
      "logits/rejected": 1.6405282020568848,
      "logps/chosen": -300.21575927734375,
      "logps/ref_chosen": -289.56573486328125,
      "logps/ref_rejected": -277.9891662597656,
      "logps/rejected": -317.1982116699219,
      "loss": 4.4159,
      "margin_dpo/margin_mean": 28.559057235717773,
      "margin_dpo/margin_std": 44.25648880004883,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -10.063196182250977,
      "KL/mean": -21.79047966003418,
      "KL/rejected_KL_mean": -33.51776123046875,
      "KL/std": 36.71989440917969,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.017224524170160294,
      "fcm_dpo/delta": 0.0054787155240774155,
      "fcm_dpo/margin": 23.45456886291504,
      "fcm_dpo/q_t": 0.41194143891334534,
      "grad_norm": 53.012786865234375,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": 1.4369432926177979,
      "logits/rejected": 1.419782280921936,
      "logps/chosen": -280.2471008300781,
      "logps/ref_chosen": -270.1839294433594,
      "logps/ref_rejected": -245.86949157714844,
      "logps/rejected": -279.3872375488281,
      "loss": 4.6084,
      "margin_dpo/margin_mean": 23.45456886291504,
      "margin_dpo/margin_std": 42.15504455566406,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -7.109289169311523,
      "KL/mean": -21.708890914916992,
      "KL/rejected_KL_mean": -36.308494567871094,
      "KL/std": 36.807960510253906,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.01777733489871025,
      "fcm_dpo/delta": -0.02218419685959816,
      "fcm_dpo/margin": 29.199207305908203,
      "fcm_dpo/q_t": 0.38586610555648804,
      "grad_norm": 45.14458465576172,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": 1.3435295820236206,
      "logits/rejected": 1.3231405019760132,
      "logps/chosen": -267.79571533203125,
      "logps/ref_chosen": -260.6864318847656,
      "logps/ref_rejected": -245.99508666992188,
      "logps/rejected": -282.3035583496094,
      "loss": 4.2077,
      "margin_dpo/margin_mean": 29.19921112060547,
      "margin_dpo/margin_std": 37.18690872192383,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -5.575187683105469,
      "KL/mean": -22.053171157836914,
      "KL/rejected_KL_mean": -38.53115463256836,
      "KL/std": 37.32596969604492,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.016929732635617256,
      "fcm_dpo/delta": -0.06079362705349922,
      "fcm_dpo/margin": 32.955970764160156,
      "fcm_dpo/q_t": 0.37992674112319946,
      "grad_norm": 33.86513137817383,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": 1.6484287977218628,
      "logits/rejected": 1.5580581426620483,
      "logps/chosen": -303.489013671875,
      "logps/ref_chosen": -297.9138488769531,
      "logps/ref_rejected": -253.39952087402344,
      "logps/rejected": -291.9306640625,
      "loss": 4.201,
      "margin_dpo/margin_mean": 32.955970764160156,
      "margin_dpo/margin_std": 46.37726974487305,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -4.775520324707031,
      "KL/mean": -21.39520835876465,
      "KL/rejected_KL_mean": -38.01490020751953,
      "KL/std": 38.70463943481445,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.016217980533838272,
      "fcm_dpo/delta": -0.042945388704538345,
      "fcm_dpo/margin": 33.2393798828125,
      "fcm_dpo/q_t": 0.38274601101875305,
      "grad_norm": 37.69220733642578,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": 1.3671672344207764,
      "logits/rejected": 1.294467806816101,
      "logps/chosen": -314.701904296875,
      "logps/ref_chosen": -309.9263916015625,
      "logps/ref_rejected": -253.452392578125,
      "logps/rejected": -291.46728515625,
      "loss": 4.3015,
      "margin_dpo/margin_mean": 33.2393798828125,
      "margin_dpo/margin_std": 48.15922927856445,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -7.89354133605957,
      "KL/mean": -22.9422607421875,
      "KL/rejected_KL_mean": -37.99098205566406,
      "KL/std": 38.2701301574707,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.015520069748163223,
      "fcm_dpo/delta": 0.03391147404909134,
      "fcm_dpo/margin": 30.097444534301758,
      "fcm_dpo/q_t": 0.3976669907569885,
      "grad_norm": 51.30897903442383,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 1.2850680351257324,
      "logits/rejected": 1.3168834447860718,
      "logps/chosen": -267.7071838378906,
      "logps/ref_chosen": -259.8136291503906,
      "logps/ref_rejected": -245.84947204589844,
      "logps/rejected": -283.8404541015625,
      "loss": 4.3213,
      "margin_dpo/margin_mean": 30.097444534301758,
      "margin_dpo/margin_std": 42.506717681884766,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -7.419759750366211,
      "KL/mean": -23.4638671875,
      "KL/rejected_KL_mean": -39.507991790771484,
      "KL/std": 38.396846771240234,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.015726212412118912,
      "fcm_dpo/delta": -0.0055916886776685715,
      "fcm_dpo/margin": 32.08823013305664,
      "fcm_dpo/q_t": 0.3862563967704773,
      "grad_norm": 40.72343063354492,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": 1.6017581224441528,
      "logits/rejected": 1.6873362064361572,
      "logps/chosen": -272.7734375,
      "logps/ref_chosen": -265.35369873046875,
      "logps/ref_rejected": -247.9174346923828,
      "logps/rejected": -287.4254455566406,
      "loss": 4.1596,
      "margin_dpo/margin_mean": 32.08823013305664,
      "margin_dpo/margin_std": 39.783416748046875,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -6.57206916809082,
      "KL/mean": -21.623151779174805,
      "KL/rejected_KL_mean": -36.674224853515625,
      "KL/std": 38.30681610107422,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.01620825007557869,
      "fcm_dpo/delta": 0.012122666463255882,
      "fcm_dpo/margin": 30.10215950012207,
      "fcm_dpo/q_t": 0.39220738410949707,
      "grad_norm": 51.266414642333984,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": 1.5376665592193604,
      "logits/rejected": 1.5359485149383545,
      "logps/chosen": -294.7952880859375,
      "logps/ref_chosen": -288.2232360839844,
      "logps/ref_rejected": -270.7300109863281,
      "logps/rejected": -307.40423583984375,
      "loss": 4.3132,
      "margin_dpo/margin_mean": 30.10215950012207,
      "margin_dpo/margin_std": 44.46660232543945,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -11.211172103881836,
      "KL/mean": -18.055570602416992,
      "KL/rejected_KL_mean": -24.899959564208984,
      "KL/std": 36.8315315246582,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.016384411603212357,
      "fcm_dpo/delta": 0.023327043280005455,
      "fcm_dpo/margin": 13.688791275024414,
      "fcm_dpo/q_t": 0.4504554271697998,
      "grad_norm": 51.26575469970703,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 1.5406413078308105,
      "logits/rejected": 1.5799825191497803,
      "logps/chosen": -292.8309020996094,
      "logps/ref_chosen": -281.61968994140625,
      "logps/ref_rejected": -271.03973388671875,
      "logps/rejected": -295.939697265625,
      "loss": 5.2032,
      "margin_dpo/margin_mean": 13.688793182373047,
      "margin_dpo/margin_std": 43.2988395690918,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -14.379045486450195,
      "KL/mean": -27.16608428955078,
      "KL/rejected_KL_mean": -39.953125,
      "KL/std": 37.585323333740234,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.01661657728254795,
      "fcm_dpo/delta": 0.038591139018535614,
      "fcm_dpo/margin": 25.574079513549805,
      "fcm_dpo/q_t": 0.40793246030807495,
      "grad_norm": 48.17567825317383,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 1.1857041120529175,
      "logits/rejected": 1.3260219097137451,
      "logps/chosen": -256.84222412109375,
      "logps/ref_chosen": -242.4631805419922,
      "logps/ref_rejected": -244.0020751953125,
      "logps/rejected": -283.9552001953125,
      "loss": 4.5915,
      "margin_dpo/margin_mean": 25.57407569885254,
      "margin_dpo/margin_std": 46.39384841918945,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -7.556356430053711,
      "KL/mean": -23.469148635864258,
      "KL/rejected_KL_mean": -39.38195037841797,
      "KL/std": 38.92152404785156,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.01681143417954445,
      "fcm_dpo/delta": -0.03762173652648926,
      "fcm_dpo/margin": 31.825603485107422,
      "fcm_dpo/q_t": 0.3830254375934601,
      "grad_norm": 62.542076110839844,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": 1.671151876449585,
      "logits/rejected": 1.6955801248550415,
      "logps/chosen": -299.5449523925781,
      "logps/ref_chosen": -291.98858642578125,
      "logps/ref_rejected": -273.35955810546875,
      "logps/rejected": -312.74151611328125,
      "loss": 4.1734,
      "margin_dpo/margin_mean": 31.825605392456055,
      "margin_dpo/margin_std": 42.62792205810547,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -13.06032943725586,
      "KL/mean": -26.578659057617188,
      "KL/rejected_KL_mean": -40.096988677978516,
      "KL/std": 38.98527526855469,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.01652543991804123,
      "fcm_dpo/delta": -0.007837500423192978,
      "fcm_dpo/margin": 27.036659240722656,
      "fcm_dpo/q_t": 0.40340811014175415,
      "grad_norm": 51.40494918823242,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": 1.213884711265564,
      "logits/rejected": 1.1890496015548706,
      "logps/chosen": -288.0487976074219,
      "logps/ref_chosen": -274.98846435546875,
      "logps/ref_rejected": -254.75782775878906,
      "logps/rejected": -294.85479736328125,
      "loss": 4.4977,
      "margin_dpo/margin_mean": 27.036659240722656,
      "margin_dpo/margin_std": 45.27447509765625,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -5.358608245849609,
      "KL/mean": -21.943449020385742,
      "KL/rejected_KL_mean": -38.528297424316406,
      "KL/std": 38.440025329589844,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.015785997733473778,
      "fcm_dpo/delta": -0.0657602846622467,
      "fcm_dpo/margin": 33.16969299316406,
      "fcm_dpo/q_t": 0.38467109203338623,
      "grad_norm": 32.09825897216797,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": 1.2250244617462158,
      "logits/rejected": 1.2877085208892822,
      "logps/chosen": -256.9847717285156,
      "logps/ref_chosen": -251.62615966796875,
      "logps/ref_rejected": -251.9549560546875,
      "logps/rejected": -290.4832458496094,
      "loss": 4.1542,
      "margin_dpo/margin_mean": 33.16969680786133,
      "margin_dpo/margin_std": 42.6336555480957,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -10.167257308959961,
      "KL/mean": -25.69939422607422,
      "KL/rejected_KL_mean": -41.231529235839844,
      "KL/std": 40.50013732910156,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.014983631670475006,
      "fcm_dpo/delta": -0.038476720452308655,
      "fcm_dpo/margin": 31.064266204833984,
      "fcm_dpo/q_t": 0.39651361107826233,
      "grad_norm": 33.54485321044922,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": 1.4084360599517822,
      "logits/rejected": 1.4557099342346191,
      "logps/chosen": -288.1038513183594,
      "logps/ref_chosen": -277.9366149902344,
      "logps/ref_rejected": -262.689453125,
      "logps/rejected": -303.92095947265625,
      "loss": 4.3433,
      "margin_dpo/margin_mean": 31.064268112182617,
      "margin_dpo/margin_std": 44.31740951538086,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -2.390134811401367,
      "KL/mean": -18.78409194946289,
      "KL/rejected_KL_mean": -35.17805480957031,
      "KL/std": 39.016632080078125,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.015158112160861492,
      "fcm_dpo/delta": 0.0013839304447174072,
      "fcm_dpo/margin": 32.78792953491211,
      "fcm_dpo/q_t": 0.3880288004875183,
      "grad_norm": 42.305423736572266,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": 1.6243656873703003,
      "logits/rejected": 1.6156667470932007,
      "logps/chosen": -272.8350524902344,
      "logps/ref_chosen": -270.4449157714844,
      "logps/ref_rejected": -259.5942077636719,
      "logps/rejected": -294.77227783203125,
      "loss": 4.2168,
      "margin_dpo/margin_mean": 32.78792953491211,
      "margin_dpo/margin_std": 41.83787155151367,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -8.872474670410156,
      "KL/mean": -22.262086868286133,
      "KL/rejected_KL_mean": -35.651702880859375,
      "KL/std": 39.52783966064453,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.015331543982028961,
      "fcm_dpo/delta": 0.04699525982141495,
      "fcm_dpo/margin": 26.779220581054688,
      "fcm_dpo/q_t": 0.40969130396842957,
      "grad_norm": 40.55670928955078,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": 1.4803292751312256,
      "logits/rejected": 1.56295645236969,
      "logps/chosen": -278.7123718261719,
      "logps/ref_chosen": -269.83990478515625,
      "logps/ref_rejected": -246.33746337890625,
      "logps/rejected": -281.9891662597656,
      "loss": 4.5925,
      "margin_dpo/margin_mean": 26.779220581054688,
      "margin_dpo/margin_std": 47.24227523803711,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -4.476598739624023,
      "KL/mean": -20.00168800354004,
      "KL/rejected_KL_mean": -35.52677536010742,
      "KL/std": 39.31858825683594,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.015467698685824871,
      "fcm_dpo/delta": -0.0235493965446949,
      "fcm_dpo/margin": 31.0501708984375,
      "fcm_dpo/q_t": 0.3943493366241455,
      "grad_norm": 33.864437103271484,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": 1.202286958694458,
      "logits/rejected": 1.1624791622161865,
      "logps/chosen": -302.0105285644531,
      "logps/ref_chosen": -297.533935546875,
      "logps/ref_rejected": -270.3407897949219,
      "logps/rejected": -305.8675537109375,
      "loss": 4.2984,
      "margin_dpo/margin_mean": 31.0501708984375,
      "margin_dpo/margin_std": 43.687923431396484,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -6.717525482177734,
      "KL/mean": -20.820730209350586,
      "KL/rejected_KL_mean": -34.92393493652344,
      "KL/std": 39.19996643066406,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.015903333202004433,
      "fcm_dpo/delta": 0.05257945507764816,
      "fcm_dpo/margin": 28.206409454345703,
      "fcm_dpo/q_t": 0.40036481618881226,
      "grad_norm": 41.2359504699707,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": 1.2838945388793945,
      "logits/rejected": 1.3247201442718506,
      "logps/chosen": -247.05636596679688,
      "logps/ref_chosen": -240.33883666992188,
      "logps/ref_rejected": -220.71717834472656,
      "logps/rejected": -255.64111328125,
      "loss": 4.4899,
      "margin_dpo/margin_mean": 28.206409454345703,
      "margin_dpo/margin_std": 47.0638542175293,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -4.442609786987305,
      "KL/mean": -21.011398315429688,
      "KL/rejected_KL_mean": -37.58018493652344,
      "KL/std": 41.059810638427734,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.015471951104700565,
      "fcm_dpo/delta": -0.057666558772325516,
      "fcm_dpo/margin": 33.13755798339844,
      "fcm_dpo/q_t": 0.3866129219532013,
      "grad_norm": 47.2930793762207,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 1.7887153625488281,
      "logits/rejected": 1.8097623586654663,
      "logps/chosen": -313.3357238769531,
      "logps/ref_chosen": -308.8930969238281,
      "logps/ref_rejected": -278.47869873046875,
      "logps/rejected": -316.05889892578125,
      "loss": 4.3149,
      "margin_dpo/margin_mean": 33.1375617980957,
      "margin_dpo/margin_std": 48.78028869628906,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -5.137346267700195,
      "KL/mean": -20.851608276367188,
      "KL/rejected_KL_mean": -36.56586837768555,
      "KL/std": 39.21592330932617,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.015665508806705475,
      "fcm_dpo/delta": 0.0064544775523245335,
      "fcm_dpo/margin": 31.428512573242188,
      "fcm_dpo/q_t": 0.39280620217323303,
      "grad_norm": 44.01938247680664,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.4176539182662964,
      "logits/rejected": 1.429800271987915,
      "logps/chosen": -280.5655517578125,
      "logps/ref_chosen": -275.42822265625,
      "logps/ref_rejected": -251.50906372070312,
      "logps/rejected": -288.074951171875,
      "loss": 4.2761,
      "margin_dpo/margin_mean": 31.428512573242188,
      "margin_dpo/margin_std": 44.101932525634766,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -3.5435142517089844,
      "KL/mean": -18.24382972717285,
      "KL/rejected_KL_mean": -32.944149017333984,
      "KL/std": 39.294761657714844,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.015083063393831253,
      "fcm_dpo/delta": -0.033804964274168015,
      "fcm_dpo/margin": 29.400625228881836,
      "fcm_dpo/q_t": 0.40162086486816406,
      "grad_norm": 74.38374328613281,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": 1.5255744457244873,
      "logits/rejected": 1.5420976877212524,
      "logps/chosen": -292.9262390136719,
      "logps/ref_chosen": -289.38275146484375,
      "logps/ref_rejected": -274.895751953125,
      "logps/rejected": -307.83990478515625,
      "loss": 4.4227,
      "margin_dpo/margin_mean": 29.400625228881836,
      "margin_dpo/margin_std": 44.6794319152832,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -4.17735481262207,
      "KL/mean": -20.349050521850586,
      "KL/rejected_KL_mean": -36.520751953125,
      "KL/std": 39.43000793457031,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.01450867298990488,
      "fcm_dpo/delta": -0.011511165648698807,
      "fcm_dpo/margin": 32.34339904785156,
      "fcm_dpo/q_t": 0.3960791230201721,
      "grad_norm": 38.35572814941406,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": 1.3950462341308594,
      "logits/rejected": 1.505737066268921,
      "logps/chosen": -289.16583251953125,
      "logps/ref_chosen": -284.98846435546875,
      "logps/ref_rejected": -273.5077819824219,
      "logps/rejected": -310.0285339355469,
      "loss": 4.3494,
      "margin_dpo/margin_mean": 32.34339904785156,
      "margin_dpo/margin_std": 47.30317687988281,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -4.125665664672852,
      "KL/mean": -21.573137283325195,
      "KL/rejected_KL_mean": -39.02061462402344,
      "KL/std": 41.27991485595703,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.014382775872945786,
      "fcm_dpo/delta": -0.0584682896733284,
      "fcm_dpo/margin": 34.894962310791016,
      "fcm_dpo/q_t": 0.38711440563201904,
      "grad_norm": 35.132301330566406,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": 1.6787828207015991,
      "logits/rejected": 1.774768352508545,
      "logps/chosen": -266.8052673339844,
      "logps/ref_chosen": -262.6795959472656,
      "logps/ref_rejected": -236.13792419433594,
      "logps/rejected": -275.1585388183594,
      "loss": 4.2175,
      "margin_dpo/margin_mean": 34.894962310791016,
      "margin_dpo/margin_std": 45.68421936035156,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -1.4391651153564453,
      "KL/mean": -21.610889434814453,
      "KL/rejected_KL_mean": -41.78260803222656,
      "KL/std": 40.063053131103516,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.013755477964878082,
      "fcm_dpo/delta": -0.058670930564403534,
      "fcm_dpo/margin": 40.34345245361328,
      "fcm_dpo/q_t": 0.37531131505966187,
      "grad_norm": 31.039329528808594,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": 1.1244385242462158,
      "logits/rejected": 1.220582365989685,
      "logps/chosen": -247.8386993408203,
      "logps/ref_chosen": -246.39950561523438,
      "logps/ref_rejected": -247.4836883544922,
      "logps/rejected": -289.26629638671875,
      "loss": 3.9964,
      "margin_dpo/margin_mean": 40.343448638916016,
      "margin_dpo/margin_std": 44.97233581542969,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": 0.639434814453125,
      "KL/mean": -17.675588607788086,
      "KL/rejected_KL_mean": -35.990623474121094,
      "KL/std": 39.641448974609375,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.013395718298852444,
      "fcm_dpo/delta": 0.009209694340825081,
      "fcm_dpo/margin": 36.630062103271484,
      "fcm_dpo/q_t": 0.3896031975746155,
      "grad_norm": 31.35970115661621,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": 1.334702730178833,
      "logits/rejected": 1.4313724040985107,
      "logps/chosen": -288.2806396484375,
      "logps/ref_chosen": -288.9200439453125,
      "logps/ref_rejected": -291.69305419921875,
      "logps/rejected": -327.6836853027344,
      "loss": 4.1676,
      "margin_dpo/margin_mean": 36.630062103271484,
      "margin_dpo/margin_std": 44.49883270263672,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -7.824338912963867,
      "KL/mean": -21.00406837463379,
      "KL/rejected_KL_mean": -34.18379592895508,
      "KL/std": 39.38002395629883,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.014148312620818615,
      "fcm_dpo/delta": 0.08708544075489044,
      "fcm_dpo/margin": 26.359458923339844,
      "fcm_dpo/q_t": 0.41661202907562256,
      "grad_norm": 36.54499435424805,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": 1.2699054479599,
      "logits/rejected": 1.312665343284607,
      "logps/chosen": -264.9092712402344,
      "logps/ref_chosen": -257.08489990234375,
      "logps/ref_rejected": -251.61532592773438,
      "logps/rejected": -285.79913330078125,
      "loss": 4.6816,
      "margin_dpo/margin_mean": 26.359460830688477,
      "margin_dpo/margin_std": 49.395328521728516,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -11.442520141601562,
      "KL/mean": -23.775354385375977,
      "KL/rejected_KL_mean": -36.108192443847656,
      "KL/std": 41.17635726928711,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.014711221680045128,
      "fcm_dpo/delta": 0.01737670972943306,
      "fcm_dpo/margin": 24.665678024291992,
      "fcm_dpo/q_t": 0.42009076476097107,
      "grad_norm": 46.014156341552734,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": 1.529653549194336,
      "logits/rejected": 1.4980154037475586,
      "logps/chosen": -272.18389892578125,
      "logps/ref_chosen": -260.7413635253906,
      "logps/ref_rejected": -227.4325408935547,
      "logps/rejected": -263.5407409667969,
      "loss": 4.7993,
      "margin_dpo/margin_mean": 24.665678024291992,
      "margin_dpo/margin_std": 50.9749641418457,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -4.652675628662109,
      "KL/mean": -23.82745361328125,
      "KL/rejected_KL_mean": -43.00222396850586,
      "KL/std": 43.0183219909668,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.014185795560479164,
      "fcm_dpo/delta": -0.04878905415534973,
      "fcm_dpo/margin": 38.34954833984375,
      "fcm_dpo/q_t": 0.379115492105484,
      "grad_norm": 36.335636138916016,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": 1.3838316202163696,
      "logits/rejected": 1.4707328081130981,
      "logps/chosen": -307.6001892089844,
      "logps/ref_chosen": -302.947509765625,
      "logps/ref_rejected": -294.1771545410156,
      "logps/rejected": -337.1794128417969,
      "loss": 4.0849,
      "margin_dpo/margin_mean": 38.34954833984375,
      "margin_dpo/margin_std": 46.31309509277344,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -5.23646354675293,
      "KL/mean": -22.495336532592773,
      "KL/rejected_KL_mean": -39.754215240478516,
      "KL/std": 39.577083587646484,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.013934066519141197,
      "fcm_dpo/delta": -0.02017383836209774,
      "fcm_dpo/margin": 34.51774978637695,
      "fcm_dpo/q_t": 0.39311766624450684,
      "grad_norm": 36.09855651855469,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 1.5009599924087524,
      "logits/rejected": 1.5808147192001343,
      "logps/chosen": -263.1345520019531,
      "logps/ref_chosen": -257.8981018066406,
      "logps/ref_rejected": -244.9084014892578,
      "logps/rejected": -284.66259765625,
      "loss": 4.2898,
      "margin_dpo/margin_mean": 34.51774978637695,
      "margin_dpo/margin_std": 48.224239349365234,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -13.562841415405273,
      "KL/mean": -30.06568717956543,
      "KL/rejected_KL_mean": -46.56853103637695,
      "KL/std": 43.030670166015625,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.013896746560931206,
      "fcm_dpo/delta": 0.001169930212199688,
      "fcm_dpo/margin": 33.005699157714844,
      "fcm_dpo/q_t": 0.398587703704834,
      "grad_norm": 39.26085662841797,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 1.4259493350982666,
      "logits/rejected": 1.4875670671463013,
      "logps/chosen": -303.7575378417969,
      "logps/ref_chosen": -290.1946716308594,
      "logps/ref_rejected": -290.98883056640625,
      "logps/rejected": -337.557373046875,
      "loss": 4.3189,
      "margin_dpo/margin_mean": 33.00570297241211,
      "margin_dpo/margin_std": 45.40581512451172,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -11.37614631652832,
      "KL/mean": -27.142377853393555,
      "KL/rejected_KL_mean": -42.908599853515625,
      "KL/std": 43.22870635986328,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.014085488393902779,
      "fcm_dpo/delta": 0.05853241682052612,
      "fcm_dpo/margin": 31.532466888427734,
      "fcm_dpo/q_t": 0.40331754088401794,
      "grad_norm": 42.3298454284668,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": 1.4406760931015015,
      "logits/rejected": 1.4661328792572021,
      "logps/chosen": -320.2935791015625,
      "logps/ref_chosen": -308.91741943359375,
      "logps/ref_rejected": -272.2177734375,
      "logps/rejected": -315.1263427734375,
      "loss": 4.4778,
      "margin_dpo/margin_mean": 31.532466888427734,
      "margin_dpo/margin_std": 51.825706481933594,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -14.743045806884766,
      "KL/mean": -31.079559326171875,
      "KL/rejected_KL_mean": -47.41609191894531,
      "KL/std": 40.386268615722656,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.014951983466744423,
      "fcm_dpo/delta": 0.00956372544169426,
      "fcm_dpo/margin": 32.67304992675781,
      "fcm_dpo/q_t": 0.3927711844444275,
      "grad_norm": 33.60694122314453,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 1.6486581563949585,
      "logits/rejected": 1.7128368616104126,
      "logps/chosen": -262.76507568359375,
      "logps/ref_chosen": -248.02200317382812,
      "logps/ref_rejected": -234.78541564941406,
      "logps/rejected": -282.2015380859375,
      "loss": 4.3386,
      "margin_dpo/margin_mean": 32.67304992675781,
      "margin_dpo/margin_std": 47.55571746826172,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -15.345441818237305,
      "KL/mean": -31.714027404785156,
      "KL/rejected_KL_mean": -48.082603454589844,
      "KL/std": 44.058631896972656,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.014476237818598747,
      "fcm_dpo/delta": -0.011985169723629951,
      "fcm_dpo/margin": 32.737159729003906,
      "fcm_dpo/q_t": 0.3977169096469879,
      "grad_norm": 53.27794647216797,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": 1.2630878686904907,
      "logits/rejected": 1.252713918685913,
      "logps/chosen": -279.834716796875,
      "logps/ref_chosen": -264.4892578125,
      "logps/ref_rejected": -255.43092346191406,
      "logps/rejected": -303.5135192871094,
      "loss": 4.4976,
      "margin_dpo/margin_mean": 32.737159729003906,
      "margin_dpo/margin_std": 55.19682693481445,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -14.861488342285156,
      "KL/mean": -27.649024963378906,
      "KL/rejected_KL_mean": -40.436561584472656,
      "KL/std": 40.931724548339844,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.015106920152902603,
      "fcm_dpo/delta": 0.017874840646982193,
      "fcm_dpo/margin": 25.575077056884766,
      "fcm_dpo/q_t": 0.41502171754837036,
      "grad_norm": 56.862125396728516,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": 1.6194286346435547,
      "logits/rejected": 1.6365852355957031,
      "logps/chosen": -316.3301696777344,
      "logps/ref_chosen": -301.46868896484375,
      "logps/ref_rejected": -274.2380676269531,
      "logps/rejected": -314.6746826171875,
      "loss": 4.668,
      "margin_dpo/margin_mean": 25.575077056884766,
      "margin_dpo/margin_std": 46.18408966064453,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -17.712047576904297,
      "KL/mean": -31.57440948486328,
      "KL/rejected_KL_mean": -45.436767578125,
      "KL/std": 41.607032775878906,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.015099598094820976,
      "fcm_dpo/delta": 0.0441029891371727,
      "fcm_dpo/margin": 27.724714279174805,
      "fcm_dpo/q_t": 0.41000330448150635,
      "grad_norm": 41.34486389160156,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": 1.3910651206970215,
      "logits/rejected": 1.4616739749908447,
      "logps/chosen": -284.2711181640625,
      "logps/ref_chosen": -266.5590515136719,
      "logps/ref_rejected": -280.12066650390625,
      "logps/rejected": -325.5574645996094,
      "loss": 4.6433,
      "margin_dpo/margin_mean": 27.724714279174805,
      "margin_dpo/margin_std": 52.186912536621094,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -15.981010437011719,
      "KL/mean": -31.164644241333008,
      "KL/rejected_KL_mean": -46.348289489746094,
      "KL/std": 39.44988250732422,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.01538790762424469,
      "fcm_dpo/delta": -0.007101214956492186,
      "fcm_dpo/margin": 30.367271423339844,
      "fcm_dpo/q_t": 0.39640742540359497,
      "grad_norm": 44.48409652709961,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": 1.206976294517517,
      "logits/rejected": 1.2523952722549438,
      "logps/chosen": -285.9173583984375,
      "logps/ref_chosen": -269.93634033203125,
      "logps/ref_rejected": -232.32330322265625,
      "logps/rejected": -278.6716003417969,
      "loss": 4.4422,
      "margin_dpo/margin_mean": 30.367271423339844,
      "margin_dpo/margin_std": 49.026466369628906,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -16.695110321044922,
      "KL/mean": -31.353757858276367,
      "KL/rejected_KL_mean": -46.01241683959961,
      "KL/std": 41.6192626953125,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.015901876613497734,
      "fcm_dpo/delta": 0.03297813609242439,
      "fcm_dpo/margin": 29.317310333251953,
      "fcm_dpo/q_t": 0.40031519532203674,
      "grad_norm": 45.97467803955078,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": 1.6034586429595947,
      "logits/rejected": 1.5711491107940674,
      "logps/chosen": -300.7406005859375,
      "logps/ref_chosen": -284.04547119140625,
      "logps/ref_rejected": -261.19378662109375,
      "logps/rejected": -307.2062072753906,
      "loss": 4.4885,
      "margin_dpo/margin_mean": 29.317312240600586,
      "margin_dpo/margin_std": 49.80121994018555,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -20.93366813659668,
      "KL/mean": -31.944856643676758,
      "KL/rejected_KL_mean": -42.95604705810547,
      "KL/std": 43.197059631347656,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.016051705926656723,
      "fcm_dpo/delta": 0.05727365240454674,
      "fcm_dpo/margin": 22.022388458251953,
      "fcm_dpo/q_t": 0.4241880774497986,
      "grad_norm": 48.64494705200195,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": 1.3685553073883057,
      "logits/rejected": 1.3306169509887695,
      "logps/chosen": -344.21490478515625,
      "logps/ref_chosen": -323.28125,
      "logps/ref_rejected": -274.2093200683594,
      "logps/rejected": -317.1653747558594,
      "loss": 4.8857,
      "margin_dpo/margin_mean": 22.02238655090332,
      "margin_dpo/margin_std": 50.32847595214844,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -16.306974411010742,
      "KL/mean": -29.675161361694336,
      "KL/rejected_KL_mean": -43.043357849121094,
      "KL/std": 42.26116180419922,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.016355913132429123,
      "fcm_dpo/delta": 0.013551563024520874,
      "fcm_dpo/margin": 26.73638916015625,
      "fcm_dpo/q_t": 0.40635085105895996,
      "grad_norm": 56.89421463012695,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 1.3439819812774658,
      "logits/rejected": 1.3048815727233887,
      "logps/chosen": -333.74237060546875,
      "logps/ref_chosen": -317.4353942871094,
      "logps/ref_rejected": -309.32403564453125,
      "logps/rejected": -352.367431640625,
      "loss": 4.5701,
      "margin_dpo/margin_mean": 26.736392974853516,
      "margin_dpo/margin_std": 47.0720100402832,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -13.452327728271484,
      "KL/mean": -32.588897705078125,
      "KL/rejected_KL_mean": -51.7254638671875,
      "KL/std": 45.26586151123047,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.01562550477683544,
      "fcm_dpo/delta": -0.1071409061551094,
      "fcm_dpo/margin": 38.27313232421875,
      "fcm_dpo/q_t": 0.37067312002182007,
      "grad_norm": 46.3168830871582,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 1.1085205078125,
      "logits/rejected": 1.2523558139801025,
      "logps/chosen": -278.9605407714844,
      "logps/ref_chosen": -265.50823974609375,
      "logps/ref_rejected": -275.1380310058594,
      "logps/rejected": -326.8634948730469,
      "loss": 4.1104,
      "margin_dpo/margin_mean": 38.27313232421875,
      "margin_dpo/margin_std": 50.77440643310547,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -10.37523078918457,
      "KL/mean": -26.09268569946289,
      "KL/rejected_KL_mean": -41.81013107299805,
      "KL/std": 42.26549530029297,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.015030492097139359,
      "fcm_dpo/delta": -0.014139977283775806,
      "fcm_dpo/margin": 31.43490219116211,
      "fcm_dpo/q_t": 0.39510560035705566,
      "grad_norm": 42.15945816040039,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": 1.316124439239502,
      "logits/rejected": 1.4355056285858154,
      "logps/chosen": -260.27655029296875,
      "logps/ref_chosen": -249.9013214111328,
      "logps/ref_rejected": -274.58245849609375,
      "logps/rejected": -316.392578125,
      "loss": 4.383,
      "margin_dpo/margin_mean": 31.434900283813477,
      "margin_dpo/margin_std": 47.02584457397461,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -16.569005966186523,
      "KL/mean": -32.62763214111328,
      "KL/rejected_KL_mean": -48.68626403808594,
      "KL/std": 44.5269775390625,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.01536480151116848,
      "fcm_dpo/delta": 0.00532765407115221,
      "fcm_dpo/margin": 32.117252349853516,
      "fcm_dpo/q_t": 0.3929012417793274,
      "grad_norm": 63.18702697753906,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": 1.4893841743469238,
      "logits/rejected": 1.452648639678955,
      "logps/chosen": -312.87493896484375,
      "logps/ref_chosen": -296.30596923828125,
      "logps/ref_rejected": -268.0640869140625,
      "logps/rejected": -316.75030517578125,
      "loss": 4.3452,
      "margin_dpo/margin_mean": 32.117252349853516,
      "margin_dpo/margin_std": 48.59520721435547,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -14.054628372192383,
      "KL/mean": -28.08951187133789,
      "KL/rejected_KL_mean": -42.12439727783203,
      "KL/std": 42.72382354736328,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.01564633660018444,
      "fcm_dpo/delta": 0.06218118965625763,
      "fcm_dpo/margin": 28.069774627685547,
      "fcm_dpo/q_t": 0.40510934591293335,
      "grad_norm": 65.71768188476562,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": 1.5372658967971802,
      "logits/rejected": 1.5897046327590942,
      "logps/chosen": -247.9376220703125,
      "logps/ref_chosen": -233.8829803466797,
      "logps/ref_rejected": -224.14334106445312,
      "logps/rejected": -266.2677307128906,
      "loss": 4.5378,
      "margin_dpo/margin_mean": 28.069772720336914,
      "margin_dpo/margin_std": 48.48573303222656,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -13.859193801879883,
      "KL/mean": -30.19824981689453,
      "KL/rejected_KL_mean": -46.53730010986328,
      "KL/std": 42.845977783203125,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.015710314735770226,
      "fcm_dpo/delta": -0.014896679669618607,
      "fcm_dpo/margin": 32.6781005859375,
      "fcm_dpo/q_t": 0.3911248445510864,
      "grad_norm": 41.92913818359375,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": 1.4306520223617554,
      "logits/rejected": 1.4379838705062866,
      "logps/chosen": -257.97052001953125,
      "logps/ref_chosen": -244.111328125,
      "logps/ref_rejected": -240.18373107910156,
      "logps/rejected": -286.72100830078125,
      "loss": 4.3774,
      "margin_dpo/margin_mean": 32.6781005859375,
      "margin_dpo/margin_std": 51.69953918457031,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -10.962980270385742,
      "KL/mean": -33.08680725097656,
      "KL/rejected_KL_mean": -55.210636138916016,
      "KL/std": 44.94438171386719,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.01434393972158432,
      "fcm_dpo/delta": -0.1839500367641449,
      "fcm_dpo/margin": 44.247676849365234,
      "fcm_dpo/q_t": 0.35920286178588867,
      "grad_norm": 38.719696044921875,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 1.368753433227539,
      "logits/rejected": 1.5311853885650635,
      "logps/chosen": -295.6435546875,
      "logps/ref_chosen": -284.6805725097656,
      "logps/ref_rejected": -282.1923828125,
      "logps/rejected": -337.4029541015625,
      "loss": 3.9277,
      "margin_dpo/margin_mean": 44.2476806640625,
      "margin_dpo/margin_std": 51.90763854980469,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -12.484909057617188,
      "KL/mean": -29.351577758789062,
      "KL/rejected_KL_mean": -46.21825408935547,
      "KL/std": 43.827003479003906,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.013538949191570282,
      "fcm_dpo/delta": 0.045136142522096634,
      "fcm_dpo/margin": 33.73333740234375,
      "fcm_dpo/q_t": 0.39992275834083557,
      "grad_norm": 31.333938598632812,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": 1.6786330938339233,
      "logits/rejected": 1.684911847114563,
      "logps/chosen": -292.7135314941406,
      "logps/ref_chosen": -280.2286376953125,
      "logps/ref_rejected": -280.8172912597656,
      "logps/rejected": -327.0355529785156,
      "loss": 4.3845,
      "margin_dpo/margin_mean": 33.73333740234375,
      "margin_dpo/margin_std": 49.82573699951172,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -6.231689453125,
      "KL/mean": -25.284042358398438,
      "KL/rejected_KL_mean": -44.336395263671875,
      "KL/std": 44.84620666503906,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.013426918536424637,
      "fcm_dpo/delta": -0.09871545433998108,
      "fcm_dpo/margin": 38.104705810546875,
      "fcm_dpo/q_t": 0.3882766664028168,
      "grad_norm": 36.2880744934082,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": 1.4756728410720825,
      "logits/rejected": 1.481322169303894,
      "logps/chosen": -303.23199462890625,
      "logps/ref_chosen": -297.0003356933594,
      "logps/ref_rejected": -279.1561279296875,
      "logps/rejected": -323.4925231933594,
      "loss": 4.2516,
      "margin_dpo/margin_mean": 38.10470962524414,
      "margin_dpo/margin_std": 52.553897857666016,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -14.645639419555664,
      "KL/mean": -29.94749641418457,
      "KL/rejected_KL_mean": -45.24934768676758,
      "KL/std": 44.20826721191406,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.013270791620016098,
      "fcm_dpo/delta": 0.03532692790031433,
      "fcm_dpo/margin": 30.603721618652344,
      "fcm_dpo/q_t": 0.40843456983566284,
      "grad_norm": 38.40726089477539,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": 1.4676116704940796,
      "logits/rejected": 1.5662314891815186,
      "logps/chosen": -286.1170654296875,
      "logps/ref_chosen": -271.471435546875,
      "logps/ref_rejected": -268.0870056152344,
      "logps/rejected": -313.3363952636719,
      "loss": 4.4503,
      "margin_dpo/margin_mean": 30.603721618652344,
      "margin_dpo/margin_std": 44.665855407714844,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -13.798622131347656,
      "KL/mean": -29.73674964904785,
      "KL/rejected_KL_mean": -45.67487716674805,
      "KL/std": 42.875526428222656,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.013386827893555164,
      "fcm_dpo/delta": 0.026485633105039597,
      "fcm_dpo/margin": 31.876262664794922,
      "fcm_dpo/q_t": 0.40528398752212524,
      "grad_norm": 52.97160339355469,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 1.549126386642456,
      "logits/rejected": 1.5453648567199707,
      "logps/chosen": -320.1564636230469,
      "logps/ref_chosen": -306.35784912109375,
      "logps/ref_rejected": -268.64776611328125,
      "logps/rejected": -314.3226318359375,
      "loss": 4.4874,
      "margin_dpo/margin_mean": 31.87626075744629,
      "margin_dpo/margin_std": 51.67327117919922,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -13.766092300415039,
      "KL/mean": -29.497844696044922,
      "KL/rejected_KL_mean": -45.22959518432617,
      "KL/std": 44.3377685546875,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.014049072749912739,
      "fcm_dpo/delta": 0.060421112924814224,
      "fcm_dpo/margin": 31.463499069213867,
      "fcm_dpo/q_t": 0.40344834327697754,
      "grad_norm": 38.997249603271484,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.4078246355056763,
      "logits/rejected": 1.4678350687026978,
      "logps/chosen": -283.7585754394531,
      "logps/ref_chosen": -269.9924621582031,
      "logps/ref_rejected": -271.3135681152344,
      "logps/rejected": -316.5431823730469,
      "loss": 4.482,
      "margin_dpo/margin_mean": 31.463499069213867,
      "margin_dpo/margin_std": 52.16905975341797,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -12.28561019897461,
      "KL/mean": -30.415241241455078,
      "KL/rejected_KL_mean": -48.54487609863281,
      "KL/std": 43.351680755615234,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.013804363086819649,
      "fcm_dpo/delta": -0.045740146189928055,
      "fcm_dpo/margin": 36.25926208496094,
      "fcm_dpo/q_t": 0.3881801962852478,
      "grad_norm": 35.4017333984375,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": 1.3231117725372314,
      "logits/rejected": 1.3943911790847778,
      "logps/chosen": -272.746337890625,
      "logps/ref_chosen": -260.460693359375,
      "logps/ref_rejected": -254.2005615234375,
      "logps/rejected": -302.7454528808594,
      "loss": 4.2829,
      "margin_dpo/margin_mean": 36.25926208496094,
      "margin_dpo/margin_std": 50.766666412353516,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -10.889986038208008,
      "KL/mean": -27.962642669677734,
      "KL/rejected_KL_mean": -45.03529739379883,
      "KL/std": 44.08222961425781,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.013741711154580116,
      "fcm_dpo/delta": -0.027668483555316925,
      "fcm_dpo/margin": 34.14532470703125,
      "fcm_dpo/q_t": 0.3947531580924988,
      "grad_norm": 52.28190994262695,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": 1.2091736793518066,
      "logits/rejected": 1.2438241243362427,
      "logps/chosen": -306.9862060546875,
      "logps/ref_chosen": -296.09625244140625,
      "logps/ref_rejected": -264.7806701660156,
      "logps/rejected": -309.8159484863281,
      "loss": 4.4105,
      "margin_dpo/margin_mean": 34.14532470703125,
      "margin_dpo/margin_std": 52.268306732177734,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -19.357511520385742,
      "KL/mean": -34.78844451904297,
      "KL/rejected_KL_mean": -50.21937942504883,
      "KL/std": 47.11204528808594,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.013923787511885166,
      "fcm_dpo/delta": 0.07317540049552917,
      "fcm_dpo/margin": 30.861858367919922,
      "fcm_dpo/q_t": 0.4055994749069214,
      "grad_norm": 44.67448425292969,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": 1.3354222774505615,
      "logits/rejected": 1.3308500051498413,
      "logps/chosen": -301.20697021484375,
      "logps/ref_chosen": -281.8494873046875,
      "logps/ref_rejected": -234.3673858642578,
      "logps/rejected": -284.5867919921875,
      "loss": 4.4795,
      "margin_dpo/margin_mean": 30.861858367919922,
      "margin_dpo/margin_std": 50.34181213378906,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -7.214826583862305,
      "KL/mean": -29.187097549438477,
      "KL/rejected_KL_mean": -51.15937805175781,
      "KL/std": 46.68242263793945,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.013611304573714733,
      "fcm_dpo/delta": -0.10693888366222382,
      "fcm_dpo/margin": 43.94454574584961,
      "fcm_dpo/q_t": 0.36742639541625977,
      "grad_norm": 41.020694732666016,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": 1.372054934501648,
      "logits/rejected": 1.4698508977890015,
      "logps/chosen": -255.25759887695312,
      "logps/ref_chosen": -248.0427703857422,
      "logps/ref_rejected": -263.464599609375,
      "logps/rejected": -314.6239929199219,
      "loss": 3.9986,
      "margin_dpo/margin_mean": 43.94454574584961,
      "margin_dpo/margin_std": 51.79741287231445,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -5.608304977416992,
      "KL/mean": -24.684911727905273,
      "KL/rejected_KL_mean": -43.761531829833984,
      "KL/std": 43.980751037597656,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.013166727498173714,
      "fcm_dpo/delta": -0.003459164872765541,
      "fcm_dpo/margin": 38.153221130371094,
      "fcm_dpo/q_t": 0.38826751708984375,
      "grad_norm": 35.93940734863281,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": 1.4362926483154297,
      "logits/rejected": 1.354178547859192,
      "logps/chosen": -316.8941955566406,
      "logps/ref_chosen": -311.2859191894531,
      "logps/ref_rejected": -268.4949645996094,
      "logps/rejected": -312.2564697265625,
      "loss": 4.2279,
      "margin_dpo/margin_mean": 38.153221130371094,
      "margin_dpo/margin_std": 50.497310638427734,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -11.799850463867188,
      "KL/mean": -32.273380279541016,
      "KL/rejected_KL_mean": -52.74691390991211,
      "KL/std": 47.105037689208984,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.01281946524977684,
      "fcm_dpo/delta": -0.08256404846906662,
      "fcm_dpo/margin": 40.947059631347656,
      "fcm_dpo/q_t": 0.3810916543006897,
      "grad_norm": 34.223690032958984,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": 1.3900225162506104,
      "logits/rejected": 1.3799841403961182,
      "logps/chosen": -282.0122985839844,
      "logps/ref_chosen": -270.21246337890625,
      "logps/ref_rejected": -239.1932830810547,
      "logps/rejected": -291.940185546875,
      "loss": 4.1356,
      "margin_dpo/margin_mean": 40.947059631347656,
      "margin_dpo/margin_std": 49.64616394042969,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -15.471633911132812,
      "KL/mean": -30.037256240844727,
      "KL/rejected_KL_mean": -44.602874755859375,
      "KL/std": 45.06877899169922,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.012730204500257969,
      "fcm_dpo/delta": 0.0930689349770546,
      "fcm_dpo/margin": 29.131240844726562,
      "fcm_dpo/q_t": 0.41686949133872986,
      "grad_norm": 31.213634490966797,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": 1.3804243803024292,
      "logits/rejected": 1.3768757581710815,
      "logps/chosen": -303.99444580078125,
      "logps/ref_chosen": -288.5228271484375,
      "logps/ref_rejected": -253.90597534179688,
      "logps/rejected": -298.50885009765625,
      "loss": 4.6095,
      "margin_dpo/margin_mean": 29.131240844726562,
      "margin_dpo/margin_std": 50.31481170654297,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -14.340890884399414,
      "KL/mean": -34.70792770385742,
      "KL/rejected_KL_mean": -55.074954986572266,
      "KL/std": 46.75768280029297,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.01314602792263031,
      "fcm_dpo/delta": -0.03805946558713913,
      "fcm_dpo/margin": 40.73405838012695,
      "fcm_dpo/q_t": 0.38235780596733093,
      "grad_norm": 33.81840133666992,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 1.2575311660766602,
      "logits/rejected": 1.3542282581329346,
      "logps/chosen": -286.235595703125,
      "logps/ref_chosen": -271.8947448730469,
      "logps/ref_rejected": -260.5709228515625,
      "logps/rejected": -315.6458740234375,
      "loss": 4.1607,
      "margin_dpo/margin_mean": 40.73405838012695,
      "margin_dpo/margin_std": 53.28679656982422,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -18.504587173461914,
      "KL/mean": -35.105079650878906,
      "KL/rejected_KL_mean": -51.70558166503906,
      "KL/std": 44.0412712097168,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.013084903359413147,
      "fcm_dpo/delta": 0.02494225651025772,
      "fcm_dpo/margin": 33.20099639892578,
      "fcm_dpo/q_t": 0.40340176224708557,
      "grad_norm": 42.67045211791992,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 1.2592557668685913,
      "logits/rejected": 1.3678982257843018,
      "logps/chosen": -288.6800537109375,
      "logps/ref_chosen": -270.17547607421875,
      "logps/ref_rejected": -257.6781311035156,
      "logps/rejected": -309.38372802734375,
      "loss": 4.389,
      "margin_dpo/margin_mean": 33.20099639892578,
      "margin_dpo/margin_std": 48.23236083984375,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -17.50929069519043,
      "KL/mean": -35.8475341796875,
      "KL/rejected_KL_mean": -54.18577575683594,
      "KL/std": 46.46718978881836,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.013369562104344368,
      "fcm_dpo/delta": 0.008456213399767876,
      "fcm_dpo/margin": 36.676483154296875,
      "fcm_dpo/q_t": 0.3922154903411865,
      "grad_norm": 38.1241340637207,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 1.4399981498718262,
      "logits/rejected": 1.59267258644104,
      "logps/chosen": -281.27154541015625,
      "logps/ref_chosen": -263.76226806640625,
      "logps/ref_rejected": -263.8758850097656,
      "logps/rejected": -318.0616455078125,
      "loss": 4.382,
      "margin_dpo/margin_mean": 36.676483154296875,
      "margin_dpo/margin_std": 56.2867431640625,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -16.186304092407227,
      "KL/mean": -34.22721862792969,
      "KL/rejected_KL_mean": -52.26814270019531,
      "KL/std": 44.33396530151367,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.013058342039585114,
      "fcm_dpo/delta": 0.029355261474847794,
      "fcm_dpo/margin": 36.08184051513672,
      "fcm_dpo/q_t": 0.3947826623916626,
      "grad_norm": 36.34197998046875,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": 1.1081104278564453,
      "logits/rejected": 1.0402052402496338,
      "logps/chosen": -290.8587341308594,
      "logps/ref_chosen": -274.67242431640625,
      "logps/ref_rejected": -242.1553955078125,
      "logps/rejected": -294.42352294921875,
      "loss": 4.3544,
      "margin_dpo/margin_mean": 36.08184051513672,
      "margin_dpo/margin_std": 52.198509216308594,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -18.420726776123047,
      "KL/mean": -35.76188659667969,
      "KL/rejected_KL_mean": -53.103050231933594,
      "KL/std": 48.401283264160156,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.013798207975924015,
      "fcm_dpo/delta": -0.023233484476804733,
      "fcm_dpo/margin": 34.682308197021484,
      "fcm_dpo/q_t": 0.396364688873291,
      "grad_norm": 38.87579345703125,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": 1.451599359512329,
      "logits/rejected": 1.5189623832702637,
      "logps/chosen": -313.60858154296875,
      "logps/ref_chosen": -295.1878662109375,
      "logps/ref_rejected": -271.3431091308594,
      "logps/rejected": -324.4461669921875,
      "loss": 4.4657,
      "margin_dpo/margin_mean": 34.682308197021484,
      "margin_dpo/margin_std": 55.476966857910156,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -16.307640075683594,
      "KL/mean": -33.97926712036133,
      "KL/rejected_KL_mean": -51.65089416503906,
      "KL/std": 46.13542938232422,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.013689286075532436,
      "fcm_dpo/delta": 0.015347246080636978,
      "fcm_dpo/margin": 35.34326171875,
      "fcm_dpo/q_t": 0.3928660750389099,
      "grad_norm": 40.221580505371094,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": 1.4175779819488525,
      "logits/rejected": 1.463738203048706,
      "logps/chosen": -306.07550048828125,
      "logps/ref_chosen": -289.767822265625,
      "logps/ref_rejected": -267.0824279785156,
      "logps/rejected": -318.7333068847656,
      "loss": 4.3737,
      "margin_dpo/margin_mean": 35.343257904052734,
      "margin_dpo/margin_std": 53.63142776489258,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -14.33755111694336,
      "KL/mean": -34.36400604248047,
      "KL/rejected_KL_mean": -54.390472412109375,
      "KL/std": 46.66265869140625,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.013273580931127071,
      "fcm_dpo/delta": -0.03443418815732002,
      "fcm_dpo/margin": 40.052913665771484,
      "fcm_dpo/q_t": 0.3833125829696655,
      "grad_norm": 35.91927719116211,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": 1.658705234527588,
      "logits/rejected": 1.676369309425354,
      "logps/chosen": -312.75665283203125,
      "logps/ref_chosen": -298.4190673828125,
      "logps/ref_rejected": -266.64447021484375,
      "logps/rejected": -321.0349426269531,
      "loss": 4.182,
      "margin_dpo/margin_mean": 40.052913665771484,
      "margin_dpo/margin_std": 53.714263916015625,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -17.59918785095215,
      "KL/mean": -38.561065673828125,
      "KL/rejected_KL_mean": -59.52294921875,
      "KL/std": 47.79172897338867,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.012827414087951183,
      "fcm_dpo/delta": -0.04083697497844696,
      "fcm_dpo/margin": 41.923763275146484,
      "fcm_dpo/q_t": 0.38242459297180176,
      "grad_norm": 34.760379791259766,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": 1.3507654666900635,
      "logits/rejected": 1.333088755607605,
      "logps/chosen": -319.8290710449219,
      "logps/ref_chosen": -302.2298583984375,
      "logps/ref_rejected": -272.7162170410156,
      "logps/rejected": -332.2391662597656,
      "loss": 4.1648,
      "margin_dpo/margin_mean": 41.92375946044922,
      "margin_dpo/margin_std": 55.86959457397461,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -12.706916809082031,
      "KL/mean": -31.48340606689453,
      "KL/rejected_KL_mean": -50.25989532470703,
      "KL/std": 45.218467712402344,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.012761475518345833,
      "fcm_dpo/delta": 0.021137891337275505,
      "fcm_dpo/margin": 37.55297088623047,
      "fcm_dpo/q_t": 0.39349061250686646,
      "grad_norm": 52.33735656738281,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": 1.3945059776306152,
      "logits/rejected": 1.3646965026855469,
      "logps/chosen": -259.5603332519531,
      "logps/ref_chosen": -246.85342407226562,
      "logps/ref_rejected": -216.5264892578125,
      "logps/rejected": -266.7863464355469,
      "loss": 4.2784,
      "margin_dpo/margin_mean": 37.55297088623047,
      "margin_dpo/margin_std": 51.85858154296875,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -15.808090209960938,
      "KL/mean": -34.4158935546875,
      "KL/rejected_KL_mean": -53.02370071411133,
      "KL/std": 46.637229919433594,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.012783058919012547,
      "fcm_dpo/delta": 0.024539273232221603,
      "fcm_dpo/margin": 37.215614318847656,
      "fcm_dpo/q_t": 0.3923935890197754,
      "grad_norm": 39.95989990234375,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": 1.5419931411743164,
      "logits/rejected": 1.6331021785736084,
      "logps/chosen": -278.4577331542969,
      "logps/ref_chosen": -262.649658203125,
      "logps/ref_rejected": -251.24945068359375,
      "logps/rejected": -304.27313232421875,
      "loss": 4.2862,
      "margin_dpo/margin_mean": 37.215614318847656,
      "margin_dpo/margin_std": 50.65331268310547,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -14.569469451904297,
      "KL/mean": -34.721336364746094,
      "KL/rejected_KL_mean": -54.87321090698242,
      "KL/std": 45.957794189453125,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.012776643969118595,
      "fcm_dpo/delta": -0.017055656760931015,
      "fcm_dpo/margin": 40.30374526977539,
      "fcm_dpo/q_t": 0.3831527829170227,
      "grad_norm": 31.292123794555664,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": 1.5418123006820679,
      "logits/rejected": 1.5945549011230469,
      "logps/chosen": -307.955810546875,
      "logps/ref_chosen": -293.3863220214844,
      "logps/ref_rejected": -238.86026000976562,
      "logps/rejected": -293.7334899902344,
      "loss": 4.0975,
      "margin_dpo/margin_mean": 40.30374526977539,
      "margin_dpo/margin_std": 46.859092712402344,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -12.373815536499023,
      "KL/mean": -31.67460823059082,
      "KL/rejected_KL_mean": -50.97539520263672,
      "KL/std": 46.17754364013672,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.012631194666028023,
      "fcm_dpo/delta": -0.04142805188894272,
      "fcm_dpo/margin": 38.60157775878906,
      "fcm_dpo/q_t": 0.3912450969219208,
      "grad_norm": 42.45515060424805,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 1.3001030683517456,
      "logits/rejected": 1.3042596578598022,
      "logps/chosen": -307.87762451171875,
      "logps/ref_chosen": -295.5038146972656,
      "logps/ref_rejected": -278.524169921875,
      "logps/rejected": -329.49957275390625,
      "loss": 4.3547,
      "margin_dpo/margin_mean": 38.60157775878906,
      "margin_dpo/margin_std": 57.569393157958984,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -15.251367568969727,
      "KL/mean": -31.720348358154297,
      "KL/rejected_KL_mean": -48.1893310546875,
      "KL/std": 46.15019989013672,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.013093993999063969,
      "fcm_dpo/delta": 0.06988442689180374,
      "fcm_dpo/margin": 32.93797302246094,
      "fcm_dpo/q_t": 0.40448009967803955,
      "grad_norm": 37.85161590576172,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 1.9469295740127563,
      "logits/rejected": 1.9157428741455078,
      "logps/chosen": -296.0067443847656,
      "logps/ref_chosen": -280.7553405761719,
      "logps/ref_rejected": -237.47286987304688,
      "logps/rejected": -285.6622009277344,
      "loss": 4.4507,
      "margin_dpo/margin_mean": 32.93797302246094,
      "margin_dpo/margin_std": 52.270751953125,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -14.1746826171875,
      "KL/mean": -34.70783233642578,
      "KL/rejected_KL_mean": -55.24098205566406,
      "KL/std": 47.262413024902344,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.012875789776444435,
      "fcm_dpo/delta": -0.07170376181602478,
      "fcm_dpo/margin": 41.066307067871094,
      "fcm_dpo/q_t": 0.38240498304367065,
      "grad_norm": 51.80682373046875,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 1.409784197807312,
      "logits/rejected": 1.4089877605438232,
      "logps/chosen": -313.2660217285156,
      "logps/ref_chosen": -299.09130859375,
      "logps/ref_rejected": -268.9637756347656,
      "logps/rejected": -324.20477294921875,
      "loss": 4.1545,
      "margin_dpo/margin_mean": 41.066307067871094,
      "margin_dpo/margin_std": 52.97338104248047,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -19.369224548339844,
      "KL/mean": -33.739036560058594,
      "KL/rejected_KL_mean": -48.10884475708008,
      "KL/std": 44.40818786621094,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.012939881533384323,
      "fcm_dpo/delta": 0.07006824016571045,
      "fcm_dpo/margin": 28.739612579345703,
      "fcm_dpo/q_t": 0.4182911217212677,
      "grad_norm": 40.355072021484375,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": 1.572582721710205,
      "logits/rejected": 1.519429087638855,
      "logps/chosen": -296.0652770996094,
      "logps/ref_chosen": -276.696044921875,
      "logps/ref_rejected": -225.78790283203125,
      "logps/rejected": -273.8967590332031,
      "loss": 4.722,
      "margin_dpo/margin_mean": 28.739612579345703,
      "margin_dpo/margin_std": 56.36618423461914,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -10.341708183288574,
      "KL/mean": -32.89912414550781,
      "KL/rejected_KL_mean": -55.456539154052734,
      "KL/std": 46.52318572998047,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.012539930641651154,
      "fcm_dpo/delta": -0.07285398989915848,
      "fcm_dpo/margin": 45.114830017089844,
      "fcm_dpo/q_t": 0.37286800146102905,
      "grad_norm": 53.77448272705078,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": 1.4023828506469727,
      "logits/rejected": 1.4784023761749268,
      "logps/chosen": -240.9267120361328,
      "logps/ref_chosen": -230.58499145507812,
      "logps/ref_rejected": -229.8074493408203,
      "logps/rejected": -285.26397705078125,
      "loss": 3.9613,
      "margin_dpo/margin_mean": 45.114830017089844,
      "margin_dpo/margin_std": 48.082550048828125,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -12.79682731628418,
      "KL/mean": -31.877756118774414,
      "KL/rejected_KL_mean": -50.95867919921875,
      "KL/std": 49.65314865112305,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.012226751074194908,
      "fcm_dpo/delta": -0.011929426342248917,
      "fcm_dpo/margin": 38.161865234375,
      "fcm_dpo/q_t": 0.399667888879776,
      "grad_norm": 34.560665130615234,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": 1.3782036304473877,
      "logits/rejected": 1.4071400165557861,
      "logps/chosen": -283.35284423828125,
      "logps/ref_chosen": -270.5559997558594,
      "logps/ref_rejected": -266.328369140625,
      "logps/rejected": -317.28704833984375,
      "loss": 4.449,
      "margin_dpo/margin_mean": 38.161865234375,
      "margin_dpo/margin_std": 61.96608352661133,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -12.339679718017578,
      "KL/mean": -31.030113220214844,
      "KL/rejected_KL_mean": -49.72055435180664,
      "KL/std": 44.50080490112305,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.012614257633686066,
      "fcm_dpo/delta": 0.028930773958563805,
      "fcm_dpo/margin": 37.380882263183594,
      "fcm_dpo/q_t": 0.3950861990451813,
      "grad_norm": 45.80881118774414,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": 1.3623939752578735,
      "logits/rejected": 1.443524718284607,
      "logps/chosen": -245.57827758789062,
      "logps/ref_chosen": -233.2386016845703,
      "logps/ref_rejected": -258.154541015625,
      "logps/rejected": -307.8750915527344,
      "loss": 4.373,
      "margin_dpo/margin_mean": 37.380882263183594,
      "margin_dpo/margin_std": 56.60421371459961,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -4.6117401123046875,
      "KL/mean": -25.85860824584961,
      "KL/rejected_KL_mean": -47.1054801940918,
      "KL/std": 45.93081283569336,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.012446708977222443,
      "fcm_dpo/delta": -0.032222066074609756,
      "fcm_dpo/margin": 42.49373245239258,
      "fcm_dpo/q_t": 0.382525235414505,
      "grad_norm": 29.410274505615234,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": 1.4965486526489258,
      "logits/rejected": 1.5177655220031738,
      "logps/chosen": -273.1579284667969,
      "logps/ref_chosen": -268.5461730957031,
      "logps/ref_rejected": -266.5316467285156,
      "logps/rejected": -313.6371154785156,
      "loss": 4.198,
      "margin_dpo/margin_mean": 42.49373245239258,
      "margin_dpo/margin_std": 56.58677673339844,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -14.383676528930664,
      "KL/mean": -32.50703048706055,
      "KL/rejected_KL_mean": -50.6303825378418,
      "KL/std": 46.564186096191406,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.01244023721665144,
      "fcm_dpo/delta": 0.05114942044019699,
      "fcm_dpo/margin": 36.246700286865234,
      "fcm_dpo/q_t": 0.3990764319896698,
      "grad_norm": 38.035972595214844,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": 1.4758961200714111,
      "logits/rejected": 1.4977091550827026,
      "logps/chosen": -309.26043701171875,
      "logps/ref_chosen": -294.8768005371094,
      "logps/ref_rejected": -281.09771728515625,
      "logps/rejected": -331.72808837890625,
      "loss": 4.3636,
      "margin_dpo/margin_mean": 36.2467041015625,
      "margin_dpo/margin_std": 52.427391052246094,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -20.12338638305664,
      "KL/mean": -36.20626449584961,
      "KL/rejected_KL_mean": -52.28914260864258,
      "KL/std": 50.770198822021484,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.012919146567583084,
      "fcm_dpo/delta": 0.011806602589786053,
      "fcm_dpo/margin": 32.165748596191406,
      "fcm_dpo/q_t": 0.4106605052947998,
      "grad_norm": 36.93961715698242,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": 1.4946539402008057,
      "logits/rejected": 1.506049633026123,
      "logps/chosen": -332.42120361328125,
      "logps/ref_chosen": -312.2977600097656,
      "logps/ref_rejected": -304.6536560058594,
      "logps/rejected": -356.9427795410156,
      "loss": 4.653,
      "margin_dpo/margin_mean": 32.165748596191406,
      "margin_dpo/margin_std": 60.81819534301758,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -17.288908004760742,
      "KL/mean": -31.68438720703125,
      "KL/rejected_KL_mean": -46.07987976074219,
      "KL/std": 47.34357452392578,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.013511172495782375,
      "fcm_dpo/delta": 0.08170526474714279,
      "fcm_dpo/margin": 28.790979385375977,
      "fcm_dpo/q_t": 0.4162774682044983,
      "grad_norm": 38.787452697753906,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 1.411144733428955,
      "logits/rejected": 1.4703601598739624,
      "logps/chosen": -311.4931335449219,
      "logps/ref_chosen": -294.2042236328125,
      "logps/ref_rejected": -296.2102355957031,
      "logps/rejected": -342.2901306152344,
      "loss": 4.6832,
      "margin_dpo/margin_mean": 28.79097557067871,
      "margin_dpo/margin_std": 56.725772857666016,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -11.527647018432617,
      "KL/mean": -29.826597213745117,
      "KL/rejected_KL_mean": -48.125545501708984,
      "KL/std": 47.519588470458984,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.0134944012388587,
      "fcm_dpo/delta": -0.04068208113312721,
      "fcm_dpo/margin": 36.597896575927734,
      "fcm_dpo/q_t": 0.3915608525276184,
      "grad_norm": 44.11063003540039,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.4614328145980835,
      "logits/rejected": 1.4742947816848755,
      "logps/chosen": -279.8403015136719,
      "logps/ref_chosen": -268.3126525878906,
      "logps/ref_rejected": -255.5543212890625,
      "logps/rejected": -303.67987060546875,
      "loss": 4.3183,
      "margin_dpo/margin_mean": 36.597896575927734,
      "margin_dpo/margin_std": 53.826419830322266,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -13.320642471313477,
      "KL/mean": -34.397422790527344,
      "KL/rejected_KL_mean": -55.474212646484375,
      "KL/std": 47.297210693359375,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.01276309508830309,
      "fcm_dpo/delta": -0.04244554787874222,
      "fcm_dpo/margin": 42.15357971191406,
      "fcm_dpo/q_t": 0.3816434442996979,
      "grad_norm": 35.26375961303711,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": 1.5122060775756836,
      "logits/rejected": 1.5571489334106445,
      "logps/chosen": -277.0710754394531,
      "logps/ref_chosen": -263.75042724609375,
      "logps/ref_rejected": -234.5144500732422,
      "logps/rejected": -289.9886474609375,
      "loss": 4.2022,
      "margin_dpo/margin_mean": 42.15357971191406,
      "margin_dpo/margin_std": 57.79307556152344,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -10.100080490112305,
      "KL/mean": -26.60880470275879,
      "KL/rejected_KL_mean": -43.117530822753906,
      "KL/std": 45.50019454956055,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.013388711027801037,
      "fcm_dpo/delta": 0.05893842875957489,
      "fcm_dpo/margin": 33.0174446105957,
      "fcm_dpo/q_t": 0.4015706777572632,
      "grad_norm": 60.252986907958984,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": 1.5346916913986206,
      "logits/rejected": 1.5161986351013184,
      "logps/chosen": -296.91650390625,
      "logps/ref_chosen": -286.81646728515625,
      "logps/ref_rejected": -260.4169616699219,
      "logps/rejected": -303.5345153808594,
      "loss": 4.4261,
      "margin_dpo/margin_mean": 33.0174446105957,
      "margin_dpo/margin_std": 50.952545166015625,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -9.144468307495117,
      "KL/mean": -28.38404083251953,
      "KL/rejected_KL_mean": -47.62362289428711,
      "KL/std": 47.89649200439453,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.013395161367952824,
      "fcm_dpo/delta": -0.05985581874847412,
      "fcm_dpo/margin": 38.479148864746094,
      "fcm_dpo/q_t": 0.3883460462093353,
      "grad_norm": 38.407737731933594,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": 1.4567736387252808,
      "logits/rejected": 1.4541912078857422,
      "logps/chosen": -302.45538330078125,
      "logps/ref_chosen": -293.3109130859375,
      "logps/ref_rejected": -264.1449279785156,
      "logps/rejected": -311.7685546875,
      "loss": 4.3852,
      "margin_dpo/margin_mean": 38.47914505004883,
      "margin_dpo/margin_std": 59.14542007446289,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -20.13387107849121,
      "KL/mean": -37.077816009521484,
      "KL/rejected_KL_mean": -54.021766662597656,
      "KL/std": 46.60589599609375,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.012947499752044678,
      "fcm_dpo/delta": 0.012973852455615997,
      "fcm_dpo/margin": 33.88789749145508,
      "fcm_dpo/q_t": 0.40465325117111206,
      "grad_norm": 51.741294860839844,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 1.2475910186767578,
      "logits/rejected": 1.3279668092727661,
      "logps/chosen": -289.8902282714844,
      "logps/ref_chosen": -269.75634765625,
      "logps/ref_rejected": -241.08438110351562,
      "logps/rejected": -295.10614013671875,
      "loss": 4.578,
      "margin_dpo/margin_mean": 33.88789749145508,
      "margin_dpo/margin_std": 59.64500045776367,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -14.130420684814453,
      "KL/mean": -30.50165367126465,
      "KL/rejected_KL_mean": -46.872894287109375,
      "KL/std": 46.808467864990234,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.013152127154171467,
      "fcm_dpo/delta": 0.02539416216313839,
      "fcm_dpo/margin": 32.74248504638672,
      "fcm_dpo/q_t": 0.4063722491264343,
      "grad_norm": 43.122684478759766,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": 1.450997233390808,
      "logits/rejected": 1.4767577648162842,
      "logps/chosen": -251.50221252441406,
      "logps/ref_chosen": -237.37179565429688,
      "logps/ref_rejected": -253.5463409423828,
      "logps/rejected": -300.4192199707031,
      "loss": 4.498,
      "margin_dpo/margin_mean": 32.74248504638672,
      "margin_dpo/margin_std": 54.583091735839844,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -16.961597442626953,
      "KL/mean": -35.94770431518555,
      "KL/rejected_KL_mean": -54.933807373046875,
      "KL/std": 46.306297302246094,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.013217407278716564,
      "fcm_dpo/delta": -0.04166708141565323,
      "fcm_dpo/margin": 37.97222137451172,
      "fcm_dpo/q_t": 0.39123642444610596,
      "grad_norm": 41.61693572998047,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": 1.2226417064666748,
      "logits/rejected": 1.265523076057434,
      "logps/chosen": -278.26922607421875,
      "logps/ref_chosen": -261.3076477050781,
      "logps/ref_rejected": -252.3170166015625,
      "logps/rejected": -307.2508239746094,
      "loss": 4.3258,
      "margin_dpo/margin_mean": 37.972225189208984,
      "margin_dpo/margin_std": 56.71925354003906,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -4.682476043701172,
      "KL/mean": -26.78110694885254,
      "KL/rejected_KL_mean": -48.879737854003906,
      "KL/std": 48.163658142089844,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.012619329616427422,
      "fcm_dpo/delta": -0.06084993854165077,
      "fcm_dpo/margin": 44.1972541809082,
      "fcm_dpo/q_t": 0.3777793049812317,
      "grad_norm": 37.24745559692383,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": 1.4409071207046509,
      "logits/rejected": 1.41244637966156,
      "logps/chosen": -273.4542236328125,
      "logps/ref_chosen": -268.7717590332031,
      "logps/ref_rejected": -232.3352813720703,
      "logps/rejected": -281.2149963378906,
      "loss": 4.1026,
      "margin_dpo/margin_mean": 44.1972541809082,
      "margin_dpo/margin_std": 57.43903350830078,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -11.40013313293457,
      "KL/mean": -33.50927734375,
      "KL/rejected_KL_mean": -55.61842346191406,
      "KL/std": 49.599510192871094,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.012066394090652466,
      "fcm_dpo/delta": -0.03616008162498474,
      "fcm_dpo/margin": 44.218292236328125,
      "fcm_dpo/q_t": 0.38079872727394104,
      "grad_norm": 37.5029182434082,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": 1.208491563796997,
      "logits/rejected": 1.160207986831665,
      "logps/chosen": -305.01324462890625,
      "logps/ref_chosen": -293.6131286621094,
      "logps/ref_rejected": -281.87738037109375,
      "logps/rejected": -337.4957580566406,
      "loss": 4.1741,
      "margin_dpo/margin_mean": 44.218292236328125,
      "margin_dpo/margin_std": 59.2026252746582,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -15.754535675048828,
      "KL/mean": -34.442012786865234,
      "KL/rejected_KL_mean": -53.12950134277344,
      "KL/std": 49.682655334472656,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.01211779285222292,
      "fcm_dpo/delta": 0.04854670539498329,
      "fcm_dpo/margin": 37.37496566772461,
      "fcm_dpo/q_t": 0.40092936158180237,
      "grad_norm": 34.39699935913086,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 1.1355693340301514,
      "logits/rejected": 1.2497148513793945,
      "logps/chosen": -267.5115966796875,
      "logps/ref_chosen": -251.75704956054688,
      "logps/ref_rejected": -255.76441955566406,
      "logps/rejected": -308.8938903808594,
      "loss": 4.4147,
      "margin_dpo/margin_mean": 37.374969482421875,
      "margin_dpo/margin_std": 57.61641311645508,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -25.47631072998047,
      "KL/mean": -40.65039825439453,
      "KL/rejected_KL_mean": -55.82447052001953,
      "KL/std": 51.787601470947266,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.012780530378222466,
      "fcm_dpo/delta": 0.058902494609355927,
      "fcm_dpo/margin": 30.348173141479492,
      "fcm_dpo/q_t": 0.41605713963508606,
      "grad_norm": 45.69449996948242,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 1.554054617881775,
      "logits/rejected": 1.6678777933120728,
      "logps/chosen": -332.12396240234375,
      "logps/ref_chosen": -306.64764404296875,
      "logps/ref_rejected": -301.4248352050781,
      "logps/rejected": -357.24932861328125,
      "loss": 4.7246,
      "margin_dpo/margin_mean": 30.348173141479492,
      "margin_dpo/margin_std": 60.227508544921875,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -13.030889511108398,
      "KL/mean": -31.930089950561523,
      "KL/rejected_KL_mean": -50.82929229736328,
      "KL/std": 49.390323638916016,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.01291895192116499,
      "fcm_dpo/delta": -0.02801799774169922,
      "fcm_dpo/margin": 37.798404693603516,
      "fcm_dpo/q_t": 0.3925938904285431,
      "grad_norm": 33.364803314208984,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": 1.5304491519927979,
      "logits/rejected": 1.5840882062911987,
      "logps/chosen": -275.6998291015625,
      "logps/ref_chosen": -262.6689147949219,
      "logps/ref_rejected": -260.8409118652344,
      "logps/rejected": -311.6701965332031,
      "loss": 4.2999,
      "margin_dpo/margin_mean": 37.798404693603516,
      "margin_dpo/margin_std": 53.70538330078125,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -13.02655029296875,
      "KL/mean": -29.98611831665039,
      "KL/rejected_KL_mean": -46.9456901550293,
      "KL/std": 48.777008056640625,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.012764501385390759,
      "fcm_dpo/delta": 0.03197219595313072,
      "fcm_dpo/margin": 33.919151306152344,
      "fcm_dpo/q_t": 0.4057978093624115,
      "grad_norm": 37.98701095581055,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": 1.6392532587051392,
      "logits/rejected": 1.653444766998291,
      "logps/chosen": -312.6524963378906,
      "logps/ref_chosen": -299.6259765625,
      "logps/ref_rejected": -261.2701416015625,
      "logps/rejected": -308.2158508300781,
      "loss": 4.5102,
      "margin_dpo/margin_mean": 33.919151306152344,
      "margin_dpo/margin_std": 57.027896881103516,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -14.320127487182617,
      "KL/mean": -34.01076126098633,
      "KL/rejected_KL_mean": -53.701393127441406,
      "KL/std": 49.646976470947266,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.013286210596561432,
      "fcm_dpo/delta": -0.026181932538747787,
      "fcm_dpo/margin": 39.38126754760742,
      "fcm_dpo/q_t": 0.3865654766559601,
      "grad_norm": 33.01263427734375,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": 1.323685646057129,
      "logits/rejected": 1.3540161848068237,
      "logps/chosen": -305.27056884765625,
      "logps/ref_chosen": -290.950439453125,
      "logps/ref_rejected": -255.15374755859375,
      "logps/rejected": -308.8551330566406,
      "loss": 4.2803,
      "margin_dpo/margin_mean": 39.38126754760742,
      "margin_dpo/margin_std": 56.44644546508789,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -9.577024459838867,
      "KL/mean": -27.12582778930664,
      "KL/rejected_KL_mean": -44.67463302612305,
      "KL/std": 46.86163330078125,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.013141563162207603,
      "fcm_dpo/delta": 0.03842107579112053,
      "fcm_dpo/margin": 35.097618103027344,
      "fcm_dpo/q_t": 0.3978886008262634,
      "grad_norm": 41.3137321472168,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": 1.424147367477417,
      "logits/rejected": 1.4952611923217773,
      "logps/chosen": -282.8428955078125,
      "logps/ref_chosen": -273.2658996582031,
      "logps/ref_rejected": -263.9664306640625,
      "logps/rejected": -308.64105224609375,
      "loss": 4.324,
      "margin_dpo/margin_mean": 35.097618103027344,
      "margin_dpo/margin_std": 48.852508544921875,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -20.610496520996094,
      "KL/mean": -39.96180725097656,
      "KL/rejected_KL_mean": -59.31312561035156,
      "KL/std": 48.72319793701172,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.012680443935096264,
      "fcm_dpo/delta": -0.07478740811347961,
      "fcm_dpo/margin": 38.70262145996094,
      "fcm_dpo/q_t": 0.3907240629196167,
      "grad_norm": 37.88601303100586,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": 1.3109580278396606,
      "logits/rejected": 1.3563287258148193,
      "logps/chosen": -283.29266357421875,
      "logps/ref_chosen": -262.6821594238281,
      "logps/ref_rejected": -226.46859741210938,
      "logps/rejected": -285.78173828125,
      "loss": 4.3094,
      "margin_dpo/margin_mean": 38.7026252746582,
      "margin_dpo/margin_std": 52.88188934326172,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -18.278636932373047,
      "KL/mean": -35.115604400634766,
      "KL/rejected_KL_mean": -51.95257568359375,
      "KL/std": 48.2205810546875,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.012718463316559792,
      "fcm_dpo/delta": 0.07482835650444031,
      "fcm_dpo/margin": 33.67393112182617,
      "fcm_dpo/q_t": 0.404192179441452,
      "grad_norm": 37.14983367919922,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": 1.2017208337783813,
      "logits/rejected": 1.2093007564544678,
      "logps/chosen": -282.1032409667969,
      "logps/ref_chosen": -263.8246154785156,
      "logps/ref_rejected": -237.9121551513672,
      "logps/rejected": -289.86474609375,
      "loss": 4.4851,
      "margin_dpo/margin_mean": 33.67393112182617,
      "margin_dpo/margin_std": 54.516719818115234,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -12.367683410644531,
      "KL/mean": -27.56141471862793,
      "KL/rejected_KL_mean": -42.755149841308594,
      "KL/std": 48.170433044433594,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.013109380379319191,
      "fcm_dpo/delta": 0.015499202534556389,
      "fcm_dpo/margin": 30.387466430664062,
      "fcm_dpo/q_t": 0.41311803460121155,
      "grad_norm": 34.940467834472656,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 1.3621962070465088,
      "logits/rejected": 1.4037489891052246,
      "logps/chosen": -290.52099609375,
      "logps/ref_chosen": -278.1533203125,
      "logps/ref_rejected": -263.39691162109375,
      "logps/rejected": -306.1520690917969,
      "loss": 4.5752,
      "margin_dpo/margin_mean": 30.38746452331543,
      "margin_dpo/margin_std": 51.522705078125,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -15.412906646728516,
      "KL/mean": -36.251834869384766,
      "KL/rejected_KL_mean": -57.09075927734375,
      "KL/std": 51.451927185058594,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.0127632524818182,
      "fcm_dpo/delta": -0.08633626252412796,
      "fcm_dpo/margin": 41.677852630615234,
      "fcm_dpo/q_t": 0.38427579402923584,
      "grad_norm": 34.390968322753906,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": 1.5914475917816162,
      "logits/rejected": 1.632763147354126,
      "logps/chosen": -333.8680114746094,
      "logps/ref_chosen": -318.455078125,
      "logps/ref_rejected": -288.5691833496094,
      "logps/rejected": -345.6598815917969,
      "loss": 4.2345,
      "margin_dpo/margin_mean": 41.677852630615234,
      "margin_dpo/margin_std": 57.5543098449707,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -18.433570861816406,
      "KL/mean": -37.07556915283203,
      "KL/rejected_KL_mean": -55.71757125854492,
      "KL/std": 50.70511245727539,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.012344998307526112,
      "fcm_dpo/delta": -0.058857060968875885,
      "fcm_dpo/margin": 37.284000396728516,
      "fcm_dpo/q_t": 0.3986093997955322,
      "grad_norm": 39.316429138183594,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 1.4777393341064453,
      "logits/rejected": 1.4841684103012085,
      "logps/chosen": -302.6227722167969,
      "logps/ref_chosen": -284.1891784667969,
      "logps/ref_rejected": -247.94589233398438,
      "logps/rejected": -303.6634216308594,
      "loss": 4.4579,
      "margin_dpo/margin_mean": 37.284000396728516,
      "margin_dpo/margin_std": 59.04204559326172,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -15.320968627929688,
      "KL/mean": -29.141212463378906,
      "KL/rejected_KL_mean": -42.961463928222656,
      "KL/std": 48.9239501953125,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.012761571444571018,
      "fcm_dpo/delta": 0.10713023692369461,
      "fcm_dpo/margin": 27.640499114990234,
      "fcm_dpo/q_t": 0.4253620505332947,
      "grad_norm": 40.844154357910156,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 1.5713138580322266,
      "logits/rejected": 1.554193377494812,
      "logps/chosen": -318.6290588378906,
      "logps/ref_chosen": -303.30810546875,
      "logps/ref_rejected": -260.1866455078125,
      "logps/rejected": -303.1481018066406,
      "loss": 4.8371,
      "margin_dpo/margin_mean": 27.640499114990234,
      "margin_dpo/margin_std": 60.0678825378418,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -14.111026763916016,
      "KL/mean": -34.870216369628906,
      "KL/rejected_KL_mean": -55.62941360473633,
      "KL/std": 50.322898864746094,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.012738817371428013,
      "fcm_dpo/delta": -0.03292613476514816,
      "fcm_dpo/margin": 41.51838684082031,
      "fcm_dpo/q_t": 0.38391929864883423,
      "grad_norm": 34.38352966308594,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": 1.3273053169250488,
      "logits/rejected": 1.5018115043640137,
      "logps/chosen": -260.9181213378906,
      "logps/ref_chosen": -246.80709838867188,
      "logps/ref_rejected": -253.13223266601562,
      "logps/rejected": -308.76165771484375,
      "loss": 4.2449,
      "margin_dpo/margin_mean": 41.51838684082031,
      "margin_dpo/margin_std": 56.561466217041016,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -16.110679626464844,
      "KL/mean": -36.73103332519531,
      "KL/rejected_KL_mean": -57.35138702392578,
      "KL/std": 49.72696304321289,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.012210158631205559,
      "fcm_dpo/delta": -0.047825928777456284,
      "fcm_dpo/margin": 41.240718841552734,
      "fcm_dpo/q_t": 0.38875794410705566,
      "grad_norm": 31.977479934692383,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": 1.3661457300186157,
      "logits/rejected": 1.4408009052276611,
      "logps/chosen": -277.712158203125,
      "logps/ref_chosen": -261.6014709472656,
      "logps/ref_rejected": -262.74761962890625,
      "logps/rejected": -320.0989990234375,
      "loss": 4.2498,
      "margin_dpo/margin_mean": 41.240718841552734,
      "margin_dpo/margin_std": 55.97351837158203,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -16.645668029785156,
      "KL/mean": -28.97772979736328,
      "KL/rejected_KL_mean": -41.309791564941406,
      "KL/std": 49.5717658996582,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.01239807903766632,
      "fcm_dpo/delta": 0.06780020892620087,
      "fcm_dpo/margin": 24.664133071899414,
      "fcm_dpo/q_t": 0.43435850739479065,
      "grad_norm": 39.93256378173828,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": 1.4120670557022095,
      "logits/rejected": 1.524681806564331,
      "logps/chosen": -268.7940979003906,
      "logps/ref_chosen": -252.14840698242188,
      "logps/ref_rejected": -244.1017608642578,
      "logps/rejected": -285.41156005859375,
      "loss": 4.9794,
      "margin_dpo/margin_mean": 24.664133071899414,
      "margin_dpo/margin_std": 61.16976547241211,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -15.782157897949219,
      "KL/mean": -36.80220031738281,
      "KL/rejected_KL_mean": -57.82224655151367,
      "KL/std": 50.836952209472656,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.012236448936164379,
      "fcm_dpo/delta": -0.05564156174659729,
      "fcm_dpo/margin": 42.04009246826172,
      "fcm_dpo/q_t": 0.3858323097229004,
      "grad_norm": 42.48344039916992,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": 1.7065458297729492,
      "logits/rejected": 1.8322043418884277,
      "logps/chosen": -301.9447326660156,
      "logps/ref_chosen": -286.16253662109375,
      "logps/ref_rejected": -272.3275146484375,
      "logps/rejected": -330.1497802734375,
      "loss": 4.2536,
      "margin_dpo/margin_mean": 42.04009246826172,
      "margin_dpo/margin_std": 58.50876998901367,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -11.883207321166992,
      "KL/mean": -28.811351776123047,
      "KL/rejected_KL_mean": -45.739498138427734,
      "KL/std": 46.64939880371094,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.012484555132687092,
      "fcm_dpo/delta": 0.04094310104846954,
      "fcm_dpo/margin": 33.856292724609375,
      "fcm_dpo/q_t": 0.40568438172340393,
      "grad_norm": 40.94906234741211,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": 1.3708176612854004,
      "logits/rejected": 1.4180381298065186,
      "logps/chosen": -210.0360107421875,
      "logps/ref_chosen": -198.1527862548828,
      "logps/ref_rejected": -209.80569458007812,
      "logps/rejected": -255.54519653320312,
      "loss": 4.459,
      "margin_dpo/margin_mean": 33.856292724609375,
      "margin_dpo/margin_std": 52.29646682739258,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -18.143770217895508,
      "KL/mean": -37.03746032714844,
      "KL/rejected_KL_mean": -55.93115234375,
      "KL/std": 48.10342025756836,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.012503281235694885,
      "fcm_dpo/delta": -0.016977887600660324,
      "fcm_dpo/margin": 37.78738021850586,
      "fcm_dpo/q_t": 0.3984060287475586,
      "grad_norm": 39.011871337890625,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": 1.423306941986084,
      "logits/rejected": 1.4794446229934692,
      "logps/chosen": -274.65093994140625,
      "logps/ref_chosen": -256.50714111328125,
      "logps/ref_rejected": -237.40769958496094,
      "logps/rejected": -293.3388671875,
      "loss": 4.4058,
      "margin_dpo/margin_mean": 37.78738021850586,
      "margin_dpo/margin_std": 58.759315490722656,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -10.057107925415039,
      "KL/mean": -31.1195011138916,
      "KL/rejected_KL_mean": -52.181884765625,
      "KL/std": 49.69719314575195,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.012407714501023293,
      "fcm_dpo/delta": -0.025305557996034622,
      "fcm_dpo/margin": 42.124786376953125,
      "fcm_dpo/q_t": 0.3860827684402466,
      "grad_norm": 33.80720520019531,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": 1.494760274887085,
      "logits/rejected": 1.431020736694336,
      "logps/chosen": -302.5043640136719,
      "logps/ref_chosen": -292.447265625,
      "logps/ref_rejected": -255.47872924804688,
      "logps/rejected": -307.66058349609375,
      "loss": 4.2915,
      "margin_dpo/margin_mean": 42.124786376953125,
      "margin_dpo/margin_std": 61.09071350097656,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -10.344650268554688,
      "KL/mean": -29.984092712402344,
      "KL/rejected_KL_mean": -49.62353515625,
      "KL/std": 49.39704895019531,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.012367994524538517,
      "fcm_dpo/delta": 0.013840976171195507,
      "fcm_dpo/margin": 39.27888488769531,
      "fcm_dpo/q_t": 0.39089885354042053,
      "grad_norm": 35.920753479003906,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": 1.4770780801773071,
      "logits/rejected": 1.4356333017349243,
      "logps/chosen": -304.1046447753906,
      "logps/ref_chosen": -293.7599792480469,
      "logps/ref_rejected": -259.4142761230469,
      "logps/rejected": -309.037841796875,
      "loss": 4.2748,
      "margin_dpo/margin_mean": 39.27888488769531,
      "margin_dpo/margin_std": 53.73188018798828,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -17.249046325683594,
      "KL/mean": -34.646240234375,
      "KL/rejected_KL_mean": -52.04343795776367,
      "KL/std": 49.733341217041016,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.012465525418519974,
      "fcm_dpo/delta": 0.018586482852697372,
      "fcm_dpo/margin": 34.79438018798828,
      "fcm_dpo/q_t": 0.40308043360710144,
      "grad_norm": 42.6588249206543,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 1.562841773033142,
      "logits/rejected": 1.5320976972579956,
      "logps/chosen": -318.16278076171875,
      "logps/ref_chosen": -300.91375732421875,
      "logps/ref_rejected": -256.4080810546875,
      "logps/rejected": -308.4515075683594,
      "loss": 4.4226,
      "margin_dpo/margin_mean": 34.79438018798828,
      "margin_dpo/margin_std": 52.40127944946289,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -8.791114807128906,
      "KL/mean": -31.921140670776367,
      "KL/rejected_KL_mean": -55.051170349121094,
      "KL/std": 51.1580696105957,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.012219679541885853,
      "fcm_dpo/delta": -0.06903190910816193,
      "fcm_dpo/margin": 46.260047912597656,
      "fcm_dpo/q_t": 0.3762603998184204,
      "grad_norm": 34.936614990234375,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 1.26888906955719,
      "logits/rejected": 1.3520534038543701,
      "logps/chosen": -265.3514404296875,
      "logps/ref_chosen": -256.5603332519531,
      "logps/ref_rejected": -271.043701171875,
      "logps/rejected": -326.0948791503906,
      "loss": 4.1039,
      "margin_dpo/margin_mean": 46.26005172729492,
      "margin_dpo/margin_std": 60.488426208496094,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -12.84410285949707,
      "KL/mean": -31.293392181396484,
      "KL/rejected_KL_mean": -49.7426872253418,
      "KL/std": 46.2922248840332,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.011996576562523842,
      "fcm_dpo/delta": 0.01662357896566391,
      "fcm_dpo/margin": 36.898582458496094,
      "fcm_dpo/q_t": 0.40106621384620667,
      "grad_norm": 53.68181228637695,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": 1.5610554218292236,
      "logits/rejected": 1.4788790941238403,
      "logps/chosen": -307.5445861816406,
      "logps/ref_chosen": -294.700439453125,
      "logps/ref_rejected": -239.67922973632812,
      "logps/rejected": -289.42193603515625,
      "loss": 4.4101,
      "margin_dpo/margin_mean": 36.898582458496094,
      "margin_dpo/margin_std": 55.3452262878418,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -12.827596664428711,
      "KL/mean": -34.86578369140625,
      "KL/rejected_KL_mean": -56.90397262573242,
      "KL/std": 51.55328369140625,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.011891039088368416,
      "fcm_dpo/delta": -0.02649863436818123,
      "fcm_dpo/margin": 44.076385498046875,
      "fcm_dpo/q_t": 0.38380613923072815,
      "grad_norm": 39.051631927490234,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": 1.5084611177444458,
      "logits/rejected": 1.5218902826309204,
      "logps/chosen": -306.5034484863281,
      "logps/ref_chosen": -293.67584228515625,
      "logps/ref_rejected": -260.85675048828125,
      "logps/rejected": -317.7607116699219,
      "loss": 4.199,
      "margin_dpo/margin_mean": 44.076385498046875,
      "margin_dpo/margin_std": 59.4390869140625,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -27.132047653198242,
      "KL/mean": -43.73137664794922,
      "KL/rejected_KL_mean": -60.330711364746094,
      "KL/std": 49.93042755126953,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.011883941479027271,
      "fcm_dpo/delta": 0.03273960202932358,
      "fcm_dpo/margin": 33.19866943359375,
      "fcm_dpo/q_t": 0.4131779670715332,
      "grad_norm": 31.561676025390625,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": 1.2199729681015015,
      "logits/rejected": 1.2736754417419434,
      "logps/chosen": -306.0274353027344,
      "logps/ref_chosen": -278.8953857421875,
      "logps/ref_rejected": -283.8898620605469,
      "logps/rejected": -344.2205505371094,
      "loss": 4.6119,
      "margin_dpo/margin_mean": 33.19866943359375,
      "margin_dpo/margin_std": 58.49749755859375,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -13.769161224365234,
      "KL/mean": -35.04008483886719,
      "KL/rejected_KL_mean": -56.311004638671875,
      "KL/std": 51.645721435546875,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.012099886313080788,
      "fcm_dpo/delta": -0.016041912138462067,
      "fcm_dpo/margin": 42.54185104370117,
      "fcm_dpo/q_t": 0.38729989528656006,
      "grad_norm": 50.63489532470703,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": 1.2765657901763916,
      "logits/rejected": 1.3114327192306519,
      "logps/chosen": -279.4130859375,
      "logps/ref_chosen": -265.6439208984375,
      "logps/ref_rejected": -255.40138244628906,
      "logps/rejected": -311.71240234375,
      "loss": 4.2045,
      "margin_dpo/margin_mean": 42.541847229003906,
      "margin_dpo/margin_std": 57.605743408203125,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -17.469297409057617,
      "KL/mean": -38.477752685546875,
      "KL/rejected_KL_mean": -59.486202239990234,
      "KL/std": 50.94304275512695,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.01176812220364809,
      "fcm_dpo/delta": 0.005113358609378338,
      "fcm_dpo/margin": 42.016902923583984,
      "fcm_dpo/q_t": 0.3894684910774231,
      "grad_norm": 32.130062103271484,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": 1.3754706382751465,
      "logits/rejected": 1.3794599771499634,
      "logps/chosen": -297.69378662109375,
      "logps/ref_chosen": -280.2244873046875,
      "logps/ref_rejected": -248.72366333007812,
      "logps/rejected": -308.2098388671875,
      "loss": 4.2481,
      "margin_dpo/margin_mean": 42.016902923583984,
      "margin_dpo/margin_std": 56.55713653564453,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -15.25233268737793,
      "KL/mean": -37.11049270629883,
      "KL/rejected_KL_mean": -58.968650817871094,
      "KL/std": 50.2314567565918,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.011735728941857815,
      "fcm_dpo/delta": -0.014661801978945732,
      "fcm_dpo/margin": 43.7163200378418,
      "fcm_dpo/q_t": 0.38544559478759766,
      "grad_norm": 46.04490661621094,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": 1.309849500656128,
      "logits/rejected": 1.300220012664795,
      "logps/chosen": -308.1889953613281,
      "logps/ref_chosen": -292.9366760253906,
      "logps/ref_rejected": -261.88763427734375,
      "logps/rejected": -320.8562927246094,
      "loss": 4.1445,
      "margin_dpo/margin_mean": 43.7163200378418,
      "margin_dpo/margin_std": 54.62760925292969,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -10.715866088867188,
      "KL/mean": -35.51803207397461,
      "KL/rejected_KL_mean": -60.320194244384766,
      "KL/std": 52.26353454589844,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.01123238168656826,
      "fcm_dpo/delta": -0.10469198226928711,
      "fcm_dpo/margin": 49.60432434082031,
      "fcm_dpo/q_t": 0.3740662634372711,
      "grad_norm": 32.79896545410156,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": 1.3678100109100342,
      "logits/rejected": 1.454975962638855,
      "logps/chosen": -295.6216125488281,
      "logps/ref_chosen": -284.90576171875,
      "logps/ref_rejected": -256.59197998046875,
      "logps/rejected": -316.91217041015625,
      "loss": 4.009,
      "margin_dpo/margin_mean": 49.60432434082031,
      "margin_dpo/margin_std": 55.263427734375,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -16.20769691467285,
      "KL/mean": -33.79865646362305,
      "KL/rejected_KL_mean": -51.389617919921875,
      "KL/std": 49.159400939941406,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.01104483287781477,
      "fcm_dpo/delta": 0.03468402475118637,
      "fcm_dpo/margin": 35.181915283203125,
      "fcm_dpo/q_t": 0.41284868121147156,
      "grad_norm": 41.2286491394043,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": 1.4856497049331665,
      "logits/rejected": 1.4963682889938354,
      "logps/chosen": -292.1236267089844,
      "logps/ref_chosen": -275.91595458984375,
      "logps/ref_rejected": -264.79736328125,
      "logps/rejected": -316.1869812011719,
      "loss": 4.5661,
      "margin_dpo/margin_mean": 35.181915283203125,
      "margin_dpo/margin_std": 57.942752838134766,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -15.372261047363281,
      "KL/mean": -37.30007553100586,
      "KL/rejected_KL_mean": -59.2278938293457,
      "KL/std": 50.3149299621582,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.010954920202493668,
      "fcm_dpo/delta": -0.017903342843055725,
      "fcm_dpo/margin": 43.855621337890625,
      "fcm_dpo/q_t": 0.39159974455833435,
      "grad_norm": 32.101924896240234,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 1.245024561882019,
      "logits/rejected": 1.3026959896087646,
      "logps/chosen": -260.3143005371094,
      "logps/ref_chosen": -244.9420166015625,
      "logps/ref_rejected": -235.61978149414062,
      "logps/rejected": -294.8476867675781,
      "loss": 4.2738,
      "margin_dpo/margin_mean": 43.85562515258789,
      "margin_dpo/margin_std": 60.13688659667969,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -18.599119186401367,
      "KL/mean": -36.44662857055664,
      "KL/rejected_KL_mean": -54.294132232666016,
      "KL/std": 50.45078659057617,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.011639012955129147,
      "fcm_dpo/delta": 0.08642654120922089,
      "fcm_dpo/margin": 35.69501495361328,
      "fcm_dpo/q_t": 0.40664297342300415,
      "grad_norm": 70.78218841552734,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 1.413140058517456,
      "logits/rejected": 1.4163942337036133,
      "logps/chosen": -307.9489440917969,
      "logps/ref_chosen": -289.3498229980469,
      "logps/ref_rejected": -243.70205688476562,
      "logps/rejected": -297.9961853027344,
      "loss": 4.4679,
      "margin_dpo/margin_mean": 35.69501876831055,
      "margin_dpo/margin_std": 55.568878173828125,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -15.151107788085938,
      "KL/mean": -40.52812957763672,
      "KL/rejected_KL_mean": -65.90514373779297,
      "KL/std": 53.086029052734375,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.011207924224436283,
      "fcm_dpo/delta": -0.0756114274263382,
      "fcm_dpo/margin": 50.7540283203125,
      "fcm_dpo/q_t": 0.37166255712509155,
      "grad_norm": 40.7602424621582,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": 1.5834723711013794,
      "logits/rejected": 1.5679600238800049,
      "logps/chosen": -336.80462646484375,
      "logps/ref_chosen": -321.65350341796875,
      "logps/ref_rejected": -282.73773193359375,
      "logps/rejected": -348.6428527832031,
      "loss": 4.0123,
      "margin_dpo/margin_mean": 50.7540283203125,
      "margin_dpo/margin_std": 57.93414306640625,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -16.105470657348633,
      "KL/mean": -38.43436050415039,
      "KL/rejected_KL_mean": -60.76324462890625,
      "KL/std": 53.908477783203125,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.011064110323786736,
      "fcm_dpo/delta": 0.005283408798277378,
      "fcm_dpo/margin": 44.65777587890625,
      "fcm_dpo/q_t": 0.38940995931625366,
      "grad_norm": 38.84586715698242,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": 1.4237099885940552,
      "logits/rejected": 1.3922898769378662,
      "logps/chosen": -325.85595703125,
      "logps/ref_chosen": -309.75048828125,
      "logps/ref_rejected": -267.41925048828125,
      "logps/rejected": -328.1824951171875,
      "loss": 4.2329,
      "margin_dpo/margin_mean": 44.65777587890625,
      "margin_dpo/margin_std": 58.917694091796875,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -17.231529235839844,
      "KL/mean": -34.09688186645508,
      "KL/rejected_KL_mean": -50.96222686767578,
      "KL/std": 51.5207405090332,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.011154398322105408,
      "fcm_dpo/delta": 0.04066167771816254,
      "fcm_dpo/margin": 33.73068618774414,
      "fcm_dpo/q_t": 0.4165722131729126,
      "grad_norm": 42.78074645996094,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 1.5480916500091553,
      "logits/rejected": 1.5205127000808716,
      "logps/chosen": -328.2256164550781,
      "logps/ref_chosen": -310.9941101074219,
      "logps/ref_rejected": -279.94915771484375,
      "logps/rejected": -330.9114074707031,
      "loss": 4.596,
      "margin_dpo/margin_mean": 33.73068618774414,
      "margin_dpo/margin_std": 57.40940856933594,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -17.0875301361084,
      "KL/mean": -36.890655517578125,
      "KL/rejected_KL_mean": -56.69377899169922,
      "KL/std": 49.554542541503906,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.011920232325792313,
      "fcm_dpo/delta": 0.02717231586575508,
      "fcm_dpo/margin": 39.60626220703125,
      "fcm_dpo/q_t": 0.3969038128852844,
      "grad_norm": 33.1613655090332,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": 1.5148221254348755,
      "logits/rejected": 1.4312556982040405,
      "logps/chosen": -326.2709655761719,
      "logps/ref_chosen": -309.18341064453125,
      "logps/ref_rejected": -256.9963073730469,
      "logps/rejected": -313.6900939941406,
      "loss": 4.3855,
      "margin_dpo/margin_mean": 39.606258392333984,
      "margin_dpo/margin_std": 60.91539764404297,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -23.11258888244629,
      "KL/mean": -41.729217529296875,
      "KL/rejected_KL_mean": -60.34584045410156,
      "KL/std": 52.117401123046875,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.01182305347174406,
      "fcm_dpo/delta": 0.013918593525886536,
      "fcm_dpo/margin": 37.233245849609375,
      "fcm_dpo/q_t": 0.40345197916030884,
      "grad_norm": 42.10469055175781,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": 1.596171259880066,
      "logits/rejected": 1.6737086772918701,
      "logps/chosen": -305.38775634765625,
      "logps/ref_chosen": -282.275146484375,
      "logps/ref_rejected": -255.17941284179688,
      "logps/rejected": -315.5252685546875,
      "loss": 4.4922,
      "margin_dpo/margin_mean": 37.23324966430664,
      "margin_dpo/margin_std": 59.430572509765625,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -19.70028305053711,
      "KL/mean": -36.93345642089844,
      "KL/rejected_KL_mean": -54.166629791259766,
      "KL/std": 49.282630920410156,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.012087942101061344,
      "fcm_dpo/delta": 0.04796898365020752,
      "fcm_dpo/margin": 34.466339111328125,
      "fcm_dpo/q_t": 0.4094892740249634,
      "grad_norm": 35.60456848144531,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": 1.4271339178085327,
      "logits/rejected": 1.4634987115859985,
      "logps/chosen": -312.5775146484375,
      "logps/ref_chosen": -292.8772277832031,
      "logps/ref_rejected": -266.78509521484375,
      "logps/rejected": -320.95172119140625,
      "loss": 4.5336,
      "margin_dpo/margin_mean": 34.466339111328125,
      "margin_dpo/margin_std": 58.78628921508789,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -19.72160530090332,
      "KL/mean": -36.699974060058594,
      "KL/rejected_KL_mean": -53.67835235595703,
      "KL/std": 51.60020446777344,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.012709878385066986,
      "fcm_dpo/delta": 0.027609670534729958,
      "fcm_dpo/margin": 33.95673751831055,
      "fcm_dpo/q_t": 0.4057046175003052,
      "grad_norm": 44.56471252441406,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": 1.4383747577667236,
      "logits/rejected": 1.4960356950759888,
      "logps/chosen": -292.6153259277344,
      "logps/ref_chosen": -272.8937072753906,
      "logps/ref_rejected": -293.058349609375,
      "logps/rejected": -346.7366943359375,
      "loss": 4.4854,
      "margin_dpo/margin_mean": 33.95673370361328,
      "margin_dpo/margin_std": 54.99250793457031,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -19.308679580688477,
      "KL/mean": -37.56743240356445,
      "KL/rejected_KL_mean": -55.82619094848633,
      "KL/std": 48.611541748046875,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.012768435291945934,
      "fcm_dpo/delta": 0.03443355858325958,
      "fcm_dpo/margin": 36.51750946044922,
      "fcm_dpo/q_t": 0.398612916469574,
      "grad_norm": 51.5749397277832,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": 1.5058550834655762,
      "logits/rejected": 1.5141184329986572,
      "logps/chosen": -278.4502868652344,
      "logps/ref_chosen": -259.1416015625,
      "logps/ref_rejected": -229.64996337890625,
      "logps/rejected": -285.4761657714844,
      "loss": 4.4789,
      "margin_dpo/margin_mean": 36.51750946044922,
      "margin_dpo/margin_std": 61.26940155029297,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -23.124292373657227,
      "KL/mean": -42.074790954589844,
      "KL/rejected_KL_mean": -61.025299072265625,
      "KL/std": 52.943092346191406,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.012788144871592522,
      "fcm_dpo/delta": -0.046134088188409805,
      "fcm_dpo/margin": 37.90099334716797,
      "fcm_dpo/q_t": 0.3970465362071991,
      "grad_norm": 53.65964126586914,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 1.2722187042236328,
      "logits/rejected": 1.1910076141357422,
      "logps/chosen": -319.46221923828125,
      "logps/ref_chosen": -296.33795166015625,
      "logps/ref_rejected": -283.7359924316406,
      "logps/rejected": -344.76129150390625,
      "loss": 4.3863,
      "margin_dpo/margin_mean": 37.90099334716797,
      "margin_dpo/margin_std": 59.1367301940918,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -22.579885482788086,
      "KL/mean": -42.8243293762207,
      "KL/rejected_KL_mean": -63.06876754760742,
      "KL/std": 51.808650970458984,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.012468318454921246,
      "fcm_dpo/delta": -0.005604317411780357,
      "fcm_dpo/margin": 40.48888397216797,
      "fcm_dpo/q_t": 0.3899284303188324,
      "grad_norm": 46.653629302978516,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 1.2423440217971802,
      "logits/rejected": 1.273317575454712,
      "logps/chosen": -271.5531005859375,
      "logps/ref_chosen": -248.97320556640625,
      "logps/ref_rejected": -259.5140075683594,
      "logps/rejected": -322.582763671875,
      "loss": 4.2553,
      "margin_dpo/margin_mean": 40.48888397216797,
      "margin_dpo/margin_std": 57.554447174072266,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -12.573789596557617,
      "KL/mean": -37.9705924987793,
      "KL/rejected_KL_mean": -63.36738967895508,
      "KL/std": 51.714759826660156,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.011762147769331932,
      "fcm_dpo/delta": -0.1425057351589203,
      "fcm_dpo/margin": 50.793601989746094,
      "fcm_dpo/q_t": 0.36649465560913086,
      "grad_norm": 40.971343994140625,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": 1.4624993801116943,
      "logits/rejected": 1.6188793182373047,
      "logps/chosen": -293.2460632324219,
      "logps/ref_chosen": -280.67230224609375,
      "logps/ref_rejected": -272.4359130859375,
      "logps/rejected": -335.80328369140625,
      "loss": 3.9684,
      "margin_dpo/margin_mean": 50.793601989746094,
      "margin_dpo/margin_std": 58.03511047363281,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -18.167877197265625,
      "KL/mean": -37.51932144165039,
      "KL/rejected_KL_mean": -56.87077331542969,
      "KL/std": 50.49702072143555,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.01122565008699894,
      "fcm_dpo/delta": 0.012828906066715717,
      "fcm_dpo/margin": 38.70289611816406,
      "fcm_dpo/q_t": 0.40281277894973755,
      "grad_norm": 48.54627227783203,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": 1.1088013648986816,
      "logits/rejected": 1.155146837234497,
      "logps/chosen": -285.82635498046875,
      "logps/ref_chosen": -267.658447265625,
      "logps/ref_rejected": -262.9810791015625,
      "logps/rejected": -319.85186767578125,
      "loss": 4.4169,
      "margin_dpo/margin_mean": 38.7028923034668,
      "margin_dpo/margin_std": 57.64866256713867,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -22.250295639038086,
      "KL/mean": -36.89751434326172,
      "KL/rejected_KL_mean": -51.544734954833984,
      "KL/std": 49.36395263671875,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.011644646525382996,
      "fcm_dpo/delta": 0.0835588350892067,
      "fcm_dpo/margin": 29.29442596435547,
      "fcm_dpo/q_t": 0.4251154363155365,
      "grad_norm": 64.78131103515625,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": 1.354371428489685,
      "logits/rejected": 1.5280797481536865,
      "logps/chosen": -288.4968566894531,
      "logps/ref_chosen": -266.2465515136719,
      "logps/ref_rejected": -281.1450500488281,
      "logps/rejected": -332.6897888183594,
      "loss": 4.7896,
      "margin_dpo/margin_mean": 29.29442596435547,
      "margin_dpo/margin_std": 61.07029724121094,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -19.610706329345703,
      "KL/mean": -38.35960388183594,
      "KL/rejected_KL_mean": -57.108489990234375,
      "KL/std": 50.63059997558594,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.012117886915802956,
      "fcm_dpo/delta": 0.004870301112532616,
      "fcm_dpo/margin": 37.49778747558594,
      "fcm_dpo/q_t": 0.4002341628074646,
      "grad_norm": 45.722957611083984,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": 1.4360607862472534,
      "logits/rejected": 1.4193494319915771,
      "logps/chosen": -293.9165344238281,
      "logps/ref_chosen": -274.30584716796875,
      "logps/ref_rejected": -242.94247436523438,
      "logps/rejected": -300.05096435546875,
      "loss": 4.4096,
      "margin_dpo/margin_mean": 37.49778747558594,
      "margin_dpo/margin_std": 57.76074981689453,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -14.037437438964844,
      "KL/mean": -37.606449127197266,
      "KL/rejected_KL_mean": -61.175453186035156,
      "KL/std": 54.39940643310547,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.011380741372704506,
      "fcm_dpo/delta": -0.09181643277406693,
      "fcm_dpo/margin": 47.13800048828125,
      "fcm_dpo/q_t": 0.3790362775325775,
      "grad_norm": 33.48923110961914,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": 1.4692318439483643,
      "logits/rejected": 1.5392388105392456,
      "logps/chosen": -323.30389404296875,
      "logps/ref_chosen": -309.2664489746094,
      "logps/ref_rejected": -298.4182434082031,
      "logps/rejected": -359.59368896484375,
      "loss": 4.1508,
      "margin_dpo/margin_mean": 47.13800048828125,
      "margin_dpo/margin_std": 59.184669494628906,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -15.954225540161133,
      "KL/mean": -37.39420700073242,
      "KL/rejected_KL_mean": -58.834205627441406,
      "KL/std": 52.05155563354492,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.011135599575936794,
      "fcm_dpo/delta": -0.02464967593550682,
      "fcm_dpo/margin": 42.87996292114258,
      "fcm_dpo/q_t": 0.39456868171691895,
      "grad_norm": 40.46133804321289,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 1.4636626243591309,
      "logits/rejected": 1.5047798156738281,
      "logps/chosen": -279.97515869140625,
      "logps/ref_chosen": -264.0209045410156,
      "logps/ref_rejected": -264.3447570800781,
      "logps/rejected": -323.178955078125,
      "loss": 4.3339,
      "margin_dpo/margin_mean": 42.879966735839844,
      "margin_dpo/margin_std": 61.15083694458008,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -25.36756706237793,
      "KL/mean": -41.74524688720703,
      "KL/rejected_KL_mean": -58.12294006347656,
      "KL/std": 49.13310241699219,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.010979863815009594,
      "fcm_dpo/delta": 0.011472878977656364,
      "fcm_dpo/margin": 32.755374908447266,
      "fcm_dpo/q_t": 0.4195972681045532,
      "grad_norm": 64.9629135131836,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": 1.2022345066070557,
      "logits/rejected": 1.227638602256775,
      "logps/chosen": -288.5863952636719,
      "logps/ref_chosen": -263.2188415527344,
      "logps/ref_rejected": -236.08546447753906,
      "logps/rejected": -294.2083740234375,
      "loss": 4.679,
      "margin_dpo/margin_mean": 32.75537872314453,
      "margin_dpo/margin_std": 59.96107864379883,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -18.261789321899414,
      "KL/mean": -40.03676223754883,
      "KL/rejected_KL_mean": -61.811737060546875,
      "KL/std": 49.217979431152344,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.010898372158408165,
      "fcm_dpo/delta": -0.013979313895106316,
      "fcm_dpo/margin": 43.549949645996094,
      "fcm_dpo/q_t": 0.3924688696861267,
      "grad_norm": 35.77434158325195,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": 1.3733233213424683,
      "logits/rejected": 1.3439801931381226,
      "logps/chosen": -292.0596923828125,
      "logps/ref_chosen": -273.79791259765625,
      "logps/ref_rejected": -242.4090118408203,
      "logps/rejected": -304.2207336425781,
      "loss": 4.2749,
      "margin_dpo/margin_mean": 43.54994583129883,
      "margin_dpo/margin_std": 58.43647384643555,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -18.243600845336914,
      "KL/mean": -39.306129455566406,
      "KL/rejected_KL_mean": -60.36865997314453,
      "KL/std": 49.500877380371094,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.011104627512395382,
      "fcm_dpo/delta": -0.011893635615706444,
      "fcm_dpo/margin": 42.12506103515625,
      "fcm_dpo/q_t": 0.39543405175209045,
      "grad_norm": 34.24272537231445,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 1.5424901247024536,
      "logits/rejected": 1.608802080154419,
      "logps/chosen": -293.55877685546875,
      "logps/ref_chosen": -275.3151550292969,
      "logps/ref_rejected": -278.1514587402344,
      "logps/rejected": -338.5201416015625,
      "loss": 4.39,
      "margin_dpo/margin_mean": 42.12506103515625,
      "margin_dpo/margin_std": 63.668968200683594,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -24.000417709350586,
      "KL/mean": -43.769691467285156,
      "KL/rejected_KL_mean": -63.538963317871094,
      "KL/std": 52.55134582519531,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.010798408649861813,
      "fcm_dpo/delta": -0.016203459352254868,
      "fcm_dpo/margin": 39.53855514526367,
      "fcm_dpo/q_t": 0.40386566519737244,
      "grad_norm": 34.11899185180664,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.3245618343353271,
      "logits/rejected": 1.3044065237045288,
      "logps/chosen": -299.19976806640625,
      "logps/ref_chosen": -275.1993408203125,
      "logps/ref_rejected": -261.32122802734375,
      "logps/rejected": -324.86016845703125,
      "loss": 4.3741,
      "margin_dpo/margin_mean": 39.53855514526367,
      "margin_dpo/margin_std": 54.482337951660156,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -17.678415298461914,
      "KL/mean": -43.60883712768555,
      "KL/rejected_KL_mean": -69.53927612304688,
      "KL/std": 52.538875579833984,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.010155164636671543,
      "fcm_dpo/delta": -0.08313286304473877,
      "fcm_dpo/margin": 51.86085891723633,
      "fcm_dpo/q_t": 0.380168080329895,
      "grad_norm": 29.42951011657715,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": 1.3510220050811768,
      "logits/rejected": 1.345265507698059,
      "logps/chosen": -317.5061950683594,
      "logps/ref_chosen": -299.8277587890625,
      "logps/ref_rejected": -251.24798583984375,
      "logps/rejected": -320.7872619628906,
      "loss": 4.037,
      "margin_dpo/margin_mean": 51.86085891723633,
      "margin_dpo/margin_std": 55.776878356933594,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -16.977706909179688,
      "KL/mean": -35.843013763427734,
      "KL/rejected_KL_mean": -54.708316802978516,
      "KL/std": 49.79145050048828,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.010442487895488739,
      "fcm_dpo/delta": 0.0673639252781868,
      "fcm_dpo/margin": 37.730594635009766,
      "fcm_dpo/q_t": 0.41075730323791504,
      "grad_norm": 31.711402893066406,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": 1.3629209995269775,
      "logits/rejected": 1.3401265144348145,
      "logps/chosen": -297.4388122558594,
      "logps/ref_chosen": -280.4610900878906,
      "logps/ref_rejected": -228.34823608398438,
      "logps/rejected": -283.0565490722656,
      "loss": 4.4996,
      "margin_dpo/margin_mean": 37.730594635009766,
      "margin_dpo/margin_std": 59.0628662109375,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -14.191116333007812,
      "KL/mean": -32.12702178955078,
      "KL/rejected_KL_mean": -50.06291961669922,
      "KL/std": 49.38643264770508,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.01082461979240179,
      "fcm_dpo/delta": 0.06465308368206024,
      "fcm_dpo/margin": 35.871803283691406,
      "fcm_dpo/q_t": 0.4143372178077698,
      "grad_norm": 38.789031982421875,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": 1.4471261501312256,
      "logits/rejected": 1.553557276725769,
      "logps/chosen": -333.79119873046875,
      "logps/ref_chosen": -319.60009765625,
      "logps/ref_rejected": -316.42822265625,
      "logps/rejected": -366.49114990234375,
      "loss": 4.5299,
      "margin_dpo/margin_mean": 35.871803283691406,
      "margin_dpo/margin_std": 56.565269470214844,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -20.945091247558594,
      "KL/mean": -40.40441131591797,
      "KL/rejected_KL_mean": -59.86372756958008,
      "KL/std": 49.495975494384766,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.011146181263029575,
      "fcm_dpo/delta": 0.026772109791636467,
      "fcm_dpo/margin": 38.91862106323242,
      "fcm_dpo/q_t": 0.4034496545791626,
      "grad_norm": 34.591041564941406,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": 1.2266935110092163,
      "logits/rejected": 1.3463408946990967,
      "logps/chosen": -281.3629150390625,
      "logps/ref_chosen": -260.4178466796875,
      "logps/ref_rejected": -270.04498291015625,
      "logps/rejected": -329.9087219238281,
      "loss": 4.3858,
      "margin_dpo/margin_mean": 38.91862106323242,
      "margin_dpo/margin_std": 55.231468200683594,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -19.703876495361328,
      "KL/mean": -40.05135726928711,
      "KL/rejected_KL_mean": -60.39884948730469,
      "KL/std": 51.66360092163086,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.011482182890176773,
      "fcm_dpo/delta": -0.009437860921025276,
      "fcm_dpo/margin": 40.69496154785156,
      "fcm_dpo/q_t": 0.39525914192199707,
      "grad_norm": 45.27464294433594,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": 1.4995245933532715,
      "logits/rejected": 1.4933526515960693,
      "logps/chosen": -300.4068908691406,
      "logps/ref_chosen": -280.7030029296875,
      "logps/ref_rejected": -250.76385498046875,
      "logps/rejected": -311.1626892089844,
      "loss": 4.3569,
      "margin_dpo/margin_mean": 40.69496154785156,
      "margin_dpo/margin_std": 60.03588104248047,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -16.664989471435547,
      "KL/mean": -35.9298210144043,
      "KL/rejected_KL_mean": -55.19466018676758,
      "KL/std": 52.06108474731445,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.011229470372200012,
      "fcm_dpo/delta": -0.03555213287472725,
      "fcm_dpo/margin": 38.529685974121094,
      "fcm_dpo/q_t": 0.4027198851108551,
      "grad_norm": 31.177494049072266,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": 1.3935184478759766,
      "logits/rejected": 1.5095486640930176,
      "logps/chosen": -317.5514221191406,
      "logps/ref_chosen": -300.8864440917969,
      "logps/ref_rejected": -288.5682067871094,
      "logps/rejected": -343.7629089355469,
      "loss": 4.4557,
      "margin_dpo/margin_mean": 38.529685974121094,
      "margin_dpo/margin_std": 58.160621643066406,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -27.76289176940918,
      "KL/mean": -42.2302360534668,
      "KL/rejected_KL_mean": -56.69757080078125,
      "KL/std": 52.83584976196289,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.011274194344878197,
      "fcm_dpo/delta": 0.08988148719072342,
      "fcm_dpo/margin": 28.934680938720703,
      "fcm_dpo/q_t": 0.4293268024921417,
      "grad_norm": 45.63084030151367,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": 1.398150086402893,
      "logits/rejected": 1.4577387571334839,
      "logps/chosen": -321.06524658203125,
      "logps/ref_chosen": -293.3023681640625,
      "logps/ref_rejected": -270.0457763671875,
      "logps/rejected": -326.74334716796875,
      "loss": 4.9015,
      "margin_dpo/margin_mean": 28.934680938720703,
      "margin_dpo/margin_std": 67.07368469238281,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -23.152355194091797,
      "KL/mean": -40.92193603515625,
      "KL/rejected_KL_mean": -58.69153594970703,
      "KL/std": 52.87299728393555,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.0119621055200696,
      "fcm_dpo/delta": 0.03649744391441345,
      "fcm_dpo/margin": 35.5391845703125,
      "fcm_dpo/q_t": 0.40574362874031067,
      "grad_norm": 49.534996032714844,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": 1.3166630268096924,
      "logits/rejected": 1.386913776397705,
      "logps/chosen": -306.5516052246094,
      "logps/ref_chosen": -283.39923095703125,
      "logps/ref_rejected": -264.5092468261719,
      "logps/rejected": -323.2008056640625,
      "loss": 4.4921,
      "margin_dpo/margin_mean": 35.5391845703125,
      "margin_dpo/margin_std": 57.25304412841797,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -18.876728057861328,
      "KL/mean": -39.506378173828125,
      "KL/rejected_KL_mean": -60.136024475097656,
      "KL/std": 48.92250442504883,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.011903304606676102,
      "fcm_dpo/delta": -0.03581307828426361,
      "fcm_dpo/margin": 41.25928497314453,
      "fcm_dpo/q_t": 0.3908305764198303,
      "grad_norm": 36.25236892700195,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 1.219544768333435,
      "logits/rejected": 1.280843734741211,
      "logps/chosen": -273.34454345703125,
      "logps/ref_chosen": -254.4678192138672,
      "logps/ref_rejected": -252.00326538085938,
      "logps/rejected": -312.1392822265625,
      "loss": 4.3067,
      "margin_dpo/margin_mean": 41.25928497314453,
      "margin_dpo/margin_std": 57.42374038696289,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -15.89529800415039,
      "KL/mean": -40.950904846191406,
      "KL/rejected_KL_mean": -66.00650787353516,
      "KL/std": 53.50216293334961,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.011417610570788383,
      "fcm_dpo/delta": -0.07703652232885361,
      "fcm_dpo/margin": 50.111209869384766,
      "fcm_dpo/q_t": 0.37203720211982727,
      "grad_norm": 38.11719512939453,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 1.3582206964492798,
      "logits/rejected": 1.3881762027740479,
      "logps/chosen": -326.4948425292969,
      "logps/ref_chosen": -310.5995788574219,
      "logps/ref_rejected": -292.7754821777344,
      "logps/rejected": -358.781982421875,
      "loss": 3.9913,
      "margin_dpo/margin_mean": 50.111209869384766,
      "margin_dpo/margin_std": 57.675537109375,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -15.676643371582031,
      "KL/mean": -36.687774658203125,
      "KL/rejected_KL_mean": -57.69890594482422,
      "KL/std": 52.177425384521484,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.011293474584817886,
      "fcm_dpo/delta": 0.02552787959575653,
      "fcm_dpo/margin": 42.02225112915039,
      "fcm_dpo/q_t": 0.394657701253891,
      "grad_norm": 35.380149841308594,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": 1.3912174701690674,
      "logits/rejected": 1.312447428703308,
      "logps/chosen": -315.52801513671875,
      "logps/ref_chosen": -299.8513488769531,
      "logps/ref_rejected": -278.5664367675781,
      "logps/rejected": -336.2653503417969,
      "loss": 4.3311,
      "margin_dpo/margin_mean": 42.02225112915039,
      "margin_dpo/margin_std": 60.69839859008789,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -14.702194213867188,
      "KL/mean": -42.04186248779297,
      "KL/rejected_KL_mean": -69.38153076171875,
      "KL/std": 54.13357162475586,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.010973302647471428,
      "fcm_dpo/delta": -0.10578624904155731,
      "fcm_dpo/margin": 54.67934036254883,
      "fcm_dpo/q_t": 0.3643578886985779,
      "grad_norm": 32.24766159057617,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": 1.3879142999649048,
      "logits/rejected": 1.4177368879318237,
      "logps/chosen": -317.1338195800781,
      "logps/ref_chosen": -302.4316101074219,
      "logps/ref_rejected": -270.951171875,
      "logps/rejected": -340.33270263671875,
      "loss": 3.8348,
      "margin_dpo/margin_mean": 54.679344177246094,
      "margin_dpo/margin_std": 53.50773620605469,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -23.825780868530273,
      "KL/mean": -42.96757507324219,
      "KL/rejected_KL_mean": -62.10936737060547,
      "KL/std": 54.089176177978516,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.010387415066361427,
      "fcm_dpo/delta": 0.017633480951189995,
      "fcm_dpo/margin": 38.28358840942383,
      "fcm_dpo/q_t": 0.4105108380317688,
      "grad_norm": 37.02425003051758,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": 1.2301064729690552,
      "logits/rejected": 1.251104474067688,
      "logps/chosen": -313.7288818359375,
      "logps/ref_chosen": -289.9031066894531,
      "logps/ref_rejected": -252.84918212890625,
      "logps/rejected": -314.95855712890625,
      "loss": 4.5433,
      "margin_dpo/margin_mean": 38.28358459472656,
      "margin_dpo/margin_std": 62.12881851196289,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -18.074764251708984,
      "KL/mean": -40.05653762817383,
      "KL/rejected_KL_mean": -62.03831100463867,
      "KL/std": 52.979434967041016,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.010828401893377304,
      "fcm_dpo/delta": 0.023709366098046303,
      "fcm_dpo/margin": 43.963539123535156,
      "fcm_dpo/q_t": 0.3935423195362091,
      "grad_norm": 30.54352569580078,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": 1.5389039516448975,
      "logits/rejected": 1.7237071990966797,
      "logps/chosen": -266.2488098144531,
      "logps/ref_chosen": -248.17404174804688,
      "logps/ref_rejected": -252.23764038085938,
      "logps/rejected": -314.27593994140625,
      "loss": 4.2426,
      "margin_dpo/margin_mean": 43.963539123535156,
      "margin_dpo/margin_std": 56.70342254638672,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -23.380550384521484,
      "KL/mean": -40.96479415893555,
      "KL/rejected_KL_mean": -58.54903793334961,
      "KL/std": 51.98546600341797,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.010728088207542896,
      "fcm_dpo/delta": -0.0049120597541332245,
      "fcm_dpo/margin": 35.168479919433594,
      "fcm_dpo/q_t": 0.415068119764328,
      "grad_norm": 35.82801055908203,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": 1.417409062385559,
      "logits/rejected": 1.4918591976165771,
      "logps/chosen": -299.7966613769531,
      "logps/ref_chosen": -276.4161376953125,
      "logps/ref_rejected": -264.0082702636719,
      "logps/rejected": -322.55731201171875,
      "loss": 4.604,
      "margin_dpo/margin_mean": 35.168479919433594,
      "margin_dpo/margin_std": 60.155635833740234,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -28.203784942626953,
      "KL/mean": -48.73345947265625,
      "KL/rejected_KL_mean": -69.26313781738281,
      "KL/std": 54.13156509399414,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.011002177372574806,
      "fcm_dpo/delta": 0.05017382279038429,
      "fcm_dpo/margin": 41.05936050415039,
      "fcm_dpo/q_t": 0.4001734256744385,
      "grad_norm": 35.026241302490234,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": 1.376771092414856,
      "logits/rejected": 1.3795642852783203,
      "logps/chosen": -285.6411437988281,
      "logps/ref_chosen": -257.4373474121094,
      "logps/ref_rejected": -219.66390991210938,
      "logps/rejected": -288.92706298828125,
      "loss": 4.3622,
      "margin_dpo/margin_mean": 41.05936050415039,
      "margin_dpo/margin_std": 60.55333709716797,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -19.7432861328125,
      "KL/mean": -40.38613510131836,
      "KL/rejected_KL_mean": -61.02897644042969,
      "KL/std": 51.502532958984375,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.011513455770909786,
      "fcm_dpo/delta": 0.023320961743593216,
      "fcm_dpo/margin": 41.28569030761719,
      "fcm_dpo/q_t": 0.3939526677131653,
      "grad_norm": 49.8958740234375,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": 1.2136945724487305,
      "logits/rejected": 1.3158950805664062,
      "logps/chosen": -308.39666748046875,
      "logps/ref_chosen": -288.6533508300781,
      "logps/ref_rejected": -291.54107666015625,
      "logps/rejected": -352.570068359375,
      "loss": 4.3268,
      "margin_dpo/margin_mean": 41.28569030761719,
      "margin_dpo/margin_std": 57.69767761230469,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -21.483179092407227,
      "KL/mean": -43.31647872924805,
      "KL/rejected_KL_mean": -65.1497802734375,
      "KL/std": 53.140045166015625,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.011304824613034725,
      "fcm_dpo/delta": 0.005474124103784561,
      "fcm_dpo/margin": 43.666587829589844,
      "fcm_dpo/q_t": 0.3920544683933258,
      "grad_norm": 37.852779388427734,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": 1.3899508714675903,
      "logits/rejected": 1.4860026836395264,
      "logps/chosen": -299.2205810546875,
      "logps/ref_chosen": -277.7373962402344,
      "logps/ref_rejected": -272.988525390625,
      "logps/rejected": -338.1383056640625,
      "loss": 4.3333,
      "margin_dpo/margin_mean": 43.666587829589844,
      "margin_dpo/margin_std": 65.1388168334961,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -19.662260055541992,
      "KL/mean": -39.06217956542969,
      "KL/rejected_KL_mean": -58.46210479736328,
      "KL/std": 52.93661880493164,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.011583887040615082,
      "fcm_dpo/delta": 0.008973626419901848,
      "fcm_dpo/margin": 38.79985046386719,
      "fcm_dpo/q_t": 0.40156951546669006,
      "grad_norm": 42.474666595458984,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 1.5670636892318726,
      "logits/rejected": 1.5393446683883667,
      "logps/chosen": -293.88336181640625,
      "logps/ref_chosen": -274.2210998535156,
      "logps/ref_rejected": -264.33465576171875,
      "logps/rejected": -322.7967224121094,
      "loss": 4.4705,
      "margin_dpo/margin_mean": 38.79985046386719,
      "margin_dpo/margin_std": 63.52775955200195,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -16.074779510498047,
      "KL/mean": -38.73295974731445,
      "KL/rejected_KL_mean": -61.391136169433594,
      "KL/std": 53.79762649536133,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.011070022359490395,
      "fcm_dpo/delta": -0.0911090224981308,
      "fcm_dpo/margin": 45.31635284423828,
      "fcm_dpo/q_t": 0.38734516501426697,
      "grad_norm": 35.035518646240234,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 1.4694738388061523,
      "logits/rejected": 1.5356965065002441,
      "logps/chosen": -288.9427795410156,
      "logps/ref_chosen": -272.86798095703125,
      "logps/ref_rejected": -250.35772705078125,
      "logps/rejected": -311.74884033203125,
      "loss": 4.1976,
      "margin_dpo/margin_mean": 45.31635284423828,
      "margin_dpo/margin_std": 56.256591796875,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -25.159500122070312,
      "KL/mean": -44.854698181152344,
      "KL/rejected_KL_mean": -64.54989624023438,
      "KL/std": 53.80162048339844,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.010567895136773586,
      "fcm_dpo/delta": 0.0015385362785309553,
      "fcm_dpo/margin": 39.39039993286133,
      "fcm_dpo/q_t": 0.4071783423423767,
      "grad_norm": 38.27120590209961,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": 1.4318606853485107,
      "logits/rejected": 1.5380160808563232,
      "logps/chosen": -293.0787048339844,
      "logps/ref_chosen": -267.919189453125,
      "logps/ref_rejected": -271.73162841796875,
      "logps/rejected": -336.281494140625,
      "loss": 4.5042,
      "margin_dpo/margin_mean": 39.39039993286133,
      "margin_dpo/margin_std": 63.769371032714844,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -18.302806854248047,
      "KL/mean": -36.312618255615234,
      "KL/rejected_KL_mean": -54.32244110107422,
      "KL/std": 50.472713470458984,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.011323437094688416,
      "fcm_dpo/delta": 0.0946590006351471,
      "fcm_dpo/margin": 36.01963424682617,
      "fcm_dpo/q_t": 0.4100503921508789,
      "grad_norm": 39.72618865966797,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 1.3405991792678833,
      "logits/rejected": 1.3967691659927368,
      "logps/chosen": -283.2608642578125,
      "logps/ref_chosen": -264.95806884765625,
      "logps/ref_rejected": -267.40093994140625,
      "logps/rejected": -321.723388671875,
      "loss": 4.4608,
      "margin_dpo/margin_mean": 36.01963424682617,
      "margin_dpo/margin_std": 56.0051155090332,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -24.045391082763672,
      "KL/mean": -44.290157318115234,
      "KL/rejected_KL_mean": -64.534912109375,
      "KL/std": 53.478126525878906,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.011572899296879768,
      "fcm_dpo/delta": -0.01839069277048111,
      "fcm_dpo/margin": 40.48951721191406,
      "fcm_dpo/q_t": 0.3980516791343689,
      "grad_norm": 42.483646392822266,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": 1.5260902643203735,
      "logits/rejected": 1.5954906940460205,
      "logps/chosen": -314.0511779785156,
      "logps/ref_chosen": -290.00579833984375,
      "logps/ref_rejected": -266.8810119628906,
      "logps/rejected": -331.4159240722656,
      "loss": 4.4354,
      "margin_dpo/margin_mean": 40.48951721191406,
      "margin_dpo/margin_std": 64.33303833007812,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -15.697652816772461,
      "KL/mean": -40.094818115234375,
      "KL/rejected_KL_mean": -64.49199676513672,
      "KL/std": 53.052703857421875,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.011288761161267757,
      "fcm_dpo/delta": -0.053877346217632294,
      "fcm_dpo/margin": 48.794334411621094,
      "fcm_dpo/q_t": 0.37818583846092224,
      "grad_norm": 48.09061050415039,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": 1.4707834720611572,
      "logits/rejected": 1.5626002550125122,
      "logps/chosen": -305.5086669921875,
      "logps/ref_chosen": -289.81103515625,
      "logps/ref_rejected": -278.2758483886719,
      "logps/rejected": -342.76788330078125,
      "loss": 4.1,
      "margin_dpo/margin_mean": 48.794334411621094,
      "margin_dpo/margin_std": 61.095375061035156,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -15.125234603881836,
      "KL/mean": -38.87482452392578,
      "KL/rejected_KL_mean": -62.624412536621094,
      "KL/std": 52.5377082824707,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.010466261766850948,
      "fcm_dpo/delta": -0.04488556832075119,
      "fcm_dpo/margin": 47.499176025390625,
      "fcm_dpo/q_t": 0.38805490732192993,
      "grad_norm": 34.695640563964844,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": 1.4594428539276123,
      "logits/rejected": 1.427567720413208,
      "logps/chosen": -319.7740478515625,
      "logps/ref_chosen": -304.6488037109375,
      "logps/ref_rejected": -264.2703857421875,
      "logps/rejected": -326.894775390625,
      "loss": 4.2265,
      "margin_dpo/margin_mean": 47.499176025390625,
      "margin_dpo/margin_std": 61.805381774902344,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -19.748428344726562,
      "KL/mean": -39.969627380371094,
      "KL/rejected_KL_mean": -60.19082260131836,
      "KL/std": 51.62503433227539,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.010661320760846138,
      "fcm_dpo/delta": 0.02371985837817192,
      "fcm_dpo/margin": 40.44239807128906,
      "fcm_dpo/q_t": 0.40345117449760437,
      "grad_norm": 40.2073860168457,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": 1.415156364440918,
      "logits/rejected": 1.4939631223678589,
      "logps/chosen": -321.6373291015625,
      "logps/ref_chosen": -301.8889465332031,
      "logps/ref_rejected": -275.0347595214844,
      "logps/rejected": -335.2255859375,
      "loss": 4.4399,
      "margin_dpo/margin_mean": 40.44239807128906,
      "margin_dpo/margin_std": 60.58783721923828,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -26.304859161376953,
      "KL/mean": -44.249900817871094,
      "KL/rejected_KL_mean": -62.194942474365234,
      "KL/std": 53.157920837402344,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.01086009107530117,
      "fcm_dpo/delta": 0.00010268203914165497,
      "fcm_dpo/margin": 35.890098571777344,
      "fcm_dpo/q_t": 0.41238659620285034,
      "grad_norm": 82.71565246582031,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": 1.2534568309783936,
      "logits/rejected": 1.2661184072494507,
      "logps/chosen": -295.55218505859375,
      "logps/ref_chosen": -269.2473449707031,
      "logps/ref_rejected": -275.5192565917969,
      "logps/rejected": -337.7142028808594,
      "loss": 4.5475,
      "margin_dpo/margin_mean": 35.890098571777344,
      "margin_dpo/margin_std": 56.72612762451172,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -24.882080078125,
      "KL/mean": -49.287391662597656,
      "KL/rejected_KL_mean": -73.69270324707031,
      "KL/std": 56.32617950439453,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.010389548726379871,
      "fcm_dpo/delta": -0.04790262132883072,
      "fcm_dpo/margin": 48.810630798339844,
      "fcm_dpo/q_t": 0.38619670271873474,
      "grad_norm": 34.34525680541992,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": 1.4124712944030762,
      "logits/rejected": 1.370522379875183,
      "logps/chosen": -316.04248046875,
      "logps/ref_chosen": -291.1604309082031,
      "logps/ref_rejected": -245.18325805664062,
      "logps/rejected": -318.8759460449219,
      "loss": 4.229,
      "margin_dpo/margin_mean": 48.810630798339844,
      "margin_dpo/margin_std": 65.267822265625,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -22.45132064819336,
      "KL/mean": -45.44701385498047,
      "KL/rejected_KL_mean": -68.44271087646484,
      "KL/std": 55.12582015991211,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.010479898191988468,
      "fcm_dpo/delta": 0.018012573942542076,
      "fcm_dpo/margin": 45.99137878417969,
      "fcm_dpo/q_t": 0.39130717515945435,
      "grad_norm": 34.439666748046875,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 1.088689923286438,
      "logits/rejected": 1.018422245979309,
      "logps/chosen": -280.9211120605469,
      "logps/ref_chosen": -258.4698181152344,
      "logps/ref_rejected": -242.916015625,
      "logps/rejected": -311.35870361328125,
      "loss": 4.2769,
      "margin_dpo/margin_mean": 45.99137878417969,
      "margin_dpo/margin_std": 63.322425842285156,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -14.223943710327148,
      "KL/mean": -37.86970520019531,
      "KL/rejected_KL_mean": -61.51546859741211,
      "KL/std": 51.51787185668945,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.010311231017112732,
      "fcm_dpo/delta": -0.027793942019343376,
      "fcm_dpo/margin": 47.291534423828125,
      "fcm_dpo/q_t": 0.3888552784919739,
      "grad_norm": 34.040496826171875,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.3643343448638916,
      "logits/rejected": 1.3591418266296387,
      "logps/chosen": -304.36474609375,
      "logps/ref_chosen": -290.1407775878906,
      "logps/ref_rejected": -239.65676879882812,
      "logps/rejected": -301.1722106933594,
      "loss": 4.1796,
      "margin_dpo/margin_mean": 47.29153060913086,
      "margin_dpo/margin_std": 57.309906005859375,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -22.291967391967773,
      "KL/mean": -40.77346420288086,
      "KL/rejected_KL_mean": -59.254966735839844,
      "KL/std": 52.1849479675293,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.01080926414579153,
      "fcm_dpo/delta": 0.10365541279315948,
      "fcm_dpo/margin": 36.96299362182617,
      "fcm_dpo/q_t": 0.4106569290161133,
      "grad_norm": 43.90497970581055,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": 1.2788583040237427,
      "logits/rejected": 1.34122633934021,
      "logps/chosen": -299.2279968261719,
      "logps/ref_chosen": -276.93603515625,
      "logps/ref_rejected": -250.398681640625,
      "logps/rejected": -309.6536560058594,
      "loss": 4.5799,
      "margin_dpo/margin_mean": 36.96299362182617,
      "margin_dpo/margin_std": 63.60847473144531,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -17.57908058166504,
      "KL/mean": -42.03938293457031,
      "KL/rejected_KL_mean": -66.49967956542969,
      "KL/std": 54.36211013793945,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.0109159080311656,
      "fcm_dpo/delta": -0.03674282133579254,
      "fcm_dpo/margin": 48.92060470581055,
      "fcm_dpo/q_t": 0.38194626569747925,
      "grad_norm": 41.604766845703125,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": 1.5626533031463623,
      "logits/rejected": 1.6782840490341187,
      "logps/chosen": -303.0091857910156,
      "logps/ref_chosen": -285.43011474609375,
      "logps/ref_rejected": -258.0743408203125,
      "logps/rejected": -324.57403564453125,
      "loss": 4.1594,
      "margin_dpo/margin_mean": 48.92060470581055,
      "margin_dpo/margin_std": 64.17723846435547,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -14.592008590698242,
      "KL/mean": -37.97684860229492,
      "KL/rejected_KL_mean": -61.3616943359375,
      "KL/std": 50.84525680541992,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.010871369391679764,
      "fcm_dpo/delta": -0.010207153856754303,
      "fcm_dpo/margin": 46.76968765258789,
      "fcm_dpo/q_t": 0.3851187229156494,
      "grad_norm": 41.6907958984375,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": 1.292434811592102,
      "logits/rejected": 1.2671380043029785,
      "logps/chosen": -313.93701171875,
      "logps/ref_chosen": -299.3450012207031,
      "logps/ref_rejected": -244.60169982910156,
      "logps/rejected": -305.96337890625,
      "loss": 4.1437,
      "margin_dpo/margin_mean": 46.76968765258789,
      "margin_dpo/margin_std": 55.90518569946289,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -20.206003189086914,
      "KL/mean": -40.522098541259766,
      "KL/rejected_KL_mean": -60.83819580078125,
      "KL/std": 54.030799865722656,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.010696171782910824,
      "fcm_dpo/delta": -0.004731988534331322,
      "fcm_dpo/margin": 40.6321907043457,
      "fcm_dpo/q_t": 0.40271416306495667,
      "grad_norm": 32.46276092529297,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": 1.5355827808380127,
      "logits/rejected": 1.629493236541748,
      "logps/chosen": -293.1557312011719,
      "logps/ref_chosen": -272.9497375488281,
      "logps/ref_rejected": -274.6335754394531,
      "logps/rejected": -335.47174072265625,
      "loss": 4.4223,
      "margin_dpo/margin_mean": 40.63219451904297,
      "margin_dpo/margin_std": 58.643898010253906,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -16.533414840698242,
      "KL/mean": -38.90666198730469,
      "KL/rejected_KL_mean": -61.279911041259766,
      "KL/std": 52.8712272644043,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.010716606862843037,
      "fcm_dpo/delta": -0.016841070726513863,
      "fcm_dpo/margin": 44.746498107910156,
      "fcm_dpo/q_t": 0.3919427692890167,
      "grad_norm": 27.23710060119629,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 1.3140270709991455,
      "logits/rejected": 1.3445709943771362,
      "logps/chosen": -298.4750061035156,
      "logps/ref_chosen": -281.9415588378906,
      "logps/ref_rejected": -253.20486450195312,
      "logps/rejected": -314.4847717285156,
      "loss": 4.2321,
      "margin_dpo/margin_mean": 44.74650192260742,
      "margin_dpo/margin_std": 57.66529083251953,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -14.179662704467773,
      "KL/mean": -43.86739730834961,
      "KL/rejected_KL_mean": -73.55513763427734,
      "KL/std": 55.219818115234375,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.010349413380026817,
      "fcm_dpo/delta": -0.11878267675638199,
      "fcm_dpo/margin": 59.37547302246094,
      "fcm_dpo/q_t": 0.361929327249527,
      "grad_norm": 34.215457916259766,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": 1.553816795349121,
      "logits/rejected": 1.5606337785720825,
      "logps/chosen": -294.7962646484375,
      "logps/ref_chosen": -280.6166076660156,
      "logps/ref_rejected": -263.7532958984375,
      "logps/rejected": -337.3084411621094,
      "loss": 3.842,
      "margin_dpo/margin_mean": 59.37546920776367,
      "margin_dpo/margin_std": 58.703224182128906,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -20.23667335510254,
      "KL/mean": -47.65900421142578,
      "KL/rejected_KL_mean": -75.08132934570312,
      "KL/std": 55.13923263549805,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.009594901464879513,
      "fcm_dpo/delta": -0.028165843337774277,
      "fcm_dpo/margin": 54.84465408325195,
      "fcm_dpo/q_t": 0.379350870847702,
      "grad_norm": 51.099918365478516,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": 1.4333425760269165,
      "logits/rejected": 1.5475447177886963,
      "logps/chosen": -306.3988952636719,
      "logps/ref_chosen": -286.1622314453125,
      "logps/ref_rejected": -273.3803405761719,
      "logps/rejected": -348.461669921875,
      "loss": 4.0078,
      "margin_dpo/margin_mean": 54.84465789794922,
      "margin_dpo/margin_std": 56.17855453491211,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -19.632305145263672,
      "KL/mean": -41.673988342285156,
      "KL/rejected_KL_mean": -63.715675354003906,
      "KL/std": 53.110145568847656,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.009629269130527973,
      "fcm_dpo/delta": 0.03850511834025383,
      "fcm_dpo/margin": 44.08338928222656,
      "fcm_dpo/q_t": 0.40505072474479675,
      "grad_norm": 28.9234619140625,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": 1.3080880641937256,
      "logits/rejected": 1.3045574426651,
      "logps/chosen": -285.9688720703125,
      "logps/ref_chosen": -266.3365783691406,
      "logps/ref_rejected": -255.01956176757812,
      "logps/rejected": -318.7352294921875,
      "loss": 4.4815,
      "margin_dpo/margin_mean": 44.0833854675293,
      "margin_dpo/margin_std": 69.21772003173828,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -19.95611572265625,
      "KL/mean": -38.97095489501953,
      "KL/rejected_KL_mean": -57.98578643798828,
      "KL/std": 51.8824462890625,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.010187498293817043,
      "fcm_dpo/delta": 0.05494547262787819,
      "fcm_dpo/margin": 38.02967071533203,
      "fcm_dpo/q_t": 0.41339510679244995,
      "grad_norm": 85.29779815673828,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 1.4678411483764648,
      "logits/rejected": 1.502480387687683,
      "logps/chosen": -302.10748291015625,
      "logps/ref_chosen": -282.1513977050781,
      "logps/ref_rejected": -263.4792175292969,
      "logps/rejected": -321.46502685546875,
      "loss": 4.5683,
      "margin_dpo/margin_mean": 38.02967071533203,
      "margin_dpo/margin_std": 61.966678619384766,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -22.195566177368164,
      "KL/mean": -40.68260192871094,
      "KL/rejected_KL_mean": -59.16963195800781,
      "KL/std": 53.342193603515625,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.010736832395195961,
      "fcm_dpo/delta": 0.06706257164478302,
      "fcm_dpo/margin": 36.97406768798828,
      "fcm_dpo/q_t": 0.4124898612499237,
      "grad_norm": 36.599178314208984,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 1.5840988159179688,
      "logits/rejected": 1.750473976135254,
      "logps/chosen": -290.90869140625,
      "logps/ref_chosen": -268.713134765625,
      "logps/ref_rejected": -266.13446044921875,
      "logps/rejected": -325.3041076660156,
      "loss": 4.5388,
      "margin_dpo/margin_mean": 36.97406768798828,
      "margin_dpo/margin_std": 61.29048538208008,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -14.437786102294922,
      "KL/mean": -37.64185333251953,
      "KL/rejected_KL_mean": -60.84591293334961,
      "KL/std": 52.954586029052734,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.010399827733635902,
      "fcm_dpo/delta": -0.021383237093687057,
      "fcm_dpo/margin": 46.408103942871094,
      "fcm_dpo/q_t": 0.39185044169425964,
      "grad_norm": 32.29280090332031,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": 1.3812689781188965,
      "logits/rejected": 1.4003459215164185,
      "logps/chosen": -298.4219970703125,
      "logps/ref_chosen": -283.9842224121094,
      "logps/ref_rejected": -253.9568634033203,
      "logps/rejected": -314.80279541015625,
      "loss": 4.2494,
      "margin_dpo/margin_mean": 46.408103942871094,
      "margin_dpo/margin_std": 62.030731201171875,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -27.02897071838379,
      "KL/mean": -45.94511795043945,
      "KL/rejected_KL_mean": -64.86125183105469,
      "KL/std": 53.14583969116211,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.010812818072736263,
      "fcm_dpo/delta": 0.09579076617956161,
      "fcm_dpo/margin": 37.8322868347168,
      "fcm_dpo/q_t": 0.40683862566947937,
      "grad_norm": 32.76231002807617,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": 1.3321561813354492,
      "logits/rejected": 1.353327751159668,
      "logps/chosen": -269.47216796875,
      "logps/ref_chosen": -242.4431915283203,
      "logps/ref_rejected": -252.87979125976562,
      "logps/rejected": -317.74102783203125,
      "loss": 4.5071,
      "margin_dpo/margin_mean": 37.8322868347168,
      "margin_dpo/margin_std": 60.308780670166016,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -22.811893463134766,
      "KL/mean": -44.53324890136719,
      "KL/rejected_KL_mean": -66.25459289550781,
      "KL/std": 53.37626647949219,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.01126051414757967,
      "fcm_dpo/delta": -0.02770174853503704,
      "fcm_dpo/margin": 43.44271469116211,
      "fcm_dpo/q_t": 0.39198434352874756,
      "grad_norm": 36.75082015991211,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": 1.6450413465499878,
      "logits/rejected": 1.5962193012237549,
      "logps/chosen": -311.0960693359375,
      "logps/ref_chosen": -288.2841796875,
      "logps/ref_rejected": -252.9123992919922,
      "logps/rejected": -319.1669921875,
      "loss": 4.2905,
      "margin_dpo/margin_mean": 43.44271469116211,
      "margin_dpo/margin_std": 61.451515197753906,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -16.973628997802734,
      "KL/mean": -36.778438568115234,
      "KL/rejected_KL_mean": -56.5832405090332,
      "KL/std": 53.24059295654297,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.01143056619912386,
      "fcm_dpo/delta": 0.04742370545864105,
      "fcm_dpo/margin": 39.60961151123047,
      "fcm_dpo/q_t": 0.3988434970378876,
      "grad_norm": 60.7858772277832,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": 1.4080660343170166,
      "logits/rejected": 1.4974591732025146,
      "logps/chosen": -265.98846435546875,
      "logps/ref_chosen": -249.01483154296875,
      "logps/ref_rejected": -243.81192016601562,
      "logps/rejected": -300.3951416015625,
      "loss": 4.3873,
      "margin_dpo/margin_mean": 39.60961151123047,
      "margin_dpo/margin_std": 58.649330139160156,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -22.817415237426758,
      "KL/mean": -37.12867736816406,
      "KL/rejected_KL_mean": -51.43994903564453,
      "KL/std": 54.305686950683594,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.01214659959077835,
      "fcm_dpo/delta": 0.11464737355709076,
      "fcm_dpo/margin": 28.622535705566406,
      "fcm_dpo/q_t": 0.42519038915634155,
      "grad_norm": 40.93937301635742,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": 1.419950008392334,
      "logits/rejected": 1.5200401544570923,
      "logps/chosen": -335.74749755859375,
      "logps/ref_chosen": -312.9300537109375,
      "logps/ref_rejected": -296.2721862792969,
      "logps/rejected": -347.712158203125,
      "loss": 4.7831,
      "margin_dpo/margin_mean": 28.62253761291504,
      "margin_dpo/margin_std": 60.185829162597656,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -16.646114349365234,
      "KL/mean": -40.418540954589844,
      "KL/rejected_KL_mean": -64.19097137451172,
      "KL/std": 55.71061325073242,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.012227932922542095,
      "fcm_dpo/delta": -0.08742790669202805,
      "fcm_dpo/margin": 47.54484939575195,
      "fcm_dpo/q_t": 0.37338733673095703,
      "grad_norm": 37.443363189697266,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": 1.5355091094970703,
      "logits/rejected": 1.5813335180282593,
      "logps/chosen": -284.96002197265625,
      "logps/ref_chosen": -268.31390380859375,
      "logps/ref_rejected": -268.7603759765625,
      "logps/rejected": -332.9513244628906,
      "loss": 4.1216,
      "margin_dpo/margin_mean": 47.54485321044922,
      "margin_dpo/margin_std": 62.3380126953125,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -28.772960662841797,
      "KL/mean": -40.1857795715332,
      "KL/rejected_KL_mean": -51.59859085083008,
      "KL/std": 52.28449249267578,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.012267351150512695,
      "fcm_dpo/delta": 0.10389041900634766,
      "fcm_dpo/margin": 22.82562255859375,
      "fcm_dpo/q_t": 0.43969616293907166,
      "grad_norm": 106.53709411621094,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": 1.248260498046875,
      "logits/rejected": 1.3186777830123901,
      "logps/chosen": -284.0048522949219,
      "logps/ref_chosen": -255.23190307617188,
      "logps/ref_rejected": -241.1186981201172,
      "logps/rejected": -292.7173156738281,
      "loss": 5.0678,
      "margin_dpo/margin_mean": 22.82562255859375,
      "margin_dpo/margin_std": 62.28240203857422,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -22.00450325012207,
      "KL/mean": -42.11490249633789,
      "KL/rejected_KL_mean": -62.225303649902344,
      "KL/std": 51.57564926147461,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.012811698019504547,
      "fcm_dpo/delta": -0.01640464924275875,
      "fcm_dpo/margin": 40.22080612182617,
      "fcm_dpo/q_t": 0.3890855312347412,
      "grad_norm": 38.171173095703125,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 1.272616982460022,
      "logits/rejected": 1.2808241844177246,
      "logps/chosen": -272.9848327636719,
      "logps/ref_chosen": -250.9803466796875,
      "logps/ref_rejected": -239.9314422607422,
      "logps/rejected": -302.15673828125,
      "loss": 4.3168,
      "margin_dpo/margin_mean": 40.220802307128906,
      "margin_dpo/margin_std": 60.64070129394531,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -27.578351974487305,
      "KL/mean": -43.23225402832031,
      "KL/rejected_KL_mean": -58.886165618896484,
      "KL/std": 51.587276458740234,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.012691297568380833,
      "fcm_dpo/delta": -0.02061920240521431,
      "fcm_dpo/margin": 31.30782127380371,
      "fcm_dpo/q_t": 0.41399869322776794,
      "grad_norm": 47.27680587768555,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 1.4274678230285645,
      "logits/rejected": 1.3561245203018188,
      "logps/chosen": -328.58111572265625,
      "logps/ref_chosen": -301.0027770996094,
      "logps/ref_rejected": -263.628662109375,
      "logps/rejected": -322.51483154296875,
      "loss": 4.7267,
      "margin_dpo/margin_mean": 31.30782127380371,
      "margin_dpo/margin_std": 60.780643463134766,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -18.036035537719727,
      "KL/mean": -41.56196594238281,
      "KL/rejected_KL_mean": -65.087890625,
      "KL/std": 53.90050506591797,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.012225436978042126,
      "fcm_dpo/delta": -0.07978902012109756,
      "fcm_dpo/margin": 47.051849365234375,
      "fcm_dpo/q_t": 0.37316304445266724,
      "grad_norm": 46.22322463989258,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 1.375359296798706,
      "logits/rejected": 1.459717035293579,
      "logps/chosen": -318.88482666015625,
      "logps/ref_chosen": -300.8487854003906,
      "logps/ref_rejected": -296.7118835449219,
      "logps/rejected": -361.7997741699219,
      "loss": 4.0438,
      "margin_dpo/margin_mean": 47.051849365234375,
      "margin_dpo/margin_std": 54.25644302368164,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -28.1229305267334,
      "KL/mean": -46.05691146850586,
      "KL/rejected_KL_mean": -63.99089050292969,
      "KL/std": 51.497562408447266,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.01181031484156847,
      "fcm_dpo/delta": 0.08106688410043716,
      "fcm_dpo/margin": 35.86796188354492,
      "fcm_dpo/q_t": 0.40860190987586975,
      "grad_norm": 66.3699951171875,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": 1.4770475625991821,
      "logits/rejected": 1.535913109779358,
      "logps/chosen": -299.7713623046875,
      "logps/ref_chosen": -271.6484069824219,
      "logps/ref_rejected": -238.21197509765625,
      "logps/rejected": -302.202880859375,
      "loss": 4.6627,
      "margin_dpo/margin_mean": 35.86796188354492,
      "margin_dpo/margin_std": 68.42342376708984,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -22.526569366455078,
      "KL/mean": -40.864990234375,
      "KL/rejected_KL_mean": -59.20341873168945,
      "KL/std": 51.709598541259766,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.012591186910867691,
      "fcm_dpo/delta": -0.004123002290725708,
      "fcm_dpo/margin": 36.676849365234375,
      "fcm_dpo/q_t": 0.3989989459514618,
      "grad_norm": 43.68937683105469,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": 1.351765751838684,
      "logits/rejected": 1.4346871376037598,
      "logps/chosen": -288.8231201171875,
      "logps/ref_chosen": -266.2965087890625,
      "logps/ref_rejected": -263.7326965332031,
      "logps/rejected": -322.93609619140625,
      "loss": 4.474,
      "margin_dpo/margin_mean": 36.676849365234375,
      "margin_dpo/margin_std": 58.30961990356445,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -21.57791519165039,
      "KL/mean": -41.526363372802734,
      "KL/rejected_KL_mean": -61.47480392456055,
      "KL/std": 54.093265533447266,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.012349730357527733,
      "fcm_dpo/delta": 0.006710456684231758,
      "fcm_dpo/margin": 39.89689254760742,
      "fcm_dpo/q_t": 0.3943599760532379,
      "grad_norm": 58.24032974243164,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": 1.5280330181121826,
      "logits/rejected": 1.5512609481811523,
      "logps/chosen": -313.93865966796875,
      "logps/ref_chosen": -292.36077880859375,
      "logps/ref_rejected": -264.35662841796875,
      "logps/rejected": -325.8314208984375,
      "loss": 4.4077,
      "margin_dpo/margin_mean": 39.89689254760742,
      "margin_dpo/margin_std": 64.57376861572266,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -21.134288787841797,
      "KL/mean": -38.106689453125,
      "KL/rejected_KL_mean": -55.07908248901367,
      "KL/std": 53.27201461791992,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.012474942952394485,
      "fcm_dpo/delta": -0.016402624547481537,
      "fcm_dpo/margin": 33.944793701171875,
      "fcm_dpo/q_t": 0.4077484607696533,
      "grad_norm": 59.7107048034668,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": 1.3515840768814087,
      "logits/rejected": 1.4538625478744507,
      "logps/chosen": -267.85406494140625,
      "logps/ref_chosen": -246.71978759765625,
      "logps/ref_rejected": -261.339599609375,
      "logps/rejected": -316.418701171875,
      "loss": 4.608,
      "margin_dpo/margin_mean": 33.944793701171875,
      "margin_dpo/margin_std": 58.99382781982422,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -15.758590698242188,
      "KL/mean": -37.980262756347656,
      "KL/rejected_KL_mean": -60.201942443847656,
      "KL/std": 52.636478424072266,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.012043684720993042,
      "fcm_dpo/delta": -0.04023259878158569,
      "fcm_dpo/margin": 44.44334030151367,
      "fcm_dpo/q_t": 0.38223159313201904,
      "grad_norm": 53.719364166259766,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": 1.2751635313034058,
      "logits/rejected": 1.269358515739441,
      "logps/chosen": -314.3172912597656,
      "logps/ref_chosen": -298.5587158203125,
      "logps/ref_rejected": -264.5877685546875,
      "logps/rejected": -324.78973388671875,
      "loss": 4.2327,
      "margin_dpo/margin_mean": 44.44334411621094,
      "margin_dpo/margin_std": 60.88602066040039,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -11.940485000610352,
      "KL/mean": -35.42499542236328,
      "KL/rejected_KL_mean": -58.90951156616211,
      "KL/std": 50.84739303588867,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.011471014469861984,
      "fcm_dpo/delta": -0.042882345616817474,
      "fcm_dpo/margin": 46.969024658203125,
      "fcm_dpo/q_t": 0.3790516257286072,
      "grad_norm": 38.78754425048828,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": 1.186388611793518,
      "logits/rejected": 1.1633282899856567,
      "logps/chosen": -261.7937316894531,
      "logps/ref_chosen": -249.8532257080078,
      "logps/ref_rejected": -233.2117156982422,
      "logps/rejected": -292.1212463378906,
      "loss": 4.0711,
      "margin_dpo/margin_mean": 46.969024658203125,
      "margin_dpo/margin_std": 55.475563049316406,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -17.67400360107422,
      "KL/mean": -38.21571731567383,
      "KL/rejected_KL_mean": -58.75742721557617,
      "KL/std": 52.38951110839844,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.011301069520413876,
      "fcm_dpo/delta": -0.019945984706282616,
      "fcm_dpo/margin": 41.08341598510742,
      "fcm_dpo/q_t": 0.3963419795036316,
      "grad_norm": 45.288352966308594,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": 1.3905813694000244,
      "logits/rejected": 1.5454325675964355,
      "logps/chosen": -276.4956359863281,
      "logps/ref_chosen": -258.8216247558594,
      "logps/ref_rejected": -260.4953308105469,
      "logps/rejected": -319.25274658203125,
      "loss": 4.3267,
      "margin_dpo/margin_mean": 41.08341598510742,
      "margin_dpo/margin_std": 57.25684356689453,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -14.611082077026367,
      "KL/mean": -37.7140998840332,
      "KL/rejected_KL_mean": -60.817108154296875,
      "KL/std": 51.47083282470703,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.011271992698311806,
      "fcm_dpo/delta": -0.023328743875026703,
      "fcm_dpo/margin": 46.206024169921875,
      "fcm_dpo/q_t": 0.38329949975013733,
      "grad_norm": 41.571537017822266,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": 1.5727230310440063,
      "logits/rejected": 1.4592148065567017,
      "logps/chosen": -338.82757568359375,
      "logps/ref_chosen": -324.2165222167969,
      "logps/ref_rejected": -237.05453491210938,
      "logps/rejected": -297.87164306640625,
      "loss": 4.1639,
      "margin_dpo/margin_mean": 46.206024169921875,
      "margin_dpo/margin_std": 58.54914855957031,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -25.652637481689453,
      "KL/mean": -44.3125114440918,
      "KL/rejected_KL_mean": -62.97237777709961,
      "KL/std": 53.732666015625,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.011132254265248775,
      "fcm_dpo/delta": 0.04017575457692146,
      "fcm_dpo/margin": 37.31973648071289,
      "fcm_dpo/q_t": 0.40735793113708496,
      "grad_norm": 67.85509490966797,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 1.3242690563201904,
      "logits/rejected": 1.2981479167938232,
      "logps/chosen": -304.05389404296875,
      "logps/ref_chosen": -278.4012451171875,
      "logps/ref_rejected": -247.72671508789062,
      "logps/rejected": -310.6990966796875,
      "loss": 4.4667,
      "margin_dpo/margin_mean": 37.31973648071289,
      "margin_dpo/margin_std": 58.20713806152344,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -18.569183349609375,
      "KL/mean": -39.217308044433594,
      "KL/rejected_KL_mean": -59.86541748046875,
      "KL/std": 51.515018463134766,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.011071247048676014,
      "fcm_dpo/delta": -0.04814938083291054,
      "fcm_dpo/margin": 41.296226501464844,
      "fcm_dpo/q_t": 0.3980022072792053,
      "grad_norm": 63.41021728515625,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.213998794555664,
      "logits/rejected": 1.2152736186981201,
      "logps/chosen": -310.663818359375,
      "logps/ref_chosen": -292.0946350097656,
      "logps/ref_rejected": -252.24237060546875,
      "logps/rejected": -312.1077880859375,
      "loss": 4.3624,
      "margin_dpo/margin_mean": 41.296226501464844,
      "margin_dpo/margin_std": 58.288352966308594,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -13.666519165039062,
      "KL/mean": -33.80872344970703,
      "KL/rejected_KL_mean": -53.95093536376953,
      "KL/std": 54.188568115234375,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.010937836021184921,
      "fcm_dpo/delta": 0.020360952243208885,
      "fcm_dpo/margin": 40.284423828125,
      "fcm_dpo/q_t": 0.40231817960739136,
      "grad_norm": 39.686431884765625,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.3314428329467773,
      "logits/rejected": 1.387155294418335,
      "logps/chosen": -304.2773132324219,
      "logps/ref_chosen": -290.61077880859375,
      "logps/ref_rejected": -279.8963928222656,
      "logps/rejected": -333.8472900390625,
      "loss": 4.3779,
      "margin_dpo/margin_mean": 40.284423828125,
      "margin_dpo/margin_std": 57.91526412963867,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -19.658935546875,
      "KL/mean": -41.546363830566406,
      "KL/rejected_KL_mean": -63.43378829956055,
      "KL/std": 52.991249084472656,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.01118521112948656,
      "fcm_dpo/delta": 0.010740034282207489,
      "fcm_dpo/margin": 43.774845123291016,
      "fcm_dpo/q_t": 0.3907336890697479,
      "grad_norm": 37.65342712402344,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": 1.6133204698562622,
      "logits/rejected": 1.588179588317871,
      "logps/chosen": -335.97418212890625,
      "logps/ref_chosen": -316.3152770996094,
      "logps/ref_rejected": -300.5086975097656,
      "logps/rejected": -363.9424743652344,
      "loss": 4.236,
      "margin_dpo/margin_mean": 43.77484130859375,
      "margin_dpo/margin_std": 59.10090637207031,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -19.598690032958984,
      "KL/mean": -39.82948303222656,
      "KL/rejected_KL_mean": -60.060279846191406,
      "KL/std": 55.10944366455078,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.011482498608529568,
      "fcm_dpo/delta": 0.03580213710665703,
      "fcm_dpo/margin": 40.46157455444336,
      "fcm_dpo/q_t": 0.39750590920448303,
      "grad_norm": 38.34062957763672,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": 1.4245460033416748,
      "logits/rejected": 1.5184351205825806,
      "logps/chosen": -310.14398193359375,
      "logps/ref_chosen": -290.5452575683594,
      "logps/ref_rejected": -297.2102355957031,
      "logps/rejected": -357.2705078125,
      "loss": 4.349,
      "margin_dpo/margin_mean": 40.46157455444336,
      "margin_dpo/margin_std": 57.993045806884766,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -15.860078811645508,
      "KL/mean": -32.72433853149414,
      "KL/rejected_KL_mean": -49.588592529296875,
      "KL/std": 50.43977355957031,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.011861681938171387,
      "fcm_dpo/delta": 0.024948136880993843,
      "fcm_dpo/margin": 33.72850799560547,
      "fcm_dpo/q_t": 0.41129374504089355,
      "grad_norm": 42.147037506103516,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": 1.2730717658996582,
      "logits/rejected": 1.3691107034683228,
      "logps/chosen": -279.2804260253906,
      "logps/ref_chosen": -263.4203186035156,
      "logps/ref_rejected": -270.30291748046875,
      "logps/rejected": -319.8915100097656,
      "loss": 4.5726,
      "margin_dpo/margin_mean": 33.728511810302734,
      "margin_dpo/margin_std": 58.830020904541016,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -22.357563018798828,
      "KL/mean": -46.84614562988281,
      "KL/rejected_KL_mean": -71.33472442626953,
      "KL/std": 56.770896911621094,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.011363739147782326,
      "fcm_dpo/delta": -0.10031997412443161,
      "fcm_dpo/margin": 48.97715759277344,
      "fcm_dpo/q_t": 0.37939029932022095,
      "grad_norm": 32.342437744140625,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": 1.1650052070617676,
      "logits/rejected": 1.3061798810958862,
      "logps/chosen": -334.39495849609375,
      "logps/ref_chosen": -312.0373840332031,
      "logps/ref_rejected": -290.50946044921875,
      "logps/rejected": -361.84417724609375,
      "loss": 4.1614,
      "margin_dpo/margin_mean": 48.9771614074707,
      "margin_dpo/margin_std": 66.70311737060547,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -21.66713523864746,
      "KL/mean": -44.75026321411133,
      "KL/rejected_KL_mean": -67.83336639404297,
      "KL/std": 55.922447204589844,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.010849181562662125,
      "fcm_dpo/delta": -0.0013673584908246994,
      "fcm_dpo/margin": 46.16622543334961,
      "fcm_dpo/q_t": 0.3887263536453247,
      "grad_norm": 33.99593734741211,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": 1.427199363708496,
      "logits/rejected": 1.462519645690918,
      "logps/chosen": -318.839111328125,
      "logps/ref_chosen": -297.1719970703125,
      "logps/ref_rejected": -269.15557861328125,
      "logps/rejected": -336.98895263671875,
      "loss": 4.2556,
      "margin_dpo/margin_mean": 46.166229248046875,
      "margin_dpo/margin_std": 64.95339965820312,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -19.286815643310547,
      "KL/mean": -39.776573181152344,
      "KL/rejected_KL_mean": -60.26633071899414,
      "KL/std": 56.26765441894531,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.011127572506666183,
      "fcm_dpo/delta": 0.0033990107476711273,
      "fcm_dpo/margin": 40.979515075683594,
      "fcm_dpo/q_t": 0.3997444808483124,
      "grad_norm": 43.38941955566406,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": 1.4341590404510498,
      "logits/rejected": 1.360427737236023,
      "logps/chosen": -306.6517028808594,
      "logps/ref_chosen": -287.3648986816406,
      "logps/ref_rejected": -271.1472473144531,
      "logps/rejected": -331.41357421875,
      "loss": 4.3853,
      "margin_dpo/margin_mean": 40.979515075683594,
      "margin_dpo/margin_std": 60.304527282714844,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -22.855243682861328,
      "KL/mean": -38.68614196777344,
      "KL/rejected_KL_mean": -54.51704406738281,
      "KL/std": 52.561302185058594,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.011173278093338013,
      "fcm_dpo/delta": 0.029829924926161766,
      "fcm_dpo/margin": 31.661808013916016,
      "fcm_dpo/q_t": 0.42216265201568604,
      "grad_norm": 39.58899688720703,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": 1.318557620048523,
      "logits/rejected": 1.398245096206665,
      "logps/chosen": -311.46136474609375,
      "logps/ref_chosen": -288.6061096191406,
      "logps/ref_rejected": -285.6366271972656,
      "logps/rejected": -340.1536560058594,
      "loss": 4.7596,
      "margin_dpo/margin_mean": 31.66180419921875,
      "margin_dpo/margin_std": 63.170989990234375,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -22.551332473754883,
      "KL/mean": -38.81722640991211,
      "KL/rejected_KL_mean": -55.08310317993164,
      "KL/std": 52.450721740722656,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.011595677584409714,
      "fcm_dpo/delta": 0.06744906306266785,
      "fcm_dpo/margin": 32.531761169433594,
      "fcm_dpo/q_t": 0.418176531791687,
      "grad_norm": 67.2270736694336,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 1.292378306388855,
      "logits/rejected": 1.2360552549362183,
      "logps/chosen": -332.8032531738281,
      "logps/ref_chosen": -310.251953125,
      "logps/ref_rejected": -252.962158203125,
      "logps/rejected": -308.0452880859375,
      "loss": 4.7496,
      "margin_dpo/margin_mean": 32.53175735473633,
      "margin_dpo/margin_std": 65.01676177978516,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -20.329931259155273,
      "KL/mean": -40.108558654785156,
      "KL/rejected_KL_mean": -59.887184143066406,
      "KL/std": 54.44658279418945,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.011919771321117878,
      "fcm_dpo/delta": -0.008030976168811321,
      "fcm_dpo/margin": 39.55727005004883,
      "fcm_dpo/q_t": 0.3972996473312378,
      "grad_norm": 41.513484954833984,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 1.3930575847625732,
      "logits/rejected": 1.5206800699234009,
      "logps/chosen": -257.1103820800781,
      "logps/ref_chosen": -236.7804412841797,
      "logps/ref_rejected": -229.67550659179688,
      "logps/rejected": -289.56268310546875,
      "loss": 4.4561,
      "margin_dpo/margin_mean": 39.55727005004883,
      "margin_dpo/margin_std": 65.18107604980469,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -15.044403076171875,
      "KL/mean": -40.33625030517578,
      "KL/rejected_KL_mean": -65.62809753417969,
      "KL/std": 54.0989990234375,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.011255396530032158,
      "fcm_dpo/delta": -0.07502968609333038,
      "fcm_dpo/margin": 50.583683013916016,
      "fcm_dpo/q_t": 0.37433767318725586,
      "grad_norm": 42.016571044921875,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": 1.270332932472229,
      "logits/rejected": 1.1447429656982422,
      "logps/chosen": -350.34796142578125,
      "logps/ref_chosen": -335.30352783203125,
      "logps/ref_rejected": -250.6796875,
      "logps/rejected": -316.3077697753906,
      "loss": 4.1125,
      "margin_dpo/margin_mean": 50.583683013916016,
      "margin_dpo/margin_std": 66.76519775390625,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -21.654386520385742,
      "KL/mean": -40.395015716552734,
      "KL/rejected_KL_mean": -59.135643005371094,
      "KL/std": 54.098594665527344,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.011262796819210052,
      "fcm_dpo/delta": 0.042652830481529236,
      "fcm_dpo/margin": 37.481258392333984,
      "fcm_dpo/q_t": 0.4063204526901245,
      "grad_norm": 48.795692443847656,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": 1.5116289854049683,
      "logits/rejected": 1.3512427806854248,
      "logps/chosen": -309.8746337890625,
      "logps/ref_chosen": -288.22021484375,
      "logps/ref_rejected": -227.96673583984375,
      "logps/rejected": -287.1023864746094,
      "loss": 4.4728,
      "margin_dpo/margin_mean": 37.481258392333984,
      "margin_dpo/margin_std": 59.151817321777344,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -22.914323806762695,
      "KL/mean": -42.497108459472656,
      "KL/rejected_KL_mean": -62.07989501953125,
      "KL/std": 53.04613494873047,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.011490004137158394,
      "fcm_dpo/delta": 0.009602184407413006,
      "fcm_dpo/margin": 39.16556930541992,
      "fcm_dpo/q_t": 0.3984319269657135,
      "grad_norm": 58.42338180541992,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": 1.4579249620437622,
      "logits/rejected": 1.4834882020950317,
      "logps/chosen": -283.2512512207031,
      "logps/ref_chosen": -260.3369140625,
      "logps/ref_rejected": -254.2083740234375,
      "logps/rejected": -316.2882995605469,
      "loss": 4.4172,
      "margin_dpo/margin_mean": 39.165565490722656,
      "margin_dpo/margin_std": 59.3841667175293,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -10.917932510375977,
      "KL/mean": -35.5947151184082,
      "KL/rejected_KL_mean": -60.27149963378906,
      "KL/std": 53.68513870239258,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.010847090743482113,
      "fcm_dpo/delta": -0.10990501940250397,
      "fcm_dpo/margin": 49.35356903076172,
      "fcm_dpo/q_t": 0.3810797333717346,
      "grad_norm": 33.13408279418945,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.3459408283233643,
      "logits/rejected": 1.3777296543121338,
      "logps/chosen": -293.9873962402344,
      "logps/ref_chosen": -283.0694885253906,
      "logps/ref_rejected": -289.1370544433594,
      "logps/rejected": -349.4085693359375,
      "loss": 4.1538,
      "margin_dpo/margin_mean": 49.35356903076172,
      "margin_dpo/margin_std": 59.874977111816406,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -12.459888458251953,
      "KL/mean": -38.64053726196289,
      "KL/rejected_KL_mean": -64.82118225097656,
      "KL/std": 53.432518005371094,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.010270591825246811,
      "fcm_dpo/delta": -0.04115338623523712,
      "fcm_dpo/margin": 52.361289978027344,
      "fcm_dpo/q_t": 0.37732255458831787,
      "grad_norm": 31.32708740234375,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": 1.0453734397888184,
      "logits/rejected": 1.0986213684082031,
      "logps/chosen": -269.34490966796875,
      "logps/ref_chosen": -256.8850402832031,
      "logps/ref_rejected": -247.0802459716797,
      "logps/rejected": -311.90142822265625,
      "loss": 4.0019,
      "margin_dpo/margin_mean": 52.361289978027344,
      "margin_dpo/margin_std": 56.54520797729492,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -21.785154342651367,
      "KL/mean": -40.68267822265625,
      "KL/rejected_KL_mean": -59.5802001953125,
      "KL/std": 53.93944549560547,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.010375358164310455,
      "fcm_dpo/delta": 0.011322807520627975,
      "fcm_dpo/margin": 37.7950439453125,
      "fcm_dpo/q_t": 0.41288718581199646,
      "grad_norm": 39.69447708129883,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": 0.9834239482879639,
      "logits/rejected": 1.1340173482894897,
      "logps/chosen": -279.3092346191406,
      "logps/ref_chosen": -257.5240783691406,
      "logps/ref_rejected": -274.82122802734375,
      "logps/rejected": -334.40142822265625,
      "loss": 4.5912,
      "margin_dpo/margin_mean": 37.7950439453125,
      "margin_dpo/margin_std": 64.04612731933594,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -15.68712043762207,
      "KL/mean": -33.22931671142578,
      "KL/rejected_KL_mean": -50.771514892578125,
      "KL/std": 52.264739990234375,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.010817142203450203,
      "fcm_dpo/delta": 0.08949258178472519,
      "fcm_dpo/margin": 35.08439254760742,
      "fcm_dpo/q_t": 0.41596096754074097,
      "grad_norm": 35.705265045166016,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": 1.4097644090652466,
      "logits/rejected": 1.295135498046875,
      "logps/chosen": -327.94683837890625,
      "logps/ref_chosen": -312.2597351074219,
      "logps/ref_rejected": -259.74383544921875,
      "logps/rejected": -310.5153503417969,
      "loss": 4.5643,
      "margin_dpo/margin_mean": 35.084388732910156,
      "margin_dpo/margin_std": 56.77195739746094,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -12.383644104003906,
      "KL/mean": -35.460365295410156,
      "KL/rejected_KL_mean": -58.53707504272461,
      "KL/std": 54.93742752075195,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.011167640797793865,
      "fcm_dpo/delta": -0.017221132293343544,
      "fcm_dpo/margin": 46.15343475341797,
      "fcm_dpo/q_t": 0.3854927122592926,
      "grad_norm": 40.52635955810547,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": 1.244476079940796,
      "logits/rejected": 1.1492905616760254,
      "logps/chosen": -329.4233703613281,
      "logps/ref_chosen": -317.03973388671875,
      "logps/ref_rejected": -260.12628173828125,
      "logps/rejected": -318.663330078125,
      "loss": 4.258,
      "margin_dpo/margin_mean": 46.15343475341797,
      "margin_dpo/margin_std": 65.02312469482422,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -12.289859771728516,
      "KL/mean": -36.42634201049805,
      "KL/rejected_KL_mean": -60.562828063964844,
      "KL/std": 55.09480667114258,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.011099135503172874,
      "fcm_dpo/delta": -0.03960346058011055,
      "fcm_dpo/margin": 48.27297592163086,
      "fcm_dpo/q_t": 0.38202738761901855,
      "grad_norm": 32.672760009765625,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 1.307143211364746,
      "logits/rejected": 1.2247164249420166,
      "logps/chosen": -316.6838073730469,
      "logps/ref_chosen": -304.3939514160156,
      "logps/ref_rejected": -257.6587219238281,
      "logps/rejected": -318.2215270996094,
      "loss": 4.2072,
      "margin_dpo/margin_mean": 48.27297592163086,
      "margin_dpo/margin_std": 63.76195526123047,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -21.182043075561523,
      "KL/mean": -38.641788482666016,
      "KL/rejected_KL_mean": -56.10152816772461,
      "KL/std": 54.55463790893555,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.010983363725244999,
      "fcm_dpo/delta": 0.0785600021481514,
      "fcm_dpo/margin": 34.919490814208984,
      "fcm_dpo/q_t": 0.41502946615219116,
      "grad_norm": 57.315494537353516,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 1.2055453062057495,
      "logits/rejected": 1.2767993211746216,
      "logps/chosen": -306.1258544921875,
      "logps/ref_chosen": -284.94384765625,
      "logps/ref_rejected": -297.4357604980469,
      "logps/rejected": -353.53729248046875,
      "loss": 4.5609,
      "margin_dpo/margin_mean": 34.919490814208984,
      "margin_dpo/margin_std": 58.411277770996094,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -14.195104598999023,
      "KL/mean": -37.33507537841797,
      "KL/rejected_KL_mean": -60.47505569458008,
      "KL/std": 53.0501594543457,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.01106267236173153,
      "fcm_dpo/delta": -0.01359591819345951,
      "fcm_dpo/margin": 46.27994155883789,
      "fcm_dpo/q_t": 0.38776248693466187,
      "grad_norm": 43.73598861694336,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": 1.2788945436477661,
      "logits/rejected": 1.405962347984314,
      "logps/chosen": -246.5493927001953,
      "logps/ref_chosen": -232.35427856445312,
      "logps/ref_rejected": -245.6057891845703,
      "logps/rejected": -306.080810546875,
      "loss": 4.2191,
      "margin_dpo/margin_mean": 46.279937744140625,
      "margin_dpo/margin_std": 62.695926666259766,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -17.540185928344727,
      "KL/mean": -41.26536560058594,
      "KL/rejected_KL_mean": -64.99055480957031,
      "KL/std": 53.227718353271484,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.011172245256602764,
      "fcm_dpo/delta": -0.03223955258727074,
      "fcm_dpo/margin": 47.45037841796875,
      "fcm_dpo/q_t": 0.3825909495353699,
      "grad_norm": 36.212867736816406,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": 1.1677628755569458,
      "logits/rejected": 1.198642611503601,
      "logps/chosen": -295.5470275878906,
      "logps/ref_chosen": -278.0068664550781,
      "logps/ref_rejected": -257.19256591796875,
      "logps/rejected": -322.1831359863281,
      "loss": 4.1727,
      "margin_dpo/margin_mean": 47.45037841796875,
      "margin_dpo/margin_std": 62.01404571533203,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -24.053245544433594,
      "KL/mean": -42.171443939208984,
      "KL/rejected_KL_mean": -60.289634704589844,
      "KL/std": 52.169551849365234,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.01092322263866663,
      "fcm_dpo/delta": 0.07035918533802032,
      "fcm_dpo/margin": 36.23638916015625,
      "fcm_dpo/q_t": 0.4107322096824646,
      "grad_norm": 67.07123565673828,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": 1.1060199737548828,
      "logits/rejected": 1.1366751194000244,
      "logps/chosen": -304.81072998046875,
      "logps/ref_chosen": -280.75750732421875,
      "logps/ref_rejected": -264.34295654296875,
      "logps/rejected": -324.6325988769531,
      "loss": 4.4956,
      "margin_dpo/margin_mean": 36.23638916015625,
      "margin_dpo/margin_std": 52.92101287841797,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -16.027292251586914,
      "KL/mean": -34.71216583251953,
      "KL/rejected_KL_mean": -53.39704132080078,
      "KL/std": 50.68662643432617,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.01161264069378376,
      "fcm_dpo/delta": 0.00026100571267306805,
      "fcm_dpo/margin": 37.36973571777344,
      "fcm_dpo/q_t": 0.40460336208343506,
      "grad_norm": 40.48722457885742,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": 1.0816503763198853,
      "logits/rejected": 1.1850391626358032,
      "logps/chosen": -257.4986572265625,
      "logps/ref_chosen": -241.4713592529297,
      "logps/ref_rejected": -244.0782928466797,
      "logps/rejected": -297.4753112792969,
      "loss": 4.4727,
      "margin_dpo/margin_mean": 37.36973571777344,
      "margin_dpo/margin_std": 59.11124801635742,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -9.021137237548828,
      "KL/mean": -30.87635040283203,
      "KL/rejected_KL_mean": -52.73155975341797,
      "KL/std": 51.64687728881836,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.011579921469092369,
      "fcm_dpo/delta": -0.008368845097720623,
      "fcm_dpo/margin": 43.710411071777344,
      "fcm_dpo/q_t": 0.38903486728668213,
      "grad_norm": 31.685440063476562,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": 1.3241478204727173,
      "logits/rejected": 1.3139681816101074,
      "logps/chosen": -327.79547119140625,
      "logps/ref_chosen": -318.7742919921875,
      "logps/ref_rejected": -293.66595458984375,
      "logps/rejected": -346.3974914550781,
      "loss": 4.2629,
      "margin_dpo/margin_mean": 43.710411071777344,
      "margin_dpo/margin_std": 60.618011474609375,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -18.230266571044922,
      "KL/mean": -41.534767150878906,
      "KL/rejected_KL_mean": -64.83927154541016,
      "KL/std": 55.01382827758789,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.011523835361003876,
      "fcm_dpo/delta": -0.03934657201170921,
      "fcm_dpo/margin": 46.609004974365234,
      "fcm_dpo/q_t": 0.3828693926334381,
      "grad_norm": 39.070091247558594,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": 1.1707298755645752,
      "logits/rejected": 1.188868761062622,
      "logps/chosen": -284.16595458984375,
      "logps/ref_chosen": -265.9356994628906,
      "logps/ref_rejected": -243.6986083984375,
      "logps/rejected": -308.537841796875,
      "loss": 4.243,
      "margin_dpo/margin_mean": 46.609004974365234,
      "margin_dpo/margin_std": 65.9621810913086,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 4.685751873492195,
      "train_runtime": 9606.4037,
      "train_samples_per_second": 6.364,
      "train_steps_per_second": 0.05
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}