{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.04536247253417969,
      "KL/mean": -0.05389120429754257,
      "KL/rejected_KL_mean": -0.06241607666015625,
      "KL/std": 0.5513200759887695,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01704825460910797,
      "fcm_dpo/q_t": 0.499957412481308,
      "grad_norm": 14.619175910949707,
      "learning_rate": 0.0,
      "logits/chosen": 1.9854813814163208,
      "logits/rejected": 1.7737188339233398,
      "logps/chosen": -267.0936584472656,
      "logps/ref_chosen": -267.04827880859375,
      "logps/ref_rejected": -219.76657104492188,
      "logps/rejected": -219.82899475097656,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.01704716682434082,
      "margin_dpo/margin_std": 0.7467707991600037,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.07550811767578125,
      "KL/mean": 0.03060000389814377,
      "KL/rejected_KL_mean": -0.014301300048828125,
      "KL/std": 0.6403332352638245,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08979703485965729,
      "fcm_dpo/q_t": 0.4997755289077759,
      "grad_norm": 15.140707969665527,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 2.1993117332458496,
      "logits/rejected": 2.1368322372436523,
      "logps/chosen": -256.5101623535156,
      "logps/ref_chosen": -256.585693359375,
      "logps/ref_rejected": -239.92742919921875,
      "logps/rejected": -239.94171142578125,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08979731798171997,
      "margin_dpo/margin_std": 0.9745746850967407,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": -0.028543472290039062,
      "KL/mean": -0.06190856546163559,
      "KL/rejected_KL_mean": -0.09526252746582031,
      "KL/std": 0.627353847026825,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06673042476177216,
      "fcm_dpo/q_t": 0.4998331665992737,
      "grad_norm": 14.62266731262207,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": 2.256986618041992,
      "logits/rejected": 2.2961509227752686,
      "logps/chosen": -265.6974182128906,
      "logps/ref_chosen": -265.66888427734375,
      "logps/ref_rejected": -284.3778991699219,
      "logps/rejected": -284.4731750488281,
      "loss": 5.5426,
      "margin_dpo/margin_mean": 0.0667317658662796,
      "margin_dpo/margin_std": 0.7665271162986755,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.04369354248046875,
      "KL/mean": 0.022621460258960724,
      "KL/rejected_KL_mean": 0.0015583038330078125,
      "KL/std": 0.669308602809906,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04213587939739227,
      "fcm_dpo/q_t": 0.4998946785926819,
      "grad_norm": 15.703679084777832,
      "learning_rate": 3.125e-08,
      "logits/chosen": 1.870570182800293,
      "logits/rejected": 1.8099285364151,
      "logps/chosen": -319.4637756347656,
      "logps/ref_chosen": -319.50750732421875,
      "logps/ref_rejected": -296.22314453125,
      "logps/rejected": -296.2215881347656,
      "loss": 5.5436,
      "margin_dpo/margin_mean": 0.042136386036872864,
      "margin_dpo/margin_std": 0.9194411039352417,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.030849456787109375,
      "KL/mean": 0.025934934616088867,
      "KL/rejected_KL_mean": 0.021022796630859375,
      "KL/std": 0.6444316506385803,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.009832918643951416,
      "fcm_dpo/q_t": 0.49997544288635254,
      "grad_norm": 15.600411415100098,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": 1.9464259147644043,
      "logits/rejected": 1.9626566171646118,
      "logps/chosen": -292.8421325683594,
      "logps/ref_chosen": -292.87298583984375,
      "logps/ref_rejected": -270.3802795410156,
      "logps/rejected": -270.3592834472656,
      "loss": 5.5449,
      "margin_dpo/margin_mean": 0.00983293354511261,
      "margin_dpo/margin_std": 0.8992936015129089,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": 0.02524566650390625,
      "KL/mean": 0.03444797545671463,
      "KL/rejected_KL_mean": 0.04365730285644531,
      "KL/std": 0.61781245470047,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.01840551197528839,
      "fcm_dpo/q_t": 0.5000460147857666,
      "grad_norm": 15.542896270751953,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 2.103062629699707,
      "logits/rejected": 2.0377492904663086,
      "logps/chosen": -280.26019287109375,
      "logps/ref_chosen": -280.28546142578125,
      "logps/ref_rejected": -242.7611541748047,
      "logps/rejected": -242.71749877929688,
      "loss": 5.546,
      "margin_dpo/margin_mean": -0.018405884504318237,
      "margin_dpo/margin_std": 0.9298524260520935,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.02559661865234375,
      "KL/mean": 0.01772065833210945,
      "KL/rejected_KL_mean": 0.009845733642578125,
      "KL/std": 0.6574221849441528,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.015746213495731354,
      "fcm_dpo/q_t": 0.49996063113212585,
      "grad_norm": 15.60773754119873,
      "learning_rate": 6.25e-08,
      "logits/chosen": 1.9930392503738403,
      "logits/rejected": 1.904411792755127,
      "logps/chosen": -255.08010864257812,
      "logps/ref_chosen": -255.10572814941406,
      "logps/ref_rejected": -251.15402221679688,
      "logps/rejected": -251.14419555664062,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.015746206045150757,
      "margin_dpo/margin_std": 0.9424126744270325,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.07427597045898438,
      "KL/mean": -0.022388603538274765,
      "KL/rejected_KL_mean": 0.029504776000976562,
      "KL/std": 0.7070745229721069,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.10379504412412643,
      "fcm_dpo/q_t": 0.500259518623352,
      "grad_norm": 16.627031326293945,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": 2.2578001022338867,
      "logits/rejected": 2.0581820011138916,
      "logps/chosen": -345.25579833984375,
      "logps/ref_chosen": -345.18157958984375,
      "logps/ref_rejected": -277.3522644042969,
      "logps/rejected": -277.32281494140625,
      "loss": 5.5494,
      "margin_dpo/margin_mean": -0.10379450023174286,
      "margin_dpo/margin_std": 1.0259273052215576,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.007823944091796875,
      "KL/mean": -0.016552776098251343,
      "KL/rejected_KL_mean": -0.025281906127929688,
      "KL/std": 0.6182496547698975,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.017459988594055176,
      "fcm_dpo/q_t": 0.49995630979537964,
      "grad_norm": 14.097858428955078,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": 2.004061698913574,
      "logits/rejected": 1.975795865058899,
      "logps/chosen": -276.7381591796875,
      "logps/ref_chosen": -276.7303161621094,
      "logps/ref_rejected": -266.19952392578125,
      "logps/rejected": -266.22479248046875,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.017461583018302917,
      "margin_dpo/margin_std": 0.8485158681869507,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": -0.04392051696777344,
      "KL/mean": -0.05381091684103012,
      "KL/rejected_KL_mean": -0.06370735168457031,
      "KL/std": 0.5318792462348938,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.019780203700065613,
      "fcm_dpo/q_t": 0.4999505281448364,
      "grad_norm": 13.757820129394531,
      "learning_rate": 9.375e-08,
      "logits/chosen": 1.9255223274230957,
      "logits/rejected": 1.9753773212432861,
      "logps/chosen": -246.60617065429688,
      "logps/ref_chosen": -246.562255859375,
      "logps/ref_rejected": -224.70401000976562,
      "logps/rejected": -224.76771545410156,
      "loss": 5.5444,
      "margin_dpo/margin_mean": 0.019779890775680542,
      "margin_dpo/margin_std": 0.736269474029541,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": -0.05460929870605469,
      "KL/mean": -0.06382205337285995,
      "KL/rejected_KL_mean": -0.07303619384765625,
      "KL/std": 0.6143579483032227,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01842571794986725,
      "fcm_dpo/q_t": 0.4999539256095886,
      "grad_norm": 15.434593200683594,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 1.9822487831115723,
      "logits/rejected": 1.915013313293457,
      "logps/chosen": -290.5556945800781,
      "logps/ref_chosen": -290.5010986328125,
      "logps/ref_rejected": -253.9559326171875,
      "logps/rejected": -254.0289306640625,
      "loss": 5.5445,
      "margin_dpo/margin_mean": 0.01842629909515381,
      "margin_dpo/margin_std": 0.925983190536499,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": -0.014974594116210938,
      "KL/mean": -0.001288093626499176,
      "KL/rejected_KL_mean": 0.012399673461914062,
      "KL/std": 0.6522122621536255,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.027376040816307068,
      "fcm_dpo/q_t": 0.5000684261322021,
      "grad_norm": 14.893819808959961,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": 1.8091472387313843,
      "logits/rejected": 1.7994632720947266,
      "logps/chosen": -278.53350830078125,
      "logps/ref_chosen": -278.5185241699219,
      "logps/ref_rejected": -273.56207275390625,
      "logps/rejected": -273.5496826171875,
      "loss": 5.5464,
      "margin_dpo/margin_mean": -0.027375996112823486,
      "margin_dpo/margin_std": 0.9168465733528137,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.042331695556640625,
      "KL/mean": -0.04921622574329376,
      "KL/rejected_KL_mean": -0.05609703063964844,
      "KL/std": 0.6046156883239746,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01376757025718689,
      "fcm_dpo/q_t": 0.4999656081199646,
      "grad_norm": 14.812957763671875,
      "learning_rate": 1.25e-07,
      "logits/chosen": 1.7188992500305176,
      "logits/rejected": 1.8403123617172241,
      "logps/chosen": -264.699462890625,
      "logps/ref_chosen": -264.6571350097656,
      "logps/ref_rejected": -274.3492736816406,
      "logps/rejected": -274.4053649902344,
      "loss": 5.5447,
      "margin_dpo/margin_mean": 0.01376672089099884,
      "margin_dpo/margin_std": 0.8706319332122803,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.10950469970703125,
      "KL/mean": 0.04948730021715164,
      "KL/rejected_KL_mean": -0.010534286499023438,
      "KL/std": 0.6066107153892517,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12004035711288452,
      "fcm_dpo/q_t": 0.4996998906135559,
      "grad_norm": 15.543604850769043,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": 1.8829765319824219,
      "logits/rejected": 1.8982784748077393,
      "logps/chosen": -274.00506591796875,
      "logps/ref_chosen": -274.1145935058594,
      "logps/ref_rejected": -261.0951843261719,
      "logps/rejected": -261.105712890625,
      "loss": 5.5404,
      "margin_dpo/margin_mean": 0.12003912031650543,
      "margin_dpo/margin_std": 0.8114286661148071,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.05826568603515625,
      "KL/mean": -0.002657368779182434,
      "KL/rejected_KL_mean": 0.05295372009277344,
      "KL/std": 0.6889737248420715,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.11122214794158936,
      "fcm_dpo/q_t": 0.500278115272522,
      "grad_norm": 16.11313819885254,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": 2.0903069972991943,
      "logits/rejected": 1.9633231163024902,
      "logps/chosen": -288.6429748535156,
      "logps/ref_chosen": -288.584716796875,
      "logps/ref_rejected": -224.8094482421875,
      "logps/rejected": -224.75650024414062,
      "loss": 5.5497,
      "margin_dpo/margin_mean": -0.11122331023216248,
      "margin_dpo/margin_std": 0.9753756523132324,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": 0.09474754333496094,
      "KL/mean": 0.06570613384246826,
      "KL/rejected_KL_mean": 0.03666496276855469,
      "KL/std": 0.6686137318611145,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.058077722787857056,
      "fcm_dpo/q_t": 0.49985480308532715,
      "grad_norm": 16.589914321899414,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": 1.8110020160675049,
      "logits/rejected": 1.653507947921753,
      "logps/chosen": -283.7483825683594,
      "logps/ref_chosen": -283.8431091308594,
      "logps/ref_rejected": -248.63645935058594,
      "logps/rejected": -248.59982299804688,
      "loss": 5.5429,
      "margin_dpo/margin_mean": 0.05807873606681824,
      "margin_dpo/margin_std": 0.9010732173919678,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.06300163269042969,
      "KL/mean": 0.04177965968847275,
      "KL/rejected_KL_mean": 0.020551681518554688,
      "KL/std": 0.6154952049255371,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.042433008551597595,
      "fcm_dpo/q_t": 0.4998939037322998,
      "grad_norm": 15.178192138671875,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 1.9204367399215698,
      "logits/rejected": 1.8795719146728516,
      "logps/chosen": -254.57058715820312,
      "logps/ref_chosen": -254.6335906982422,
      "logps/ref_rejected": -245.3981170654297,
      "logps/rejected": -245.3775634765625,
      "loss": 5.5436,
      "margin_dpo/margin_mean": 0.04243408143520355,
      "margin_dpo/margin_std": 0.7990424633026123,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": 0.003818511962890625,
      "KL/mean": 0.009343221783638,
      "KL/rejected_KL_mean": 0.01486968994140625,
      "KL/std": 0.6735252141952515,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.011049985885620117,
      "fcm_dpo/q_t": 0.500027596950531,
      "grad_norm": 15.337018013000488,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": 1.8921184539794922,
      "logits/rejected": 1.8067348003387451,
      "logps/chosen": -287.7457275390625,
      "logps/ref_chosen": -287.7495422363281,
      "logps/ref_rejected": -273.03363037109375,
      "logps/rejected": -273.0187683105469,
      "loss": 5.5457,
      "margin_dpo/margin_mean": -0.01105034351348877,
      "margin_dpo/margin_std": 0.84308922290802,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.06008148193359375,
      "KL/mean": -0.034611497074365616,
      "KL/rejected_KL_mean": -0.0091400146484375,
      "KL/std": 0.5713145136833191,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.050935156643390656,
      "fcm_dpo/q_t": 0.5001273155212402,
      "grad_norm": 14.91540813446045,
      "learning_rate": 1.875e-07,
      "logits/chosen": 1.992903709411621,
      "logits/rejected": 1.9524155855178833,
      "logps/chosen": -273.4639587402344,
      "logps/ref_chosen": -273.4039001464844,
      "logps/ref_rejected": -260.09722900390625,
      "logps/rejected": -260.1063537597656,
      "loss": 5.5473,
      "margin_dpo/margin_mean": -0.05093524605035782,
      "margin_dpo/margin_std": 0.7694490551948547,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.09533882141113281,
      "KL/mean": 0.07324784249067307,
      "KL/rejected_KL_mean": 0.05116844177246094,
      "KL/std": 0.6706034541130066,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04417811334133148,
      "fcm_dpo/q_t": 0.4998895525932312,
      "grad_norm": 15.461657524108887,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.9877378940582275,
      "logits/rejected": 1.9586396217346191,
      "logps/chosen": -292.86676025390625,
      "logps/ref_chosen": -292.96209716796875,
      "logps/ref_rejected": -256.602783203125,
      "logps/rejected": -256.5516052246094,
      "loss": 5.5435,
      "margin_dpo/margin_mean": 0.044177815318107605,
      "margin_dpo/margin_std": 0.904564619064331,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": -0.001438140869140625,
      "KL/mean": 0.007837735116481781,
      "KL/rejected_KL_mean": 0.017103195190429688,
      "KL/std": 0.5882200598716736,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.018529102206230164,
      "fcm_dpo/q_t": 0.5000462532043457,
      "grad_norm": 15.381745338439941,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 2.0734658241271973,
      "logits/rejected": 2.004581928253174,
      "logps/chosen": -292.32861328125,
      "logps/ref_chosen": -292.3271789550781,
      "logps/ref_rejected": -274.20635986328125,
      "logps/rejected": -274.18927001953125,
      "loss": 5.546,
      "margin_dpo/margin_mean": -0.018529891967773438,
      "margin_dpo/margin_std": 0.8299182653427124,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.09268569946289062,
      "KL/mean": 0.030536364763975143,
      "KL/rejected_KL_mean": -0.03161430358886719,
      "KL/std": 0.5633171796798706,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12431289255619049,
      "fcm_dpo/q_t": 0.4996892511844635,
      "grad_norm": 13.744830131530762,
      "learning_rate": 2.1875e-07,
      "logits/chosen": 1.8820290565490723,
      "logits/rejected": 1.8832093477249146,
      "logps/chosen": -253.1735382080078,
      "logps/ref_chosen": -253.2662353515625,
      "logps/ref_rejected": -241.4681396484375,
      "logps/rejected": -241.49977111816406,
      "loss": 5.5403,
      "margin_dpo/margin_mean": 0.12431301176548004,
      "margin_dpo/margin_std": 0.7342843413352966,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.0868682861328125,
      "KL/mean": 0.06522892415523529,
      "KL/rejected_KL_mean": 0.043582916259765625,
      "KL/std": 0.6085357069969177,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04329556226730347,
      "fcm_dpo/q_t": 0.4998917579650879,
      "grad_norm": 15.315319061279297,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": 2.149847984313965,
      "logits/rejected": 2.1449761390686035,
      "logps/chosen": -275.334716796875,
      "logps/ref_chosen": -275.4216003417969,
      "logps/ref_rejected": -264.44024658203125,
      "logps/rejected": -264.39666748046875,
      "loss": 5.5435,
      "margin_dpo/margin_mean": 0.04329611361026764,
      "margin_dpo/margin_std": 0.8837510943412781,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": 0.04944419860839844,
      "KL/mean": 0.04837939888238907,
      "KL/rejected_KL_mean": 0.047313690185546875,
      "KL/std": 0.6477831602096558,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.002133980393409729,
      "fcm_dpo/q_t": 0.4999946653842926,
      "grad_norm": 16.428956985473633,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": 1.9889068603515625,
      "logits/rejected": 2.014568328857422,
      "logps/chosen": -293.328125,
      "logps/ref_chosen": -293.3775634765625,
      "logps/ref_rejected": -301.59014892578125,
      "logps/rejected": -301.5428466796875,
      "loss": 5.5452,
      "margin_dpo/margin_mean": 0.0021340101957321167,
      "margin_dpo/margin_std": 0.9100000262260437,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.09282875061035156,
      "KL/mean": 0.045427776873111725,
      "KL/rejected_KL_mean": -0.0019741058349609375,
      "KL/std": 0.5971906185150146,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09480215609073639,
      "fcm_dpo/q_t": 0.49976301193237305,
      "grad_norm": 14.5261812210083,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.6747711896896362,
      "logits/rejected": 1.7073544263839722,
      "logps/chosen": -240.8408660888672,
      "logps/ref_chosen": -240.93368530273438,
      "logps/ref_rejected": -235.3230743408203,
      "logps/rejected": -235.3250274658203,
      "loss": 5.5414,
      "margin_dpo/margin_mean": 0.09480218589305878,
      "margin_dpo/margin_std": 0.6946413516998291,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": 0.08955955505371094,
      "KL/mean": 0.021658282727003098,
      "KL/rejected_KL_mean": -0.046237945556640625,
      "KL/std": 0.5573495626449585,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13580022752285004,
      "fcm_dpo/q_t": 0.49966052174568176,
      "grad_norm": 15.577567100524902,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": 1.7972111701965332,
      "logits/rejected": 1.7899937629699707,
      "logps/chosen": -273.5057373046875,
      "logps/ref_chosen": -273.5953369140625,
      "logps/ref_rejected": -234.49148559570312,
      "logps/rejected": -234.5377197265625,
      "loss": 5.5398,
      "margin_dpo/margin_mean": 0.13580012321472168,
      "margin_dpo/margin_std": 0.7732752561569214,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.008098602294921875,
      "KL/mean": 0.014123409986495972,
      "KL/rejected_KL_mean": 0.020158767700195312,
      "KL/std": 0.7065997123718262,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.01206989586353302,
      "fcm_dpo/q_t": 0.5000301599502563,
      "grad_norm": 15.160636901855469,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": 2.229604959487915,
      "logits/rejected": 2.0004024505615234,
      "logps/chosen": -314.2557678222656,
      "logps/ref_chosen": -314.26385498046875,
      "logps/ref_rejected": -242.06735229492188,
      "logps/rejected": -242.04718017578125,
      "loss": 5.5458,
      "margin_dpo/margin_mean": -0.012070447206497192,
      "margin_dpo/margin_std": 0.9462671875953674,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.1826629638671875,
      "KL/mean": 0.07688292115926743,
      "KL/rejected_KL_mean": -0.028898239135742188,
      "KL/std": 0.568365216255188,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.21156074106693268,
      "fcm_dpo/q_t": 0.49947109818458557,
      "grad_norm": 15.39802074432373,
      "learning_rate": 2.8125e-07,
      "logits/chosen": 2.045712471008301,
      "logits/rejected": 2.054311513900757,
      "logps/chosen": -240.55615234375,
      "logps/ref_chosen": -240.7388153076172,
      "logps/ref_rejected": -237.8331298828125,
      "logps/rejected": -237.86204528808594,
      "loss": 5.5368,
      "margin_dpo/margin_mean": 0.21156001091003418,
      "margin_dpo/margin_std": 0.6896741390228271,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.0751800537109375,
      "KL/mean": 0.03293202817440033,
      "KL/rejected_KL_mean": -0.009305953979492188,
      "KL/std": 0.6382588744163513,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.084482342004776,
      "fcm_dpo/q_t": 0.4997888207435608,
      "grad_norm": 14.697515487670898,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": 2.130385637283325,
      "logits/rejected": 2.0454113483428955,
      "logps/chosen": -307.7542419433594,
      "logps/ref_chosen": -307.82940673828125,
      "logps/ref_rejected": -281.8109130859375,
      "logps/rejected": -281.8202209472656,
      "loss": 5.5419,
      "margin_dpo/margin_mean": 0.08448109030723572,
      "margin_dpo/margin_std": 0.8532788157463074,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.0432586669921875,
      "KL/mean": 0.0011619031429290771,
      "KL/rejected_KL_mean": -0.04094886779785156,
      "KL/std": 0.6502746343612671,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08420863747596741,
      "fcm_dpo/q_t": 0.4997894763946533,
      "grad_norm": 14.42685317993164,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.899101972579956,
      "logits/rejected": 1.8992981910705566,
      "logps/chosen": -247.97052001953125,
      "logps/ref_chosen": -248.01376342773438,
      "logps/ref_rejected": -259.0014953613281,
      "logps/rejected": -259.04241943359375,
      "loss": 5.5419,
      "margin_dpo/margin_mean": 0.0842069536447525,
      "margin_dpo/margin_std": 0.8583498597145081,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.1374034881591797,
      "KL/mean": 0.06328336149454117,
      "KL/rejected_KL_mean": -0.010850906372070312,
      "KL/std": 0.6397906541824341,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.14825786650180817,
      "fcm_dpo/q_t": 0.49962934851646423,
      "grad_norm": 13.65766716003418,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 1.8710416555404663,
      "logits/rejected": 1.8022664785385132,
      "logps/chosen": -239.92538452148438,
      "logps/ref_chosen": -240.0628204345703,
      "logps/ref_rejected": -201.91021728515625,
      "logps/rejected": -201.92105102539062,
      "loss": 5.5393,
      "margin_dpo/margin_mean": 0.14825668931007385,
      "margin_dpo/margin_std": 0.8155995607376099,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.06440162658691406,
      "KL/mean": 0.06353146582841873,
      "KL/rejected_KL_mean": 0.06265830993652344,
      "KL/std": 0.6534550189971924,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.001749292016029358,
      "fcm_dpo/q_t": 0.4999956488609314,
      "grad_norm": 15.650973320007324,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": 1.8358665704727173,
      "logits/rejected": 1.7478594779968262,
      "logps/chosen": -294.06781005859375,
      "logps/ref_chosen": -294.1322326660156,
      "logps/ref_rejected": -265.8150634765625,
      "logps/rejected": -265.7524108886719,
      "loss": 5.5452,
      "margin_dpo/margin_mean": 0.001748502254486084,
      "margin_dpo/margin_std": 0.8746620416641235,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.3090553283691406,
      "KL/mean": 0.2201586812734604,
      "KL/rejected_KL_mean": 0.13124656677246094,
      "KL/std": 0.6832847595214844,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.17780199646949768,
      "fcm_dpo/q_t": 0.4995554983615875,
      "grad_norm": 14.917777061462402,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 1.9829485416412354,
      "logits/rejected": 1.9488799571990967,
      "logps/chosen": -275.3460388183594,
      "logps/ref_chosen": -275.6551208496094,
      "logps/ref_rejected": -253.5972137451172,
      "logps/rejected": -253.46597290039062,
      "loss": 5.5382,
      "margin_dpo/margin_mean": 0.17780190706253052,
      "margin_dpo/margin_std": 1.0026662349700928,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.2649383544921875,
      "KL/mean": 0.16725589334964752,
      "KL/rejected_KL_mean": 0.06957435607910156,
      "KL/std": 0.6840182542800903,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.19535642862319946,
      "fcm_dpo/q_t": 0.49951162934303284,
      "grad_norm": 17.693098068237305,
      "learning_rate": 3.4375e-07,
      "logits/chosen": 2.040569305419922,
      "logits/rejected": 1.929030179977417,
      "logps/chosen": -332.7052001953125,
      "logps/ref_chosen": -332.9701232910156,
      "logps/ref_rejected": -278.2213439941406,
      "logps/rejected": -278.15179443359375,
      "loss": 5.5375,
      "margin_dpo/margin_mean": 0.1953568309545517,
      "margin_dpo/margin_std": 0.9173576831817627,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.1398487091064453,
      "KL/mean": 0.11773490905761719,
      "KL/rejected_KL_mean": 0.09563446044921875,
      "KL/std": 0.7019012570381165,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0442175567150116,
      "fcm_dpo/q_t": 0.49988946318626404,
      "grad_norm": 13.668089866638184,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": 1.8009995222091675,
      "logits/rejected": 1.8618836402893066,
      "logps/chosen": -258.2890319824219,
      "logps/ref_chosen": -258.42889404296875,
      "logps/ref_rejected": -259.5119934082031,
      "logps/rejected": -259.4163513183594,
      "loss": 5.5435,
      "margin_dpo/margin_mean": 0.04421742260456085,
      "margin_dpo/margin_std": 0.9619231224060059,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.4707908630371094,
      "KL/mean": 0.2687016427516937,
      "KL/rejected_KL_mean": 0.06662368774414062,
      "KL/std": 0.7441911101341248,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.40417197346687317,
      "fcm_dpo/q_t": 0.49898964166641235,
      "grad_norm": 17.63559341430664,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": 1.8289793729782104,
      "logits/rejected": 1.774060606956482,
      "logps/chosen": -290.1485595703125,
      "logps/ref_chosen": -290.6193542480469,
      "logps/ref_rejected": -280.2503967285156,
      "logps/rejected": -280.18377685546875,
      "loss": 5.5291,
      "margin_dpo/margin_mean": 0.4041725695133209,
      "margin_dpo/margin_std": 1.0568442344665527,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.2463817596435547,
      "KL/mean": 0.16464485228061676,
      "KL/rejected_KL_mean": 0.08289527893066406,
      "KL/std": 0.6793686151504517,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16348880529403687,
      "fcm_dpo/q_t": 0.49959129095077515,
      "grad_norm": 15.171287536621094,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.8307770490646362,
      "logits/rejected": 1.8201007843017578,
      "logps/chosen": -303.3264465332031,
      "logps/ref_chosen": -303.57281494140625,
      "logps/ref_rejected": -294.56878662109375,
      "logps/rejected": -294.48590087890625,
      "loss": 5.5387,
      "margin_dpo/margin_mean": 0.16348886489868164,
      "margin_dpo/margin_std": 0.8614492416381836,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.2891521453857422,
      "KL/mean": 0.22324253618717194,
      "KL/rejected_KL_mean": 0.15732574462890625,
      "KL/std": 0.6973505020141602,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13182416558265686,
      "fcm_dpo/q_t": 0.4996704161167145,
      "grad_norm": 15.391763687133789,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": 1.8631380796432495,
      "logits/rejected": 1.9231772422790527,
      "logps/chosen": -272.26434326171875,
      "logps/ref_chosen": -272.5534973144531,
      "logps/ref_rejected": -266.9136047363281,
      "logps/rejected": -266.75628662109375,
      "loss": 5.54,
      "margin_dpo/margin_mean": 0.13182410597801208,
      "margin_dpo/margin_std": 0.9926130175590515,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 0.45537376403808594,
      "KL/mean": 0.260191410779953,
      "KL/rejected_KL_mean": 0.06501197814941406,
      "KL/std": 0.7496246695518494,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.39035946130752563,
      "fcm_dpo/q_t": 0.4990241229534149,
      "grad_norm": 15.740640640258789,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": 1.7758039236068726,
      "logits/rejected": 1.710078477859497,
      "logps/chosen": -264.93115234375,
      "logps/ref_chosen": -265.38653564453125,
      "logps/ref_rejected": -244.34774780273438,
      "logps/rejected": -244.28273010253906,
      "loss": 5.5297,
      "margin_dpo/margin_mean": 0.3903595805168152,
      "margin_dpo/margin_std": 0.9144065976142883,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 0.5072460174560547,
      "KL/mean": 0.2956080734729767,
      "KL/rejected_KL_mean": 0.08396720886230469,
      "KL/std": 0.7423378825187683,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.42327427864074707,
      "fcm_dpo/q_t": 0.4989418387413025,
      "grad_norm": 14.984747886657715,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.8754465579986572,
      "logits/rejected": 1.8791749477386475,
      "logps/chosen": -274.38653564453125,
      "logps/ref_chosen": -274.893798828125,
      "logps/ref_rejected": -244.3120880126953,
      "logps/rejected": -244.22813415527344,
      "loss": 5.5284,
      "margin_dpo/margin_mean": 0.42327409982681274,
      "margin_dpo/margin_std": 1.0150760412216187,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 0.3273792266845703,
      "KL/mean": 0.2266092300415039,
      "KL/rejected_KL_mean": 0.1258392333984375,
      "KL/std": 0.749426543712616,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.20153602957725525,
      "fcm_dpo/q_t": 0.4994961619377136,
      "grad_norm": 15.851800918579102,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 2.024256467819214,
      "logits/rejected": 1.9547948837280273,
      "logps/chosen": -284.47174072265625,
      "logps/ref_chosen": -284.7991027832031,
      "logps/ref_rejected": -248.8882598876953,
      "logps/rejected": -248.76242065429688,
      "loss": 5.5372,
      "margin_dpo/margin_mean": 0.20153622329235077,
      "margin_dpo/margin_std": 1.0675899982452393,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.4927787780761719,
      "KL/mean": 0.3301990032196045,
      "KL/rejected_KL_mean": 0.16762351989746094,
      "KL/std": 0.7858631014823914,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.32515203952789307,
      "fcm_dpo/q_t": 0.4991871416568756,
      "grad_norm": 14.575394630432129,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": 1.886594295501709,
      "logits/rejected": 1.941513180732727,
      "logps/chosen": -245.3468780517578,
      "logps/ref_chosen": -245.83966064453125,
      "logps/ref_rejected": -259.2870788574219,
      "logps/rejected": -259.1194763183594,
      "loss": 5.5323,
      "margin_dpo/margin_mean": 0.32515281438827515,
      "margin_dpo/margin_std": 1.0154105424880981,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.5243682861328125,
      "KL/mean": 0.2603346109390259,
      "KL/rejected_KL_mean": -0.003696441650390625,
      "KL/std": 0.9414912462234497,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5280691385269165,
      "fcm_dpo/q_t": 0.49867987632751465,
      "grad_norm": 15.643603324890137,
      "learning_rate": 4.375e-07,
      "logits/chosen": 1.784143090248108,
      "logits/rejected": 1.7582725286483765,
      "logps/chosen": -307.8731689453125,
      "logps/ref_chosen": -308.39752197265625,
      "logps/ref_rejected": -270.3797912597656,
      "logps/rejected": -270.3835144042969,
      "loss": 5.5242,
      "margin_dpo/margin_mean": 0.5280700922012329,
      "margin_dpo/margin_std": 1.1101527214050293,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 0.6805915832519531,
      "KL/mean": 0.43057548999786377,
      "KL/rejected_KL_mean": 0.1805553436279297,
      "KL/std": 0.8954533934593201,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.500026285648346,
      "fcm_dpo/q_t": 0.4987499713897705,
      "grad_norm": 15.784941673278809,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": 1.9257066249847412,
      "logits/rejected": 1.9312987327575684,
      "logps/chosen": -274.6811218261719,
      "logps/ref_chosen": -275.36175537109375,
      "logps/ref_rejected": -279.7540588378906,
      "logps/rejected": -279.5735168457031,
      "loss": 5.5253,
      "margin_dpo/margin_mean": 0.5000274181365967,
      "margin_dpo/margin_std": 1.1391080617904663,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 0.7811336517333984,
      "KL/mean": 0.5353459715843201,
      "KL/rejected_KL_mean": 0.28954505920410156,
      "KL/std": 1.0195398330688477,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4916016459465027,
      "fcm_dpo/q_t": 0.498771071434021,
      "grad_norm": 16.861371994018555,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": 1.900986909866333,
      "logits/rejected": 1.8807578086853027,
      "logps/chosen": -314.3390808105469,
      "logps/ref_chosen": -315.1202087402344,
      "logps/ref_rejected": -290.34210205078125,
      "logps/rejected": -290.05255126953125,
      "loss": 5.5257,
      "margin_dpo/margin_mean": 0.4916006624698639,
      "margin_dpo/margin_std": 1.3243650197982788,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 0.7896308898925781,
      "KL/mean": 0.42419320344924927,
      "KL/rejected_KL_mean": 0.058757781982421875,
      "KL/std": 1.019094705581665,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7308741211891174,
      "fcm_dpo/q_t": 0.49817290902137756,
      "grad_norm": 16.868202209472656,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": 1.686781883239746,
      "logits/rejected": 1.692981481552124,
      "logps/chosen": -292.1509704589844,
      "logps/ref_chosen": -292.94061279296875,
      "logps/ref_rejected": -264.0979309082031,
      "logps/rejected": -264.0391845703125,
      "loss": 5.5162,
      "margin_dpo/margin_mean": 0.7308739423751831,
      "margin_dpo/margin_std": 1.3454350233078003,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 0.6583328247070312,
      "KL/mean": 0.48026275634765625,
      "KL/rejected_KL_mean": 0.30219268798828125,
      "KL/std": 1.0069941282272339,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3561379909515381,
      "fcm_dpo/q_t": 0.4991097152233124,
      "grad_norm": 13.528368949890137,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": 1.7513269186019897,
      "logits/rejected": 1.8528504371643066,
      "logps/chosen": -254.19436645507812,
      "logps/ref_chosen": -254.85269165039062,
      "logps/ref_rejected": -255.88380432128906,
      "logps/rejected": -255.58160400390625,
      "loss": 5.5312,
      "margin_dpo/margin_mean": 0.3561379313468933,
      "margin_dpo/margin_std": 1.472796082496643,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 0.78314208984375,
      "KL/mean": 0.47325336933135986,
      "KL/rejected_KL_mean": 0.1633625030517578,
      "KL/std": 1.0837169885635376,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6197763681411743,
      "fcm_dpo/q_t": 0.49845072627067566,
      "grad_norm": 14.833471298217773,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": 1.9131228923797607,
      "logits/rejected": 1.900570273399353,
      "logps/chosen": -261.7245788574219,
      "logps/ref_chosen": -262.5077209472656,
      "logps/ref_rejected": -239.25320434570312,
      "logps/rejected": -239.08984375,
      "loss": 5.5206,
      "margin_dpo/margin_mean": 0.6197762489318848,
      "margin_dpo/margin_std": 1.3231393098831177,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.8858737945556641,
      "KL/mean": 0.5616776943206787,
      "KL/rejected_KL_mean": 0.23749923706054688,
      "KL/std": 1.277164340019226,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6483790874481201,
      "fcm_dpo/q_t": 0.4983791708946228,
      "grad_norm": 15.283793449401855,
      "learning_rate": 5e-07,
      "logits/chosen": 2.143604278564453,
      "logits/rejected": 2.2534432411193848,
      "logps/chosen": -248.6420135498047,
      "logps/ref_chosen": -249.5278778076172,
      "logps/ref_rejected": -254.6465301513672,
      "logps/rejected": -254.40902709960938,
      "loss": 5.5196,
      "margin_dpo/margin_mean": 0.6483792066574097,
      "margin_dpo/margin_std": 1.7016257047653198,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 0.8800182342529297,
      "KL/mean": 0.7491053342819214,
      "KL/rejected_KL_mean": 0.6181755065917969,
      "KL/std": 1.1928025484085083,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.26184675097465515,
      "fcm_dpo/q_t": 0.49934545159339905,
      "grad_norm": 14.621295928955078,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 1.9818304777145386,
      "logits/rejected": 1.9931281805038452,
      "logps/chosen": -291.6217041015625,
      "logps/ref_chosen": -292.501708984375,
      "logps/ref_rejected": -294.6496887207031,
      "logps/rejected": -294.0315246582031,
      "loss": 5.535,
      "margin_dpo/margin_mean": 0.2618473172187805,
      "margin_dpo/margin_std": 1.5508722066879272,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 0.9437465667724609,
      "KL/mean": 0.6142023801803589,
      "KL/rejected_KL_mean": 0.2846508026123047,
      "KL/std": 1.30777907371521,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6590973138809204,
      "fcm_dpo/q_t": 0.49835243821144104,
      "grad_norm": 14.762439727783203,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 1.5286736488342285,
      "logits/rejected": 1.655385971069336,
      "logps/chosen": -258.91815185546875,
      "logps/ref_chosen": -259.8619079589844,
      "logps/ref_rejected": -293.1994934082031,
      "logps/rejected": -292.9148254394531,
      "loss": 5.5192,
      "margin_dpo/margin_mean": 0.6590976715087891,
      "margin_dpo/margin_std": 1.7874044179916382,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 1.251302719116211,
      "KL/mean": 0.7785479426383972,
      "KL/rejected_KL_mean": 0.30579185485839844,
      "KL/std": 1.4616304636001587,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9455054402351379,
      "fcm_dpo/q_t": 0.4976365566253662,
      "grad_norm": 17.057376861572266,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": 1.9757074117660522,
      "logits/rejected": 1.8941067457199097,
      "logps/chosen": -297.1740417480469,
      "logps/ref_chosen": -298.42529296875,
      "logps/ref_rejected": -278.5128479003906,
      "logps/rejected": -278.20703125,
      "loss": 5.5078,
      "margin_dpo/margin_mean": 0.9455058574676514,
      "margin_dpo/margin_std": 1.8139084577560425,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 1.1298675537109375,
      "KL/mean": 0.7704100012779236,
      "KL/rejected_KL_mean": 0.4109535217285156,
      "KL/std": 1.5054417848587036,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7189161777496338,
      "fcm_dpo/q_t": 0.498202919960022,
      "grad_norm": 16.02787208557129,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": 1.8206652402877808,
      "logits/rejected": 1.8901110887527466,
      "logps/chosen": -274.17864990234375,
      "logps/ref_chosen": -275.3085021972656,
      "logps/ref_rejected": -267.97381591796875,
      "logps/rejected": -267.5628967285156,
      "loss": 5.5169,
      "margin_dpo/margin_mean": 0.718916654586792,
      "margin_dpo/margin_std": 1.8696832656860352,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 1.2575931549072266,
      "KL/mean": 0.9034296870231628,
      "KL/rejected_KL_mean": 0.5492649078369141,
      "KL/std": 1.4823657274246216,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7083348035812378,
      "fcm_dpo/q_t": 0.4982295036315918,
      "grad_norm": 14.58376407623291,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": 1.6908724308013916,
      "logits/rejected": 1.7171398401260376,
      "logps/chosen": -296.8897399902344,
      "logps/ref_chosen": -298.1473388671875,
      "logps/ref_rejected": -269.7539367675781,
      "logps/rejected": -269.20465087890625,
      "loss": 5.5174,
      "margin_dpo/margin_mean": 0.7083349227905273,
      "margin_dpo/margin_std": 2.093512535095215,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.157205581665039,
      "KL/mean": 0.8328753709793091,
      "KL/rejected_KL_mean": 0.5085430145263672,
      "KL/std": 1.3730131387710571,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6486632823944092,
      "fcm_dpo/q_t": 0.49837857484817505,
      "grad_norm": 12.803271293640137,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": 1.629069447517395,
      "logits/rejected": 1.6764333248138428,
      "logps/chosen": -215.59278869628906,
      "logps/ref_chosen": -216.74998474121094,
      "logps/ref_rejected": -202.7234649658203,
      "logps/rejected": -202.2149200439453,
      "loss": 5.5196,
      "margin_dpo/margin_mean": 0.6486636400222778,
      "margin_dpo/margin_std": 1.7984572649002075,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 1.3076725006103516,
      "KL/mean": 0.7590950727462769,
      "KL/rejected_KL_mean": 0.21051406860351562,
      "KL/std": 1.8011634349822998,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0971477031707764,
      "fcm_dpo/q_t": 0.4972575902938843,
      "grad_norm": 14.409942626953125,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": 1.4794809818267822,
      "logits/rejected": 1.5605554580688477,
      "logps/chosen": -242.38372802734375,
      "logps/ref_chosen": -243.69140625,
      "logps/ref_rejected": -234.970947265625,
      "logps/rejected": -234.7604217529297,
      "loss": 5.502,
      "margin_dpo/margin_mean": 1.0971472263336182,
      "margin_dpo/margin_std": 2.2036404609680176,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 1.6642875671386719,
      "KL/mean": 0.9250648021697998,
      "KL/rejected_KL_mean": 0.18584823608398438,
      "KL/std": 1.734178066253662,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.478427767753601,
      "fcm_dpo/q_t": 0.4963049590587616,
      "grad_norm": 17.577533721923828,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": 2.0916197299957275,
      "logits/rejected": 1.9944347143173218,
      "logps/chosen": -309.08203125,
      "logps/ref_chosen": -310.746337890625,
      "logps/ref_rejected": -270.6407165527344,
      "logps/rejected": -270.454833984375,
      "loss": 5.4868,
      "margin_dpo/margin_mean": 1.478427529335022,
      "margin_dpo/margin_std": 2.131805181503296,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 1.3194465637207031,
      "KL/mean": 0.8101028800010681,
      "KL/rejected_KL_mean": 0.30075645446777344,
      "KL/std": 1.5933494567871094,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0186911821365356,
      "fcm_dpo/q_t": 0.4974534809589386,
      "grad_norm": 14.847064018249512,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": 1.7372748851776123,
      "logits/rejected": 1.7237883806228638,
      "logps/chosen": -257.4935302734375,
      "logps/ref_chosen": -258.8130187988281,
      "logps/ref_rejected": -268.2101745605469,
      "logps/rejected": -267.909423828125,
      "loss": 5.505,
      "margin_dpo/margin_mean": 1.0186913013458252,
      "margin_dpo/margin_std": 1.938852310180664,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 1.5452423095703125,
      "KL/mean": 1.065231442451477,
      "KL/rejected_KL_mean": 0.5852184295654297,
      "KL/std": 1.8685449361801147,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9600273370742798,
      "fcm_dpo/q_t": 0.49760037660598755,
      "grad_norm": 15.252073287963867,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": 1.771957516670227,
      "logits/rejected": 1.7099394798278809,
      "logps/chosen": -275.6961975097656,
      "logps/ref_chosen": -277.2414245605469,
      "logps/ref_rejected": -259.13519287109375,
      "logps/rejected": -258.54998779296875,
      "loss": 5.5075,
      "margin_dpo/margin_mean": 0.9600266218185425,
      "margin_dpo/margin_std": 2.4911017417907715,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 1.5499248504638672,
      "KL/mean": 0.8974651098251343,
      "KL/rejected_KL_mean": 0.24501800537109375,
      "KL/std": 1.6625615358352661,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3049015998840332,
      "fcm_dpo/q_t": 0.4967382848262787,
      "grad_norm": 14.738645553588867,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.9203906059265137,
      "logits/rejected": 1.9691612720489502,
      "logps/chosen": -249.79595947265625,
      "logps/ref_chosen": -251.34585571289062,
      "logps/ref_rejected": -251.4120635986328,
      "logps/rejected": -251.1670379638672,
      "loss": 5.4937,
      "margin_dpo/margin_mean": 1.3049015998840332,
      "margin_dpo/margin_std": 2.2654995918273926,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 1.7110366821289062,
      "KL/mean": 1.1365405321121216,
      "KL/rejected_KL_mean": 0.5620498657226562,
      "KL/std": 1.8249735832214355,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1489834785461426,
      "fcm_dpo/q_t": 0.49712812900543213,
      "grad_norm": 15.017074584960938,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 1.966935634613037,
      "logits/rejected": 1.9370795488357544,
      "logps/chosen": -267.2715759277344,
      "logps/ref_chosen": -268.98260498046875,
      "logps/ref_rejected": -249.34474182128906,
      "logps/rejected": -248.78269958496094,
      "loss": 5.5,
      "margin_dpo/margin_mean": 1.1489825248718262,
      "margin_dpo/margin_std": 2.4676573276519775,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 1.9911270141601562,
      "KL/mean": 1.424992561340332,
      "KL/rejected_KL_mean": 0.8588619232177734,
      "KL/std": 1.9739683866500854,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1322662830352783,
      "fcm_dpo/q_t": 0.49717003107070923,
      "grad_norm": 14.668818473815918,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 1.8789650201797485,
      "logits/rejected": 1.8489294052124023,
      "logps/chosen": -262.2123718261719,
      "logps/ref_chosen": -264.2034912109375,
      "logps/ref_rejected": -260.2911376953125,
      "logps/rejected": -259.4322814941406,
      "loss": 5.5007,
      "margin_dpo/margin_mean": 1.1322656869888306,
      "margin_dpo/margin_std": 2.535112142562866,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 1.9002227783203125,
      "KL/mean": 1.2612552642822266,
      "KL/rejected_KL_mean": 0.6222820281982422,
      "KL/std": 2.0985348224639893,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2779570817947388,
      "fcm_dpo/q_t": 0.49680599570274353,
      "grad_norm": 14.795960426330566,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": 1.9356800317764282,
      "logits/rejected": 1.9698055982589722,
      "logps/chosen": -269.56805419921875,
      "logps/ref_chosen": -271.4682922363281,
      "logps/ref_rejected": -273.0804748535156,
      "logps/rejected": -272.4582214355469,
      "loss": 5.4951,
      "margin_dpo/margin_mean": 1.277957797050476,
      "margin_dpo/margin_std": 2.780975580215454,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 2.3085575103759766,
      "KL/mean": 1.5609891414642334,
      "KL/rejected_KL_mean": 0.8134174346923828,
      "KL/std": 2.36395525932312,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4951350688934326,
      "fcm_dpo/q_t": 0.49626338481903076,
      "grad_norm": 15.672703742980957,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": 1.9378714561462402,
      "logits/rejected": 1.9545704126358032,
      "logps/chosen": -271.29486083984375,
      "logps/ref_chosen": -273.6033935546875,
      "logps/ref_rejected": -259.54107666015625,
      "logps/rejected": -258.7276611328125,
      "loss": 5.4866,
      "margin_dpo/margin_mean": 1.4951344728469849,
      "margin_dpo/margin_std": 3.078324794769287,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 2.2617340087890625,
      "KL/mean": 1.7241895198822021,
      "KL/rejected_KL_mean": 1.1866474151611328,
      "KL/std": 2.1204512119293213,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0750895738601685,
      "fcm_dpo/q_t": 0.4973127543926239,
      "grad_norm": 15.372101783752441,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": 2.083489179611206,
      "logits/rejected": 2.1773788928985596,
      "logps/chosen": -256.25592041015625,
      "logps/ref_chosen": -258.5176696777344,
      "logps/ref_rejected": -264.9944763183594,
      "logps/rejected": -263.8078308105469,
      "loss": 5.5031,
      "margin_dpo/margin_mean": 1.075089454650879,
      "margin_dpo/margin_std": 2.750234365463257,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 2.195375442504883,
      "KL/mean": 1.4930734634399414,
      "KL/rejected_KL_mean": 0.7907657623291016,
      "KL/std": 2.744586944580078,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.404599905014038,
      "fcm_dpo/q_t": 0.49648985266685486,
      "grad_norm": 14.875286102294922,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": 1.8524079322814941,
      "logits/rejected": 1.9490604400634766,
      "logps/chosen": -265.31402587890625,
      "logps/ref_chosen": -267.5094299316406,
      "logps/ref_rejected": -277.03228759765625,
      "logps/rejected": -276.2414855957031,
      "loss": 5.4906,
      "margin_dpo/margin_mean": 1.4045995473861694,
      "margin_dpo/margin_std": 3.6968605518341064,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 2.5117244720458984,
      "KL/mean": 1.915096402168274,
      "KL/rejected_KL_mean": 1.3184623718261719,
      "KL/std": 2.6644787788391113,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1932601928710938,
      "fcm_dpo/q_t": 0.497017502784729,
      "grad_norm": 14.232949256896973,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": 1.887174367904663,
      "logits/rejected": 1.919933557510376,
      "logps/chosen": -254.95321655273438,
      "logps/ref_chosen": -257.4649353027344,
      "logps/ref_rejected": -247.29161071777344,
      "logps/rejected": -245.97317504882812,
      "loss": 5.4988,
      "margin_dpo/margin_mean": 1.1932605504989624,
      "margin_dpo/margin_std": 3.3311574459075928,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 2.5270214080810547,
      "KL/mean": 1.6230802536010742,
      "KL/rejected_KL_mean": 0.7191505432128906,
      "KL/std": 2.9457080364227295,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8078699111938477,
      "fcm_dpo/q_t": 0.49548301100730896,
      "grad_norm": 14.843220710754395,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": 1.605433702468872,
      "logits/rejected": 1.542537808418274,
      "logps/chosen": -287.9571533203125,
      "logps/ref_chosen": -290.4841613769531,
      "logps/ref_rejected": -260.8398132324219,
      "logps/rejected": -260.12066650390625,
      "loss": 5.4749,
      "margin_dpo/margin_mean": 1.8078696727752686,
      "margin_dpo/margin_std": 3.9227993488311768,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 2.7792186737060547,
      "KL/mean": 1.7111592292785645,
      "KL/rejected_KL_mean": 0.6430931091308594,
      "KL/std": 2.9758267402648926,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.1361334323883057,
      "fcm_dpo/q_t": 0.49466198682785034,
      "grad_norm": 15.008553504943848,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": 1.719646692276001,
      "logits/rejected": 1.7415354251861572,
      "logps/chosen": -254.8583526611328,
      "logps/ref_chosen": -257.6375732421875,
      "logps/ref_rejected": -235.6666717529297,
      "logps/rejected": -235.02357482910156,
      "loss": 5.4617,
      "margin_dpo/margin_mean": 2.1361329555511475,
      "margin_dpo/margin_std": 3.743652105331421,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 3.323831558227539,
      "KL/mean": 2.2906320095062256,
      "KL/rejected_KL_mean": 1.2574405670166016,
      "KL/std": 2.986874580383301,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0663914680480957,
      "fcm_dpo/q_t": 0.4948362708091736,
      "grad_norm": 15.584107398986816,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.6770509481430054,
      "logits/rejected": 1.6287431716918945,
      "logps/chosen": -284.1413879394531,
      "logps/ref_chosen": -287.4651794433594,
      "logps/ref_rejected": -253.4770965576172,
      "logps/rejected": -252.21966552734375,
      "loss": 5.4644,
      "margin_dpo/margin_mean": 2.066391944885254,
      "margin_dpo/margin_std": 3.6792988777160645,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 3.5649890899658203,
      "KL/mean": 2.5647974014282227,
      "KL/rejected_KL_mean": 1.5646095275878906,
      "KL/std": 3.363189935684204,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.000389575958252,
      "fcm_dpo/q_t": 0.4950013756752014,
      "grad_norm": 14.953012466430664,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 2.1582775115966797,
      "logits/rejected": 2.1838529109954834,
      "logps/chosen": -277.3703918457031,
      "logps/ref_chosen": -280.9353942871094,
      "logps/ref_rejected": -275.998779296875,
      "logps/rejected": -274.4341735839844,
      "loss": 5.4677,
      "margin_dpo/margin_mean": 2.0003890991210938,
      "margin_dpo/margin_std": 4.392543792724609,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 3.0343589782714844,
      "KL/mean": 2.0330607891082764,
      "KL/rejected_KL_mean": 1.0317554473876953,
      "KL/std": 3.0929276943206787,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0026040077209473,
      "fcm_dpo/q_t": 0.4949961304664612,
      "grad_norm": 14.62278938293457,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": 1.8820792436599731,
      "logits/rejected": 2.0333633422851562,
      "logps/chosen": -227.11117553710938,
      "logps/ref_chosen": -230.14553833007812,
      "logps/ref_rejected": -253.37661743164062,
      "logps/rejected": -252.3448486328125,
      "loss": 5.4674,
      "margin_dpo/margin_mean": 2.0026028156280518,
      "margin_dpo/margin_std": 4.098149299621582,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 3.8216724395751953,
      "KL/mean": 2.8509998321533203,
      "KL/rejected_KL_mean": 1.8803176879882812,
      "KL/std": 3.2355880737304688,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.941349983215332,
      "fcm_dpo/q_t": 0.49514836072921753,
      "grad_norm": 14.768844604492188,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": 1.7473868131637573,
      "logits/rejected": 1.7694472074508667,
      "logps/chosen": -261.0445861816406,
      "logps/ref_chosen": -264.86627197265625,
      "logps/ref_rejected": -245.21304321289062,
      "logps/rejected": -243.33270263671875,
      "loss": 5.47,
      "margin_dpo/margin_mean": 1.941348671913147,
      "margin_dpo/margin_std": 4.3890061378479,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.6936988830566406,
      "KL/mean": 2.736142635345459,
      "KL/rejected_KL_mean": 1.7786006927490234,
      "KL/std": 3.512693166732788,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9150992631912231,
      "fcm_dpo/q_t": 0.4952147305011749,
      "grad_norm": 15.662399291992188,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": 1.7654486894607544,
      "logits/rejected": 1.818976640701294,
      "logps/chosen": -330.85980224609375,
      "logps/ref_chosen": -334.55352783203125,
      "logps/ref_rejected": -325.8403015136719,
      "logps/rejected": -324.06170654296875,
      "loss": 5.4713,
      "margin_dpo/margin_mean": 1.9151005744934082,
      "margin_dpo/margin_std": 4.61562442779541,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.566276550292969,
      "KL/mean": 3.2108068466186523,
      "KL/rejected_KL_mean": 1.8553390502929688,
      "KL/std": 3.6159720420837402,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.7109451293945312,
      "fcm_dpo/q_t": 0.4932263493537903,
      "grad_norm": 15.614794731140137,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 1.6951671838760376,
      "logits/rejected": 1.7732917070388794,
      "logps/chosen": -255.8555908203125,
      "logps/ref_chosen": -260.421875,
      "logps/ref_rejected": -273.0882263183594,
      "logps/rejected": -271.2328796386719,
      "loss": 5.4395,
      "margin_dpo/margin_mean": 2.7109434604644775,
      "margin_dpo/margin_std": 4.170916557312012,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.471260070800781,
      "KL/mean": 2.96305513381958,
      "KL/rejected_KL_mean": 1.454854965209961,
      "KL/std": 3.8794991970062256,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0164175033569336,
      "fcm_dpo/q_t": 0.49246683716773987,
      "grad_norm": 16.729005813598633,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": 2.0382630825042725,
      "logits/rejected": 2.038696765899658,
      "logps/chosen": -267.2641906738281,
      "logps/ref_chosen": -271.7354431152344,
      "logps/ref_rejected": -262.64111328125,
      "logps/rejected": -261.186279296875,
      "loss": 5.428,
      "margin_dpo/margin_mean": 3.016417980194092,
      "margin_dpo/margin_std": 4.841452598571777,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 4.849123001098633,
      "KL/mean": 3.6430552005767822,
      "KL/rejected_KL_mean": 2.4369888305664062,
      "KL/std": 4.067503452301025,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.4121294021606445,
      "fcm_dpo/q_t": 0.49397528171539307,
      "grad_norm": 14.87447452545166,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": 1.888848900794983,
      "logits/rejected": 1.7491939067840576,
      "logps/chosen": -289.821533203125,
      "logps/ref_chosen": -294.6706237792969,
      "logps/ref_rejected": -247.6900177001953,
      "logps/rejected": -245.25303649902344,
      "loss": 5.4521,
      "margin_dpo/margin_mean": 2.412130832672119,
      "margin_dpo/margin_std": 5.053922653198242,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 5.1159210205078125,
      "KL/mean": 3.6120309829711914,
      "KL/rejected_KL_mean": 2.108135223388672,
      "KL/std": 3.716535806655884,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.007789134979248,
      "fcm_dpo/q_t": 0.49248847365379333,
      "grad_norm": 14.61108112335205,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": 1.6304445266723633,
      "logits/rejected": 1.5596693754196167,
      "logps/chosen": -243.03636169433594,
      "logps/ref_chosen": -248.1522979736328,
      "logps/ref_rejected": -209.12808227539062,
      "logps/rejected": -207.01991271972656,
      "loss": 5.4286,
      "margin_dpo/margin_mean": 3.007788896560669,
      "margin_dpo/margin_std": 5.173450946807861,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 5.028097152709961,
      "KL/mean": 3.478374719619751,
      "KL/rejected_KL_mean": 1.92864990234375,
      "KL/std": 4.773340225219727,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0994443893432617,
      "fcm_dpo/q_t": 0.4922539293766022,
      "grad_norm": 16.631750106811523,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": 1.8265513181686401,
      "logits/rejected": 1.892004132270813,
      "logps/chosen": -254.4462890625,
      "logps/ref_chosen": -259.4743957519531,
      "logps/ref_rejected": -246.9627227783203,
      "logps/rejected": -245.0340576171875,
      "loss": 5.4266,
      "margin_dpo/margin_mean": 3.0994436740875244,
      "margin_dpo/margin_std": 6.184051990509033,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 5.31639289855957,
      "KL/mean": 4.028312683105469,
      "KL/rejected_KL_mean": 2.7402400970458984,
      "KL/std": 4.718283176422119,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.57615065574646,
      "fcm_dpo/q_t": 0.4935661554336548,
      "grad_norm": 15.732903480529785,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.735234022140503,
      "logits/rejected": 1.72966468334198,
      "logps/chosen": -280.6101379394531,
      "logps/ref_chosen": -285.926513671875,
      "logps/ref_rejected": -253.77996826171875,
      "logps/rejected": -251.03973388671875,
      "loss": 5.4466,
      "margin_dpo/margin_mean": 2.5761513710021973,
      "margin_dpo/margin_std": 5.702394962310791,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 5.880681991577148,
      "KL/mean": 4.445310592651367,
      "KL/rejected_KL_mean": 3.0099353790283203,
      "KL/std": 4.4786834716796875,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.8707432746887207,
      "fcm_dpo/q_t": 0.4928288161754608,
      "grad_norm": 14.834258079528809,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 1.9215072393417358,
      "logits/rejected": 1.8777186870574951,
      "logps/chosen": -256.7135314941406,
      "logps/ref_chosen": -262.5942077636719,
      "logps/ref_rejected": -236.687744140625,
      "logps/rejected": -233.6778106689453,
      "loss": 5.4355,
      "margin_dpo/margin_mean": 2.870744466781616,
      "margin_dpo/margin_std": 6.250294208526611,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 6.352638244628906,
      "KL/mean": 4.857167720794678,
      "KL/rejected_KL_mean": 3.3616943359375,
      "KL/std": 4.362043380737305,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9909467697143555,
      "fcm_dpo/q_t": 0.49253010749816895,
      "grad_norm": 16.39931869506836,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": 1.803332805633545,
      "logits/rejected": 1.7550854682922363,
      "logps/chosen": -264.6905212402344,
      "logps/ref_chosen": -271.04315185546875,
      "logps/ref_rejected": -234.03359985351562,
      "logps/rejected": -230.67189025878906,
      "loss": 5.43,
      "margin_dpo/margin_mean": 2.9909467697143555,
      "margin_dpo/margin_std": 5.566059112548828,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 6.446025848388672,
      "KL/mean": 4.756011962890625,
      "KL/rejected_KL_mean": 3.065988540649414,
      "KL/std": 4.815212249755859,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.3800432682037354,
      "fcm_dpo/q_t": 0.49156296253204346,
      "grad_norm": 15.409664154052734,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": 1.8430030345916748,
      "logits/rejected": 1.926820993423462,
      "logps/chosen": -259.8392639160156,
      "logps/ref_chosen": -266.2852783203125,
      "logps/ref_rejected": -274.32501220703125,
      "logps/rejected": -271.259033203125,
      "loss": 5.4158,
      "margin_dpo/margin_mean": 3.3800435066223145,
      "margin_dpo/margin_std": 6.471216678619385,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 6.74946403503418,
      "KL/mean": 5.361313343048096,
      "KL/rejected_KL_mean": 3.973173141479492,
      "KL/std": 5.127292633056641,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.7762832641601562,
      "fcm_dpo/q_t": 0.49306464195251465,
      "grad_norm": 15.169913291931152,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": 1.7457847595214844,
      "logits/rejected": 1.7286295890808105,
      "logps/chosen": -280.0311279296875,
      "logps/ref_chosen": -286.7806091308594,
      "logps/ref_rejected": -276.1176452636719,
      "logps/rejected": -272.1445007324219,
      "loss": 5.44,
      "margin_dpo/margin_mean": 2.776282787322998,
      "margin_dpo/margin_std": 6.751436710357666,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": 6.277626037597656,
      "KL/mean": 5.300111770629883,
      "KL/rejected_KL_mean": 4.322593688964844,
      "KL/std": 4.370006561279297,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.955026388168335,
      "fcm_dpo/q_t": 0.495115727186203,
      "grad_norm": 13.868840217590332,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": 1.7031705379486084,
      "logits/rejected": 1.804017186164856,
      "logps/chosen": -228.75225830078125,
      "logps/ref_chosen": -235.02987670898438,
      "logps/ref_rejected": -259.80999755859375,
      "logps/rejected": -255.48741149902344,
      "loss": 5.4707,
      "margin_dpo/margin_mean": 1.955026388168335,
      "margin_dpo/margin_std": 5.387626647949219,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 7.979644775390625,
      "KL/mean": 6.141674041748047,
      "KL/rejected_KL_mean": 4.303709030151367,
      "KL/std": 5.578531265258789,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.67594575881958,
      "fcm_dpo/q_t": 0.49082422256469727,
      "grad_norm": 16.438709259033203,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": 1.7990089654922485,
      "logits/rejected": 1.8373925685882568,
      "logps/chosen": -293.3330993652344,
      "logps/ref_chosen": -301.312744140625,
      "logps/ref_rejected": -288.8368225097656,
      "logps/rejected": -284.5331115722656,
      "loss": 5.4052,
      "margin_dpo/margin_mean": 3.67594575881958,
      "margin_dpo/margin_std": 7.400429725646973,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": 8.079252243041992,
      "KL/mean": 6.026641845703125,
      "KL/rejected_KL_mean": 3.9740333557128906,
      "KL/std": 5.577757358551025,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.105222702026367,
      "fcm_dpo/q_t": 0.4897513687610626,
      "grad_norm": 15.31914234161377,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": 1.6188641786575317,
      "logits/rejected": 1.5959441661834717,
      "logps/chosen": -257.58673095703125,
      "logps/ref_chosen": -265.6659851074219,
      "logps/ref_rejected": -242.0697479248047,
      "logps/rejected": -238.09573364257812,
      "loss": 5.3873,
      "margin_dpo/margin_mean": 4.105222702026367,
      "margin_dpo/margin_std": 6.561320781707764,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": 7.315607070922852,
      "KL/mean": 5.6902384757995605,
      "KL/rejected_KL_mean": 4.064859390258789,
      "KL/std": 5.568142414093018,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.250743865966797,
      "fcm_dpo/q_t": 0.49188435077667236,
      "grad_norm": 14.060829162597656,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 1.741537094116211,
      "logits/rejected": 1.7563198804855347,
      "logps/chosen": -253.4058380126953,
      "logps/ref_chosen": -260.721435546875,
      "logps/ref_rejected": -262.6593322753906,
      "logps/rejected": -258.5944519042969,
      "loss": 5.4217,
      "margin_dpo/margin_mean": 3.250744104385376,
      "margin_dpo/margin_std": 7.197340965270996,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": 8.98629379272461,
      "KL/mean": 7.044980525970459,
      "KL/rejected_KL_mean": 5.103668212890625,
      "KL/std": 5.245534420013428,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.882640838623047,
      "fcm_dpo/q_t": 0.4903075695037842,
      "grad_norm": 15.825482368469238,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": 1.7950029373168945,
      "logits/rejected": 1.7603105306625366,
      "logps/chosen": -248.0165252685547,
      "logps/ref_chosen": -257.0028076171875,
      "logps/ref_rejected": -254.66232299804688,
      "logps/rejected": -249.55865478515625,
      "loss": 5.396,
      "margin_dpo/margin_mean": 3.882641553878784,
      "margin_dpo/margin_std": 6.403326988220215,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": 8.911468505859375,
      "KL/mean": 7.1843461990356445,
      "KL/rejected_KL_mean": 5.45722770690918,
      "KL/std": 5.602308750152588,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.454246997833252,
      "fcm_dpo/q_t": 0.4913773834705353,
      "grad_norm": 21.250869750976562,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 2.1231849193573,
      "logits/rejected": 2.073535442352295,
      "logps/chosen": -289.4989318847656,
      "logps/ref_chosen": -298.4103698730469,
      "logps/ref_rejected": -262.8761901855469,
      "logps/rejected": -257.4189453125,
      "loss": 5.4136,
      "margin_dpo/margin_mean": 3.4542462825775146,
      "margin_dpo/margin_std": 6.853506088256836,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": 8.765096664428711,
      "KL/mean": 7.045851707458496,
      "KL/rejected_KL_mean": 5.3266143798828125,
      "KL/std": 5.466521739959717,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.438483238220215,
      "fcm_dpo/q_t": 0.4914211630821228,
      "grad_norm": 14.660451889038086,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 1.726427674293518,
      "logits/rejected": 1.7781046628952026,
      "logps/chosen": -218.7587890625,
      "logps/ref_chosen": -227.5238800048828,
      "logps/ref_rejected": -227.11428833007812,
      "logps/rejected": -221.78768920898438,
      "loss": 5.4147,
      "margin_dpo/margin_mean": 3.4384827613830566,
      "margin_dpo/margin_std": 7.3730669021606445,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": 8.98878288269043,
      "KL/mean": 7.099464416503906,
      "KL/rejected_KL_mean": 5.2101287841796875,
      "KL/std": 6.597339630126953,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.778658390045166,
      "fcm_dpo/q_t": 0.4905644953250885,
      "grad_norm": 15.581783294677734,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": 1.8392298221588135,
      "logits/rejected": 1.7416222095489502,
      "logps/chosen": -266.3934326171875,
      "logps/ref_chosen": -275.3822326660156,
      "logps/ref_rejected": -269.8356018066406,
      "logps/rejected": -264.62548828125,
      "loss": 5.4017,
      "margin_dpo/margin_mean": 3.7786576747894287,
      "margin_dpo/margin_std": 7.611619472503662,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": 9.172033309936523,
      "KL/mean": 7.421346187591553,
      "KL/rejected_KL_mean": 5.67066764831543,
      "KL/std": 5.763102054595947,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.5013694763183594,
      "fcm_dpo/q_t": 0.4912623167037964,
      "grad_norm": 14.231635093688965,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": 1.9079065322875977,
      "logits/rejected": 1.9036587476730347,
      "logps/chosen": -218.94436645507812,
      "logps/ref_chosen": -228.1163787841797,
      "logps/ref_rejected": -228.26266479492188,
      "logps/rejected": -222.59197998046875,
      "loss": 5.412,
      "margin_dpo/margin_mean": 3.501368999481201,
      "margin_dpo/margin_std": 7.20575475692749,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": 9.937362670898438,
      "KL/mean": 7.971536636352539,
      "KL/rejected_KL_mean": 6.005708694458008,
      "KL/std": 6.632960319519043,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.9316585063934326,
      "fcm_dpo/q_t": 0.49019432067871094,
      "grad_norm": 15.57259464263916,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": 1.5876637697219849,
      "logits/rejected": 1.6670212745666504,
      "logps/chosen": -274.0898742675781,
      "logps/ref_chosen": -284.0272521972656,
      "logps/ref_rejected": -282.1720275878906,
      "logps/rejected": -276.16632080078125,
      "loss": 5.3979,
      "margin_dpo/margin_mean": 3.931658983230591,
      "margin_dpo/margin_std": 8.775503158569336,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": 9.805849075317383,
      "KL/mean": 7.5600266456604,
      "KL/rejected_KL_mean": 5.314207077026367,
      "KL/std": 6.795499801635742,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.491641044616699,
      "fcm_dpo/q_t": 0.4887992739677429,
      "grad_norm": 36.729461669921875,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": 1.8473128080368042,
      "logits/rejected": 1.7789896726608276,
      "logps/chosen": -293.24395751953125,
      "logps/ref_chosen": -303.0498352050781,
      "logps/ref_rejected": -259.8980712890625,
      "logps/rejected": -254.58387756347656,
      "loss": 5.3757,
      "margin_dpo/margin_mean": 4.491640090942383,
      "margin_dpo/margin_std": 8.789448738098145,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": 9.875062942504883,
      "KL/mean": 7.707754135131836,
      "KL/rejected_KL_mean": 5.540445327758789,
      "KL/std": 7.043632984161377,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.334617614746094,
      "fcm_dpo/q_t": 0.48918962478637695,
      "grad_norm": 15.949915885925293,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": 1.8479840755462646,
      "logits/rejected": 1.8074871301651,
      "logps/chosen": -260.712646484375,
      "logps/ref_chosen": -270.58770751953125,
      "logps/ref_rejected": -238.90069580078125,
      "logps/rejected": -233.36024475097656,
      "loss": 5.3807,
      "margin_dpo/margin_mean": 4.3346171379089355,
      "margin_dpo/margin_std": 7.944525718688965,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": 9.827550888061523,
      "KL/mean": 7.468808650970459,
      "KL/rejected_KL_mean": 5.110067367553711,
      "KL/std": 6.926510334014893,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.717470645904541,
      "fcm_dpo/q_t": 0.48824992775917053,
      "grad_norm": 15.328713417053223,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": 1.9329965114593506,
      "logits/rejected": 1.8231956958770752,
      "logps/chosen": -286.3580322265625,
      "logps/ref_chosen": -296.1855773925781,
      "logps/ref_rejected": -260.35333251953125,
      "logps/rejected": -255.2432861328125,
      "loss": 5.3685,
      "margin_dpo/margin_mean": 4.717471122741699,
      "margin_dpo/margin_std": 9.561058044433594,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": 10.124704360961914,
      "KL/mean": 7.7446160316467285,
      "KL/rejected_KL_mean": 5.364521026611328,
      "KL/std": 7.382896423339844,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.760181427001953,
      "fcm_dpo/q_t": 0.48813796043395996,
      "grad_norm": 15.642975807189941,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": 1.9473803043365479,
      "logits/rejected": 1.9328734874725342,
      "logps/chosen": -267.5906677246094,
      "logps/ref_chosen": -277.7153625488281,
      "logps/ref_rejected": -272.9094543457031,
      "logps/rejected": -267.5449523925781,
      "loss": 5.3678,
      "margin_dpo/margin_mean": 4.760181427001953,
      "margin_dpo/margin_std": 9.506366729736328,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": 10.420711517333984,
      "KL/mean": 7.874325752258301,
      "KL/rejected_KL_mean": 5.327939987182617,
      "KL/std": 7.497433662414551,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.092775344848633,
      "fcm_dpo/q_t": 0.4872967600822449,
      "grad_norm": 15.044392585754395,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": 1.6773171424865723,
      "logits/rejected": 1.6437406539916992,
      "logps/chosen": -262.1170959472656,
      "logps/ref_chosen": -272.5378112792969,
      "logps/ref_rejected": -244.6329345703125,
      "logps/rejected": -239.30499267578125,
      "loss": 5.355,
      "margin_dpo/margin_mean": 5.092775344848633,
      "margin_dpo/margin_std": 9.872742652893066,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": 9.442846298217773,
      "KL/mean": 7.2231550216674805,
      "KL/rejected_KL_mean": 5.003469467163086,
      "KL/std": 7.713043212890625,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.439383506774902,
      "fcm_dpo/q_t": 0.48893654346466064,
      "grad_norm": 15.525213241577148,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.6838476657867432,
      "logits/rejected": 1.605763554573059,
      "logps/chosen": -280.3695068359375,
      "logps/ref_chosen": -289.8123474121094,
      "logps/ref_rejected": -274.29071044921875,
      "logps/rejected": -269.2872314453125,
      "loss": 5.3788,
      "margin_dpo/margin_mean": 4.439383506774902,
      "margin_dpo/margin_std": 8.919550895690918,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": 10.1220703125,
      "KL/mean": 7.86701774597168,
      "KL/rejected_KL_mean": 5.611967086791992,
      "KL/std": 8.03419303894043,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.51009464263916,
      "fcm_dpo/q_t": 0.4887542426586151,
      "grad_norm": 15.34907054901123,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 1.7132959365844727,
      "logits/rejected": 1.8842265605926514,
      "logps/chosen": -235.59104919433594,
      "logps/ref_chosen": -245.71311950683594,
      "logps/ref_rejected": -272.08935546875,
      "logps/rejected": -266.47735595703125,
      "loss": 5.3773,
      "margin_dpo/margin_mean": 4.5100932121276855,
      "margin_dpo/margin_std": 9.633596420288086,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": 10.513242721557617,
      "KL/mean": 7.332424163818359,
      "KL/rejected_KL_mean": 4.151611328125,
      "KL/std": 7.470064163208008,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.361630916595459,
      "fcm_dpo/q_t": 0.4841386079788208,
      "grad_norm": 15.257926940917969,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": 1.6332967281341553,
      "logits/rejected": 1.721139907836914,
      "logps/chosen": -199.42916870117188,
      "logps/ref_chosen": -209.94239807128906,
      "logps/ref_rejected": -230.66897583007812,
      "logps/rejected": -226.51736450195312,
      "loss": 5.3036,
      "margin_dpo/margin_mean": 6.361630916595459,
      "margin_dpo/margin_std": 8.899971961975098,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": 9.583869934082031,
      "KL/mean": 7.220658779144287,
      "KL/rejected_KL_mean": 4.857460021972656,
      "KL/std": 8.2432861328125,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.726404190063477,
      "fcm_dpo/q_t": 0.48822808265686035,
      "grad_norm": 15.752738952636719,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": 1.7019197940826416,
      "logits/rejected": 1.7966352701187134,
      "logps/chosen": -241.71575927734375,
      "logps/ref_chosen": -251.2996368408203,
      "logps/ref_rejected": -277.4847412109375,
      "logps/rejected": -272.6272888183594,
      "loss": 5.3698,
      "margin_dpo/margin_mean": 4.72640323638916,
      "margin_dpo/margin_std": 10.308675765991211,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": 8.475587844848633,
      "KL/mean": 6.281211853027344,
      "KL/rejected_KL_mean": 4.086828231811523,
      "KL/std": 8.48825454711914,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.388754844665527,
      "fcm_dpo/q_t": 0.48907217383384705,
      "grad_norm": 14.367971420288086,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": 1.681663155555725,
      "logits/rejected": 1.7112663984298706,
      "logps/chosen": -249.30381774902344,
      "logps/ref_chosen": -257.7794189453125,
      "logps/ref_rejected": -227.91546630859375,
      "logps/rejected": -223.82864379882812,
      "loss": 5.3844,
      "margin_dpo/margin_mean": 4.3887553215026855,
      "margin_dpo/margin_std": 10.993029594421387,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": 10.35226821899414,
      "KL/mean": 6.418466567993164,
      "KL/rejected_KL_mean": 2.4846725463867188,
      "KL/std": 9.50537109375,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.867586612701416,
      "fcm_dpo/q_t": 0.4803965091705322,
      "grad_norm": 17.97364616394043,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": 1.7604420185089111,
      "logits/rejected": 1.7131434679031372,
      "logps/chosen": -301.74176025390625,
      "logps/ref_chosen": -312.09405517578125,
      "logps/ref_rejected": -277.97332763671875,
      "logps/rejected": -275.4886474609375,
      "loss": 5.249,
      "margin_dpo/margin_mean": 7.867586135864258,
      "margin_dpo/margin_std": 10.548002243041992,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": 6.965463638305664,
      "KL/mean": 4.246558666229248,
      "KL/rejected_KL_mean": 1.5276451110839844,
      "KL/std": 8.774173736572266,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.437828063964844,
      "fcm_dpo/q_t": 0.4864606261253357,
      "grad_norm": 16.357440948486328,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": 1.904369831085205,
      "logits/rejected": 1.934171199798584,
      "logps/chosen": -278.40087890625,
      "logps/ref_chosen": -285.3663330078125,
      "logps/ref_rejected": -262.2779541015625,
      "logps/rejected": -260.75030517578125,
      "loss": 5.3438,
      "margin_dpo/margin_mean": 5.4378275871276855,
      "margin_dpo/margin_std": 10.921525955200195,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": 9.47450065612793,
      "KL/mean": 5.835460186004639,
      "KL/rejected_KL_mean": 2.196420669555664,
      "KL/std": 9.427314758300781,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.278087615966797,
      "fcm_dpo/q_t": 0.4818567633628845,
      "grad_norm": 16.52004051208496,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": 1.7797764539718628,
      "logits/rejected": 1.8473013639450073,
      "logps/chosen": -286.819091796875,
      "logps/ref_chosen": -296.2935791015625,
      "logps/ref_rejected": -291.5162353515625,
      "logps/rejected": -289.3197937011719,
      "loss": 5.2743,
      "margin_dpo/margin_mean": 7.278087139129639,
      "margin_dpo/margin_std": 11.517827987670898,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": 9.619901657104492,
      "KL/mean": 5.611678600311279,
      "KL/rejected_KL_mean": 1.6034469604492188,
      "KL/std": 8.892847061157227,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.01645565032959,
      "fcm_dpo/q_t": 0.4800313115119934,
      "grad_norm": 16.131961822509766,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": 1.8855122327804565,
      "logits/rejected": 1.7514679431915283,
      "logps/chosen": -278.50445556640625,
      "logps/ref_chosen": -288.12432861328125,
      "logps/ref_rejected": -232.35720825195312,
      "logps/rejected": -230.75375366210938,
      "loss": 5.2427,
      "margin_dpo/margin_mean": 8.01645565032959,
      "margin_dpo/margin_std": 10.481178283691406,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": 8.100959777832031,
      "KL/mean": 3.960468292236328,
      "KL/rejected_KL_mean": -0.18002700805664062,
      "KL/std": 10.000768661499023,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.280982971191406,
      "fcm_dpo/q_t": 0.47942808270454407,
      "grad_norm": 17.627527236938477,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": 1.665648102760315,
      "logits/rejected": 1.6078418493270874,
      "logps/chosen": -309.1005859375,
      "logps/ref_chosen": -317.2015686035156,
      "logps/ref_rejected": -278.0653381347656,
      "logps/rejected": -278.245361328125,
      "loss": 5.2386,
      "margin_dpo/margin_mean": 8.28098201751709,
      "margin_dpo/margin_std": 12.672077178955078,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": 8.729087829589844,
      "KL/mean": 4.769931316375732,
      "KL/rejected_KL_mean": 0.8107757568359375,
      "KL/std": 10.00121021270752,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.918315410614014,
      "fcm_dpo/q_t": 0.48029807209968567,
      "grad_norm": 15.943038940429688,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.7329730987548828,
      "logits/rejected": 1.7207520008087158,
      "logps/chosen": -254.21034240722656,
      "logps/ref_chosen": -262.939453125,
      "logps/ref_rejected": -257.82275390625,
      "logps/rejected": -257.011962890625,
      "loss": 5.2527,
      "margin_dpo/margin_mean": 7.918315410614014,
      "margin_dpo/margin_std": 13.056502342224121,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": 9.287803649902344,
      "KL/mean": 4.978600978851318,
      "KL/rejected_KL_mean": 0.6694049835205078,
      "KL/std": 11.125899314880371,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.618398666381836,
      "fcm_dpo/q_t": 0.47858572006225586,
      "grad_norm": 18.24668312072754,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 1.7459826469421387,
      "logits/rejected": 1.646510124206543,
      "logps/chosen": -309.8659973144531,
      "logps/ref_chosen": -319.15380859375,
      "logps/ref_rejected": -281.7733154296875,
      "logps/rejected": -281.1038818359375,
      "loss": 5.2259,
      "margin_dpo/margin_mean": 8.618398666381836,
      "margin_dpo/margin_std": 12.673027992248535,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": 5.487329483032227,
      "KL/mean": 3.366481304168701,
      "KL/rejected_KL_mean": 1.2456378936767578,
      "KL/std": 10.375179290771484,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.241702079772949,
      "fcm_dpo/q_t": 0.48943862318992615,
      "grad_norm": 14.573042869567871,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": 1.793430209159851,
      "logits/rejected": 1.896175503730774,
      "logps/chosen": -245.01307678222656,
      "logps/ref_chosen": -250.50042724609375,
      "logps/ref_rejected": -268.32635498046875,
      "logps/rejected": -267.08074951171875,
      "loss": 5.4002,
      "margin_dpo/margin_mean": 4.241702079772949,
      "margin_dpo/margin_std": 14.080763816833496,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": 6.745866775512695,
      "KL/mean": 1.7845063209533691,
      "KL/rejected_KL_mean": -3.1768417358398438,
      "KL/std": 11.458434104919434,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.92270565032959,
      "fcm_dpo/q_t": 0.47536635398864746,
      "grad_norm": 16.782875061035156,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": 1.6270924806594849,
      "logits/rejected": 1.6651564836502075,
      "logps/chosen": -272.65570068359375,
      "logps/ref_chosen": -279.4015808105469,
      "logps/ref_rejected": -266.6392822265625,
      "logps/rejected": -269.8161315917969,
      "loss": 5.1795,
      "margin_dpo/margin_mean": 9.922706604003906,
      "margin_dpo/margin_std": 14.003193855285645,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": 5.279003143310547,
      "KL/mean": 2.6733222007751465,
      "KL/rejected_KL_mean": 0.06764030456542969,
      "KL/std": 10.563200950622559,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.211361408233643,
      "fcm_dpo/q_t": 0.4870445430278778,
      "grad_norm": 15.553523063659668,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 1.8550914525985718,
      "logits/rejected": 1.848554253578186,
      "logps/chosen": -248.98150634765625,
      "logps/ref_chosen": -254.26048278808594,
      "logps/ref_rejected": -241.8769989013672,
      "logps/rejected": -241.80935668945312,
      "loss": 5.3579,
      "margin_dpo/margin_mean": 5.211361885070801,
      "margin_dpo/margin_std": 12.922776222229004,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": 5.433126449584961,
      "KL/mean": 2.6251416206359863,
      "KL/rejected_KL_mean": -0.1828460693359375,
      "KL/std": 11.73187255859375,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.615973472595215,
      "fcm_dpo/q_t": 0.4860786497592926,
      "grad_norm": 15.891830444335938,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": 1.7106775045394897,
      "logits/rejected": 1.745531439781189,
      "logps/chosen": -283.37139892578125,
      "logps/ref_chosen": -288.80450439453125,
      "logps/ref_rejected": -266.8573913574219,
      "logps/rejected": -267.04022216796875,
      "loss": 5.346,
      "margin_dpo/margin_mean": 5.615972518920898,
      "margin_dpo/margin_std": 14.157711029052734,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": 7.044391632080078,
      "KL/mean": 2.9818310737609863,
      "KL/rejected_KL_mean": -1.0807266235351562,
      "KL/std": 11.09715461730957,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.125114440917969,
      "fcm_dpo/q_t": 0.4798092842102051,
      "grad_norm": 16.371490478515625,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": 1.6580493450164795,
      "logits/rejected": 1.63276207447052,
      "logps/chosen": -265.14556884765625,
      "logps/ref_chosen": -272.18994140625,
      "logps/ref_rejected": -253.14334106445312,
      "logps/rejected": -254.22406005859375,
      "loss": 5.2475,
      "margin_dpo/margin_mean": 8.125115394592285,
      "margin_dpo/margin_std": 13.931629180908203,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": 5.479806900024414,
      "KL/mean": 1.3723936080932617,
      "KL/rejected_KL_mean": -2.735015869140625,
      "KL/std": 10.934646606445312,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.214825630187988,
      "fcm_dpo/q_t": 0.4795783758163452,
      "grad_norm": 15.883464813232422,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": 1.543075442314148,
      "logits/rejected": 1.52162766456604,
      "logps/chosen": -244.9228515625,
      "logps/ref_chosen": -250.4026336669922,
      "logps/ref_rejected": -229.63853454589844,
      "logps/rejected": -232.37355041503906,
      "loss": 5.242,
      "margin_dpo/margin_mean": 8.214825630187988,
      "margin_dpo/margin_std": 12.384005546569824,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": 5.444330215454102,
      "KL/mean": 1.00148606300354,
      "KL/rejected_KL_mean": -3.4413719177246094,
      "KL/std": 12.504544258117676,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.885706901550293,
      "fcm_dpo/q_t": 0.47792646288871765,
      "grad_norm": 20.7720947265625,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": 1.7138844728469849,
      "logits/rejected": 1.760939598083496,
      "logps/chosen": -252.95156860351562,
      "logps/ref_chosen": -258.3959045410156,
      "logps/ref_rejected": -268.84228515625,
      "logps/rejected": -272.2836608886719,
      "loss": 5.2195,
      "margin_dpo/margin_mean": 8.885705947875977,
      "margin_dpo/margin_std": 14.202629089355469,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": 6.0015869140625,
      "KL/mean": 2.232724189758301,
      "KL/rejected_KL_mean": -1.5361442565917969,
      "KL/std": 14.039231300354004,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.53772497177124,
      "fcm_dpo/q_t": 0.48137423396110535,
      "grad_norm": 15.806024551391602,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": 2.0376291275024414,
      "logits/rejected": 2.022418737411499,
      "logps/chosen": -311.30096435546875,
      "logps/ref_chosen": -317.30255126953125,
      "logps/ref_rejected": -300.47552490234375,
      "logps/rejected": -302.0116271972656,
      "loss": 5.282,
      "margin_dpo/margin_mean": 7.537725925445557,
      "margin_dpo/margin_std": 16.670475006103516,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": 6.922492980957031,
      "KL/mean": 2.470975875854492,
      "KL/rejected_KL_mean": -1.9805412292480469,
      "KL/std": 13.192930221557617,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.903035163879395,
      "fcm_dpo/q_t": 0.4779234230518341,
      "grad_norm": 16.1429386138916,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.9966211318969727,
      "logits/rejected": 1.9729186296463013,
      "logps/chosen": -298.3319091796875,
      "logps/ref_chosen": -305.2544250488281,
      "logps/ref_rejected": -259.40716552734375,
      "logps/rejected": -261.3876647949219,
      "loss": 5.2255,
      "margin_dpo/margin_mean": 8.903035163879395,
      "margin_dpo/margin_std": 16.129072189331055,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": 3.533395767211914,
      "KL/mean": 0.8805972933769226,
      "KL/rejected_KL_mean": -1.7722015380859375,
      "KL/std": 11.696654319763184,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.305604934692383,
      "fcm_dpo/q_t": 0.4868319630622864,
      "grad_norm": 14.720951080322266,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 1.6987946033477783,
      "logits/rejected": 1.722078800201416,
      "logps/chosen": -213.10731506347656,
      "logps/ref_chosen": -216.64073181152344,
      "logps/ref_rejected": -224.36422729492188,
      "logps/rejected": -226.13641357421875,
      "loss": 5.3584,
      "margin_dpo/margin_mean": 5.305604457855225,
      "margin_dpo/margin_std": 14.200920104980469,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": 5.770557403564453,
      "KL/mean": 2.0919559001922607,
      "KL/rejected_KL_mean": -1.5866432189941406,
      "KL/std": 13.88086986541748,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.35720157623291,
      "fcm_dpo/q_t": 0.4817720949649811,
      "grad_norm": 15.753201484680176,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": 1.6210639476776123,
      "logits/rejected": 1.6311213970184326,
      "logps/chosen": -263.0482482910156,
      "logps/ref_chosen": -268.8188171386719,
      "logps/ref_rejected": -272.55853271484375,
      "logps/rejected": -274.14520263671875,
      "loss": 5.2889,
      "margin_dpo/margin_mean": 7.357202053070068,
      "margin_dpo/margin_std": 17.223878860473633,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": 8.540637969970703,
      "KL/mean": 3.3083393573760986,
      "KL/rejected_KL_mean": -1.9239463806152344,
      "KL/std": 14.826725006103516,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.464583396911621,
      "fcm_dpo/q_t": 0.4740978479385376,
      "grad_norm": 18.191518783569336,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": 1.7198837995529175,
      "logits/rejected": 1.6990960836410522,
      "logps/chosen": -313.151123046875,
      "logps/ref_chosen": -321.6917419433594,
      "logps/ref_rejected": -279.1517333984375,
      "logps/rejected": -281.0756530761719,
      "loss": 5.1713,
      "margin_dpo/margin_mean": 10.464583396911621,
      "margin_dpo/margin_std": 18.005983352661133,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": 6.116297721862793,
      "KL/mean": 1.370105504989624,
      "KL/rejected_KL_mean": -3.376095771789551,
      "KL/std": 14.339529037475586,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.492399215698242,
      "fcm_dpo/q_t": 0.4764549434185028,
      "grad_norm": 16.534948348999023,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": 1.7621020078659058,
      "logits/rejected": 1.6561591625213623,
      "logps/chosen": -265.55889892578125,
      "logps/ref_chosen": -271.6752014160156,
      "logps/ref_rejected": -221.85577392578125,
      "logps/rejected": -225.23187255859375,
      "loss": 5.204,
      "margin_dpo/margin_mean": 9.492400169372559,
      "margin_dpo/margin_std": 15.887683868408203,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": 5.722043991088867,
      "KL/mean": 1.0211371183395386,
      "KL/rejected_KL_mean": -3.679777145385742,
      "KL/std": 14.023530960083008,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.401817321777344,
      "fcm_dpo/q_t": 0.47676485776901245,
      "grad_norm": 16.297258377075195,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": 1.5109658241271973,
      "logits/rejected": 1.5096803903579712,
      "logps/chosen": -264.677001953125,
      "logps/ref_chosen": -270.3990478515625,
      "logps/ref_rejected": -264.16925048828125,
      "logps/rejected": -267.84906005859375,
      "loss": 5.2136,
      "margin_dpo/margin_mean": 9.401817321777344,
      "margin_dpo/margin_std": 18.426313400268555,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": 6.915754318237305,
      "KL/mean": 1.655871868133545,
      "KL/rejected_KL_mean": -3.604001998901367,
      "KL/std": 15.344995498657227,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.519754409790039,
      "fcm_dpo/q_t": 0.47393113374710083,
      "grad_norm": 16.724918365478516,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": 1.8448987007141113,
      "logits/rejected": 1.8337353467941284,
      "logps/chosen": -306.3355407714844,
      "logps/ref_chosen": -313.25128173828125,
      "logps/ref_rejected": -283.0433349609375,
      "logps/rejected": -286.6473388671875,
      "loss": 5.1708,
      "margin_dpo/margin_mean": 10.519754409790039,
      "margin_dpo/margin_std": 18.007219314575195,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": 8.092655181884766,
      "KL/mean": 1.9844220876693726,
      "KL/rejected_KL_mean": -4.123804092407227,
      "KL/std": 15.954134941101074,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.216462135314941,
      "fcm_dpo/q_t": 0.46977630257606506,
      "grad_norm": 16.80269432067871,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 1.9081001281738281,
      "logits/rejected": 1.8386448621749878,
      "logps/chosen": -290.4713439941406,
      "logps/ref_chosen": -298.5639953613281,
      "logps/ref_rejected": -274.021728515625,
      "logps/rejected": -278.1455383300781,
      "loss": 5.1079,
      "margin_dpo/margin_mean": 12.216462135314941,
      "margin_dpo/margin_std": 18.06784439086914,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": 8.799736022949219,
      "KL/mean": 2.0362606048583984,
      "KL/rejected_KL_mean": -4.727212905883789,
      "KL/std": 16.766979217529297,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.526945114135742,
      "fcm_dpo/q_t": 0.46660423278808594,
      "grad_norm": 16.102502822875977,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": 1.6770257949829102,
      "logits/rejected": 1.714782953262329,
      "logps/chosen": -248.33114624023438,
      "logps/ref_chosen": -257.130859375,
      "logps/ref_rejected": -247.71636962890625,
      "logps/rejected": -252.44357299804688,
      "loss": 5.0648,
      "margin_dpo/margin_mean": 13.526945114135742,
      "margin_dpo/margin_std": 19.881969451904297,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": 9.011581420898438,
      "KL/mean": 2.835731029510498,
      "KL/rejected_KL_mean": -3.340116500854492,
      "KL/std": 16.90859031677246,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.351709365844727,
      "fcm_dpo/q_t": 0.4695398807525635,
      "grad_norm": 16.56525421142578,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": 1.7094236612319946,
      "logits/rejected": 1.755082607269287,
      "logps/chosen": -285.9489440917969,
      "logps/ref_chosen": -294.9605712890625,
      "logps/ref_rejected": -272.6249694824219,
      "logps/rejected": -275.965087890625,
      "loss": 5.1118,
      "margin_dpo/margin_mean": 12.351709365844727,
      "margin_dpo/margin_std": 20.104150772094727,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": 7.416208267211914,
      "KL/mean": 1.6348391771316528,
      "KL/rejected_KL_mean": -4.146547317504883,
      "KL/std": 15.458537101745605,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.562753677368164,
      "fcm_dpo/q_t": 0.4713679552078247,
      "grad_norm": 20.6209774017334,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 1.573374629020691,
      "logits/rejected": 1.6379965543746948,
      "logps/chosen": -269.0260009765625,
      "logps/ref_chosen": -276.4422302246094,
      "logps/ref_rejected": -290.241455078125,
      "logps/rejected": -294.38800048828125,
      "loss": 5.134,
      "margin_dpo/margin_mean": 11.562756538391113,
      "margin_dpo/margin_std": 19.1652774810791,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": 4.79444694519043,
      "KL/mean": 0.016963422298431396,
      "KL/rejected_KL_mean": -4.760524749755859,
      "KL/std": 16.37442398071289,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.554975509643555,
      "fcm_dpo/q_t": 0.47646743059158325,
      "grad_norm": 17.38300323486328,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 1.6690435409545898,
      "logits/rejected": 1.668678641319275,
      "logps/chosen": -300.0302734375,
      "logps/ref_chosen": -304.82470703125,
      "logps/ref_rejected": -289.9726867675781,
      "logps/rejected": -294.73321533203125,
      "loss": 5.2194,
      "margin_dpo/margin_mean": 9.554973602294922,
      "margin_dpo/margin_std": 20.973833084106445,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": 6.961889266967773,
      "KL/mean": 0.48800623416900635,
      "KL/rejected_KL_mean": -5.985872268676758,
      "KL/std": 18.43128776550293,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.947770118713379,
      "fcm_dpo/q_t": 0.468318909406662,
      "grad_norm": 17.931537628173828,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": 1.4748095273971558,
      "logits/rejected": 1.5850160121917725,
      "logps/chosen": -253.35284423828125,
      "logps/ref_chosen": -260.31475830078125,
      "logps/ref_rejected": -280.93756103515625,
      "logps/rejected": -286.9234313964844,
      "loss": 5.1039,
      "margin_dpo/margin_mean": 12.947770118713379,
      "margin_dpo/margin_std": 23.20977020263672,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": 7.158578872680664,
      "KL/mean": 0.4607776403427124,
      "KL/rejected_KL_mean": -6.237020492553711,
      "KL/std": 17.155658721923828,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.395610809326172,
      "fcm_dpo/q_t": 0.466865599155426,
      "grad_norm": 16.065479278564453,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": 1.6457899808883667,
      "logits/rejected": 1.6190307140350342,
      "logps/chosen": -271.9620056152344,
      "logps/ref_chosen": -279.12060546875,
      "logps/ref_rejected": -247.22982788085938,
      "logps/rejected": -253.4668731689453,
      "loss": 5.0765,
      "margin_dpo/margin_mean": 13.395610809326172,
      "margin_dpo/margin_std": 20.97544288635254,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": 1.1817970275878906,
      "KL/mean": -3.4655215740203857,
      "KL/rejected_KL_mean": -8.112831115722656,
      "KL/std": 17.41350746154785,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.294631958007812,
      "fcm_dpo/q_t": 0.4771013855934143,
      "grad_norm": 21.8698673248291,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": 1.814110517501831,
      "logits/rejected": 1.8518030643463135,
      "logps/chosen": -314.42230224609375,
      "logps/ref_chosen": -315.6040954589844,
      "logps/ref_rejected": -291.45635986328125,
      "logps/rejected": -299.5691833496094,
      "loss": 5.2251,
      "margin_dpo/margin_mean": 9.294632911682129,
      "margin_dpo/margin_std": 19.89540672302246,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": 4.847871780395508,
      "KL/mean": -1.739385724067688,
      "KL/rejected_KL_mean": -8.32664680480957,
      "KL/std": 18.00609588623047,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.17452335357666,
      "fcm_dpo/q_t": 0.4674752950668335,
      "grad_norm": 17.052688598632812,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": 1.3778854608535767,
      "logits/rejected": 1.3540982007980347,
      "logps/chosen": -278.0251159667969,
      "logps/ref_chosen": -282.87298583984375,
      "logps/ref_rejected": -265.0682373046875,
      "logps/rejected": -273.39483642578125,
      "loss": 5.0785,
      "margin_dpo/margin_mean": 13.174522399902344,
      "margin_dpo/margin_std": 19.332927703857422,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": 4.17041015625,
      "KL/mean": -2.0563316345214844,
      "KL/rejected_KL_mean": -8.283077239990234,
      "KL/std": 17.5739688873291,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.453495025634766,
      "fcm_dpo/q_t": 0.4693037271499634,
      "grad_norm": 18.788742065429688,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": 1.296678066253662,
      "logits/rejected": 1.454931378364563,
      "logps/chosen": -259.271484375,
      "logps/ref_chosen": -263.4419250488281,
      "logps/ref_rejected": -251.43150329589844,
      "logps/rejected": -259.7145690917969,
      "loss": 5.1143,
      "margin_dpo/margin_mean": 12.453493118286133,
      "margin_dpo/margin_std": 21.758752822875977,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": 2.385854721069336,
      "KL/mean": -3.197941303253174,
      "KL/rejected_KL_mean": -8.781740188598633,
      "KL/std": 19.038970947265625,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.167598724365234,
      "fcm_dpo/q_t": 0.4724290072917938,
      "grad_norm": 18.995481491088867,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": 1.6894562244415283,
      "logits/rejected": 1.6818299293518066,
      "logps/chosen": -292.8848876953125,
      "logps/ref_chosen": -295.2707214355469,
      "logps/ref_rejected": -283.9851989746094,
      "logps/rejected": -292.7669677734375,
      "loss": 5.1573,
      "margin_dpo/margin_mean": 11.167598724365234,
      "margin_dpo/margin_std": 20.438720703125,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": 1.0005474090576172,
      "KL/mean": -4.586702346801758,
      "KL/rejected_KL_mean": -10.173955917358398,
      "KL/std": 18.296493530273438,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.1745023727417,
      "fcm_dpo/q_t": 0.47258931398391724,
      "grad_norm": 16.666549682617188,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": 1.521201491355896,
      "logits/rejected": 1.5141371488571167,
      "logps/chosen": -281.557373046875,
      "logps/ref_chosen": -282.5578918457031,
      "logps/ref_rejected": -262.4772644042969,
      "logps/rejected": -272.6512145996094,
      "loss": 5.1695,
      "margin_dpo/margin_mean": 11.174501419067383,
      "margin_dpo/margin_std": 23.296871185302734,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": 2.346719741821289,
      "KL/mean": -1.7920674085617065,
      "KL/rejected_KL_mean": -5.93086051940918,
      "KL/std": 19.063568115234375,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.277572631835938,
      "fcm_dpo/q_t": 0.4796410799026489,
      "grad_norm": 16.28769874572754,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": 1.6715412139892578,
      "logits/rejected": 1.7466448545455933,
      "logps/chosen": -261.3040771484375,
      "logps/ref_chosen": -263.65081787109375,
      "logps/ref_rejected": -266.2420959472656,
      "logps/rejected": -272.1729736328125,
      "loss": 5.2773,
      "margin_dpo/margin_mean": 8.277573585510254,
      "margin_dpo/margin_std": 23.357677459716797,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": 6.053501129150391,
      "KL/mean": -1.437925100326538,
      "KL/rejected_KL_mean": -8.929346084594727,
      "KL/std": 18.9278507232666,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.982852935791016,
      "fcm_dpo/q_t": 0.46313536167144775,
      "grad_norm": 16.32257843017578,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 1.7108352184295654,
      "logits/rejected": 1.6761181354522705,
      "logps/chosen": -266.3289489746094,
      "logps/ref_chosen": -272.3824462890625,
      "logps/ref_rejected": -248.96444702148438,
      "logps/rejected": -257.893798828125,
      "loss": 5.0181,
      "margin_dpo/margin_mean": 14.982853889465332,
      "margin_dpo/margin_std": 21.298824310302734,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": 3.5351600646972656,
      "KL/mean": -3.6364939212799072,
      "KL/rejected_KL_mean": -10.808130264282227,
      "KL/std": 20.128339767456055,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.343294143676758,
      "fcm_dpo/q_t": 0.46471887826919556,
      "grad_norm": 22.70953369140625,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 1.5816199779510498,
      "logits/rejected": 1.6320933103561401,
      "logps/chosen": -284.535888671875,
      "logps/ref_chosen": -288.071044921875,
      "logps/ref_rejected": -274.83477783203125,
      "logps/rejected": -285.6428527832031,
      "loss": 5.0546,
      "margin_dpo/margin_mean": 14.343294143676758,
      "margin_dpo/margin_std": 24.213302612304688,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": 6.575605392456055,
      "KL/mean": -0.4240683317184448,
      "KL/rejected_KL_mean": -7.423738479614258,
      "KL/std": 19.916584014892578,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.999345779418945,
      "fcm_dpo/q_t": 0.4656396508216858,
      "grad_norm": 18.54561424255371,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": 1.6278738975524902,
      "logits/rejected": 1.5280481576919556,
      "logps/chosen": -282.4162292480469,
      "logps/ref_chosen": -288.9918518066406,
      "logps/ref_rejected": -228.26071166992188,
      "logps/rejected": -235.6844482421875,
      "loss": 5.0677,
      "margin_dpo/margin_mean": 13.999345779418945,
      "margin_dpo/margin_std": 24.013614654541016,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": 5.948600769042969,
      "KL/mean": -2.1315062046051025,
      "KL/rejected_KL_mean": -10.211605072021484,
      "KL/std": 21.11635971069336,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.160198211669922,
      "fcm_dpo/q_t": 0.460438996553421,
      "grad_norm": 18.703840255737305,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": 1.5843019485473633,
      "logits/rejected": 1.6281027793884277,
      "logps/chosen": -283.5258483886719,
      "logps/ref_chosen": -289.4744873046875,
      "logps/ref_rejected": -265.130615234375,
      "logps/rejected": -275.34222412109375,
      "loss": 4.9994,
      "margin_dpo/margin_mean": 16.160198211669922,
      "margin_dpo/margin_std": 26.507389068603516,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": 7.070890426635742,
      "KL/mean": 0.2884737551212311,
      "KL/rejected_KL_mean": -6.493940353393555,
      "KL/std": 20.480661392211914,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.564827919006348,
      "fcm_dpo/q_t": 0.4667653739452362,
      "grad_norm": 19.280683517456055,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": 1.6433237791061401,
      "logits/rejected": 1.62220299243927,
      "logps/chosen": -283.5155029296875,
      "logps/ref_chosen": -290.5863342285156,
      "logps/ref_rejected": -263.41046142578125,
      "logps/rejected": -269.9044189453125,
      "loss": 5.0833,
      "margin_dpo/margin_mean": 13.564827919006348,
      "margin_dpo/margin_std": 23.977294921875,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": 7.034694671630859,
      "KL/mean": -1.6266231536865234,
      "KL/rejected_KL_mean": -10.287935256958008,
      "KL/std": 19.507503509521484,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.322629928588867,
      "fcm_dpo/q_t": 0.45760300755500793,
      "grad_norm": 16.545948028564453,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": 1.539292812347412,
      "logits/rejected": 1.4416896104812622,
      "logps/chosen": -257.50555419921875,
      "logps/ref_chosen": -264.5402526855469,
      "logps/ref_rejected": -214.7958221435547,
      "logps/rejected": -225.08377075195312,
      "loss": 4.9499,
      "margin_dpo/margin_mean": 17.322629928588867,
      "margin_dpo/margin_std": 24.285079956054688,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": 2.9746456146240234,
      "KL/mean": -4.0402326583862305,
      "KL/rejected_KL_mean": -11.055110931396484,
      "KL/std": 19.696561813354492,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.029759407043457,
      "fcm_dpo/q_t": 0.4655352830886841,
      "grad_norm": 27.939193725585938,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": 1.5760091543197632,
      "logits/rejected": 1.699141263961792,
      "logps/chosen": -267.2051086425781,
      "logps/ref_chosen": -270.17974853515625,
      "logps/ref_rejected": -261.2323913574219,
      "logps/rejected": -272.2874755859375,
      "loss": 5.0668,
      "margin_dpo/margin_mean": 14.029760360717773,
      "margin_dpo/margin_std": 23.97341537475586,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": 1.4203720092773438,
      "KL/mean": -4.48462438583374,
      "KL/rejected_KL_mean": -10.389627456665039,
      "KL/std": 22.039331436157227,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.809995651245117,
      "fcm_dpo/q_t": 0.47111135721206665,
      "grad_norm": 17.265356063842773,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": 1.6139085292816162,
      "logits/rejected": 1.5951118469238281,
      "logps/chosen": -281.3602294921875,
      "logps/ref_chosen": -282.7806091308594,
      "logps/ref_rejected": -249.23399353027344,
      "logps/rejected": -259.6236267089844,
      "loss": 5.1672,
      "margin_dpo/margin_mean": 11.809995651245117,
      "margin_dpo/margin_std": 26.488300323486328,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": 1.9170112609863281,
      "KL/mean": -4.272080898284912,
      "KL/rejected_KL_mean": -10.461170196533203,
      "KL/std": 20.815189361572266,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.378179550170898,
      "fcm_dpo/q_t": 0.4697323441505432,
      "grad_norm": 16.34357452392578,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": 1.639525294303894,
      "logits/rejected": 1.6582125425338745,
      "logps/chosen": -227.00030517578125,
      "logps/ref_chosen": -228.9173126220703,
      "logps/ref_rejected": -226.81222534179688,
      "logps/rejected": -237.27340698242188,
      "loss": 5.138,
      "margin_dpo/margin_mean": 12.378179550170898,
      "margin_dpo/margin_std": 26.018535614013672,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": 3.153158187866211,
      "KL/mean": -4.687722682952881,
      "KL/rejected_KL_mean": -12.528594970703125,
      "KL/std": 21.511999130249023,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.681751251220703,
      "fcm_dpo/q_t": 0.4616222083568573,
      "grad_norm": 20.143033981323242,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": 1.6643726825714111,
      "logits/rejected": 1.6386300325393677,
      "logps/chosen": -307.9327697753906,
      "logps/ref_chosen": -311.0859375,
      "logps/ref_rejected": -303.225341796875,
      "logps/rejected": -315.75396728515625,
      "loss": 5.0121,
      "margin_dpo/margin_mean": 15.681751251220703,
      "margin_dpo/margin_std": 25.75830078125,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": 0.0993194580078125,
      "KL/mean": -7.18344783782959,
      "KL/rejected_KL_mean": -14.466217041015625,
      "KL/std": 20.710607528686523,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.56553840637207,
      "fcm_dpo/q_t": 0.46424952149391174,
      "grad_norm": 18.197158813476562,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 1.6023563146591187,
      "logits/rejected": 1.6165797710418701,
      "logps/chosen": -270.3841247558594,
      "logps/ref_chosen": -270.48345947265625,
      "logps/ref_rejected": -269.0491638183594,
      "logps/rejected": -283.5153503417969,
      "loss": 5.0467,
      "margin_dpo/margin_mean": 14.565540313720703,
      "margin_dpo/margin_std": 24.489948272705078,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": 3.364809036254883,
      "KL/mean": -7.773367404937744,
      "KL/rejected_KL_mean": -18.911540985107422,
      "KL/std": 22.851573944091797,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.276344299316406,
      "fcm_dpo/q_t": 0.4456101953983307,
      "grad_norm": 19.114213943481445,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.4876282215118408,
      "logits/rejected": 1.502536416053772,
      "logps/chosen": -284.8009033203125,
      "logps/ref_chosen": -288.16571044921875,
      "logps/ref_rejected": -264.7412414550781,
      "logps/rejected": -283.6528015136719,
      "loss": 4.7865,
      "margin_dpo/margin_mean": 22.27634620666504,
      "margin_dpo/margin_std": 28.32016944885254,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": 1.7856903076171875,
      "KL/mean": -6.674813270568848,
      "KL/rejected_KL_mean": -15.135316848754883,
      "KL/std": 23.108699798583984,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.921005249023438,
      "fcm_dpo/q_t": 0.45864319801330566,
      "grad_norm": 17.70819664001465,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": 1.42415189743042,
      "logits/rejected": 1.5056670904159546,
      "logps/chosen": -258.15283203125,
      "logps/ref_chosen": -259.9385070800781,
      "logps/ref_rejected": -266.2453308105469,
      "logps/rejected": -281.3806457519531,
      "loss": 4.9813,
      "margin_dpo/margin_mean": 16.921005249023438,
      "margin_dpo/margin_std": 27.74669075012207,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": 1.3607120513916016,
      "KL/mean": -10.789017677307129,
      "KL/rejected_KL_mean": -22.938743591308594,
      "KL/std": 24.360065460205078,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.29945945739746,
      "fcm_dpo/q_t": 0.4406447410583496,
      "grad_norm": 22.146242141723633,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 1.5560386180877686,
      "logits/rejected": 1.6215656995773315,
      "logps/chosen": -264.2162780761719,
      "logps/ref_chosen": -265.5769958496094,
      "logps/ref_rejected": -277.6246032714844,
      "logps/rejected": -300.5633544921875,
      "loss": 4.7066,
      "margin_dpo/margin_mean": 24.299457550048828,
      "margin_dpo/margin_std": 26.772855758666992,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": 0.10295677185058594,
      "KL/mean": -8.689861297607422,
      "KL/rejected_KL_mean": -17.482683181762695,
      "KL/std": 25.050600051879883,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.585634231567383,
      "fcm_dpo/q_t": 0.457046777009964,
      "grad_norm": 19.809986114501953,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": 1.5669602155685425,
      "logits/rejected": 1.5485478639602661,
      "logps/chosen": -282.1704406738281,
      "logps/ref_chosen": -282.2734069824219,
      "logps/ref_rejected": -261.5135498046875,
      "logps/rejected": -278.9962463378906,
      "loss": 4.9757,
      "margin_dpo/margin_mean": 17.585634231567383,
      "margin_dpo/margin_std": 31.055919647216797,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -0.9949264526367188,
      "KL/mean": -7.451675891876221,
      "KL/rejected_KL_mean": -13.908428192138672,
      "KL/std": 22.79074478149414,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.913494110107422,
      "fcm_dpo/q_t": 0.4684668481349945,
      "grad_norm": 19.128877639770508,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": 1.4671090841293335,
      "logits/rejected": 1.5624443292617798,
      "logps/chosen": -279.7065124511719,
      "logps/ref_chosen": -278.7115783691406,
      "logps/ref_rejected": -298.0945739746094,
      "logps/rejected": -312.0030212402344,
      "loss": 5.1317,
      "margin_dpo/margin_mean": 12.913494110107422,
      "margin_dpo/margin_std": 28.113332748413086,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": 1.1979255676269531,
      "KL/mean": -9.871267318725586,
      "KL/rejected_KL_mean": -20.940462112426758,
      "KL/std": 25.093881607055664,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.138381958007812,
      "fcm_dpo/q_t": 0.4461987316608429,
      "grad_norm": 18.894607543945312,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": 1.6051300764083862,
      "logits/rejected": 1.6317721605300903,
      "logps/chosen": -267.1531982421875,
      "logps/ref_chosen": -268.35113525390625,
      "logps/ref_rejected": -241.9660186767578,
      "logps/rejected": -262.906494140625,
      "loss": 4.821,
      "margin_dpo/margin_mean": 22.138381958007812,
      "margin_dpo/margin_std": 33.31439971923828,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -0.11229133605957031,
      "KL/mean": -7.8437042236328125,
      "KL/rejected_KL_mean": -15.575122833251953,
      "KL/std": 24.421667098999023,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.462820053100586,
      "fcm_dpo/q_t": 0.462334543466568,
      "grad_norm": 24.27852439880371,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": 1.3614033460617065,
      "logits/rejected": 1.4194438457489014,
      "logps/chosen": -265.51593017578125,
      "logps/ref_chosen": -265.40362548828125,
      "logps/ref_rejected": -257.82440185546875,
      "logps/rejected": -273.3995361328125,
      "loss": 5.0407,
      "margin_dpo/margin_mean": 15.462820053100586,
      "margin_dpo/margin_std": 28.490283966064453,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": 1.0242424011230469,
      "KL/mean": -7.378902435302734,
      "KL/rejected_KL_mean": -15.782047271728516,
      "KL/std": 24.346141815185547,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.80630111694336,
      "fcm_dpo/q_t": 0.4589482247829437,
      "grad_norm": 18.85102081298828,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": 1.3165005445480347,
      "logits/rejected": 1.349800944328308,
      "logps/chosen": -280.1380310058594,
      "logps/ref_chosen": -281.1622619628906,
      "logps/ref_rejected": -268.2899475097656,
      "logps/rejected": -284.0719909667969,
      "loss": 4.9923,
      "margin_dpo/margin_mean": 16.806304931640625,
      "margin_dpo/margin_std": 29.141841888427734,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": 0.0221405029296875,
      "KL/mean": -6.525774002075195,
      "KL/rejected_KL_mean": -13.073686599731445,
      "KL/std": 25.081445693969727,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.095821380615234,
      "fcm_dpo/q_t": 0.4683360457420349,
      "grad_norm": 23.595714569091797,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": 1.7249436378479004,
      "logits/rejected": 1.6356346607208252,
      "logps/chosen": -311.3028564453125,
      "logps/ref_chosen": -311.32501220703125,
      "logps/ref_rejected": -248.85350036621094,
      "logps/rejected": -261.9271545410156,
      "loss": 5.1599,
      "margin_dpo/margin_mean": 13.09582233428955,
      "margin_dpo/margin_std": 33.366676330566406,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": 4.155572891235352,
      "KL/mean": -6.115044593811035,
      "KL/rejected_KL_mean": -16.385665893554688,
      "KL/std": 25.131397247314453,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 20.541231155395508,
      "fcm_dpo/q_t": 0.45019182562828064,
      "grad_norm": 18.712013244628906,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": 1.650706171989441,
      "logits/rejected": 1.6717454195022583,
      "logps/chosen": -262.9533996582031,
      "logps/ref_chosen": -267.10894775390625,
      "logps/ref_rejected": -258.66162109375,
      "logps/rejected": -275.0472717285156,
      "loss": 4.8608,
      "margin_dpo/margin_mean": 20.541229248046875,
      "margin_dpo/margin_std": 29.228294372558594,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": 3.315948486328125,
      "KL/mean": -7.340821743011475,
      "KL/rejected_KL_mean": -17.997577667236328,
      "KL/std": 26.21727180480957,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 21.313528060913086,
      "fcm_dpo/q_t": 0.44835424423217773,
      "grad_norm": 31.393400192260742,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 1.6044011116027832,
      "logits/rejected": 1.4857374429702759,
      "logps/chosen": -298.5764465332031,
      "logps/ref_chosen": -301.89239501953125,
      "logps/ref_rejected": -247.645263671875,
      "logps/rejected": -265.6428527832031,
      "loss": 4.8489,
      "margin_dpo/margin_mean": 21.313528060913086,
      "margin_dpo/margin_std": 32.718753814697266,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": 4.826932907104492,
      "KL/mean": -4.909919261932373,
      "KL/rejected_KL_mean": -14.646768569946289,
      "KL/std": 25.22795867919922,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 19.473691940307617,
      "fcm_dpo/q_t": 0.4525311291217804,
      "grad_norm": 23.004985809326172,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": 1.5941333770751953,
      "logits/rejected": 1.5745580196380615,
      "logps/chosen": -283.4061279296875,
      "logps/ref_chosen": -288.2330627441406,
      "logps/ref_rejected": -252.5148162841797,
      "logps/rejected": -267.1615905761719,
      "loss": 4.9071,
      "margin_dpo/margin_mean": 19.473691940307617,
      "margin_dpo/margin_std": 30.526905059814453,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": 3.2294864654541016,
      "KL/mean": -8.448291778564453,
      "KL/rejected_KL_mean": -20.12607192993164,
      "KL/std": 24.831588745117188,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.355548858642578,
      "fcm_dpo/q_t": 0.4429917335510254,
      "grad_norm": 21.991107940673828,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": 1.6843526363372803,
      "logits/rejected": 1.6737877130508423,
      "logps/chosen": -283.5343322753906,
      "logps/ref_chosen": -286.7638244628906,
      "logps/ref_rejected": -262.8302917480469,
      "logps/rejected": -282.9563293457031,
      "loss": 4.7468,
      "margin_dpo/margin_mean": 23.355548858642578,
      "margin_dpo/margin_std": 27.76906967163086,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -4.772941589355469,
      "KL/mean": -14.609779357910156,
      "KL/rejected_KL_mean": -24.44661521911621,
      "KL/std": 26.633880615234375,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 19.673667907714844,
      "fcm_dpo/q_t": 0.45215681195259094,
      "grad_norm": 20.18552017211914,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": 1.6521281003952026,
      "logits/rejected": 1.606950283050537,
      "logps/chosen": -265.0514221191406,
      "logps/ref_chosen": -260.2784729003906,
      "logps/ref_rejected": -217.10556030273438,
      "logps/rejected": -241.55218505859375,
      "loss": 4.8952,
      "margin_dpo/margin_mean": 19.673669815063477,
      "margin_dpo/margin_std": 30.475914001464844,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -0.7737293243408203,
      "KL/mean": -9.379087448120117,
      "KL/rejected_KL_mean": -17.984455108642578,
      "KL/std": 24.806737899780273,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.210721969604492,
      "fcm_dpo/q_t": 0.4580817222595215,
      "grad_norm": 23.535436630249023,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": 1.7584097385406494,
      "logits/rejected": 1.8106874227523804,
      "logps/chosen": -271.3280334472656,
      "logps/ref_chosen": -270.5543212890625,
      "logps/ref_rejected": -259.80218505859375,
      "logps/rejected": -277.78662109375,
      "loss": 4.9758,
      "margin_dpo/margin_mean": 17.210721969604492,
      "margin_dpo/margin_std": 28.73320960998535,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -3.253591537475586,
      "KL/mean": -10.969488143920898,
      "KL/rejected_KL_mean": -18.685379028320312,
      "KL/std": 27.283424377441406,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.431792259216309,
      "fcm_dpo/q_t": 0.46283063292503357,
      "grad_norm": 18.9977970123291,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 1.6888079643249512,
      "logits/rejected": 1.6851260662078857,
      "logps/chosen": -273.54803466796875,
      "logps/ref_chosen": -270.29443359375,
      "logps/ref_rejected": -230.40151977539062,
      "logps/rejected": -249.08688354492188,
      "loss": 5.0816,
      "margin_dpo/margin_mean": 15.431793212890625,
      "margin_dpo/margin_std": 35.09666442871094,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -3.86016845703125,
      "KL/mean": -13.06332778930664,
      "KL/rejected_KL_mean": -22.266483306884766,
      "KL/std": 26.854843139648438,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 18.406314849853516,
      "fcm_dpo/q_t": 0.45531928539276123,
      "grad_norm": 19.47430419921875,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": 1.6576471328735352,
      "logits/rejected": 1.614959716796875,
      "logps/chosen": -277.3207702636719,
      "logps/ref_chosen": -273.4605712890625,
      "logps/ref_rejected": -248.86997985839844,
      "logps/rejected": -271.136474609375,
      "loss": 4.9522,
      "margin_dpo/margin_mean": 18.40631675720215,
      "margin_dpo/margin_std": 32.23647689819336,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": 0.1979961395263672,
      "KL/mean": -11.68295669555664,
      "KL/rejected_KL_mean": -23.563915252685547,
      "KL/std": 28.871519088745117,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.76191520690918,
      "fcm_dpo/q_t": 0.44233226776123047,
      "grad_norm": 20.828195571899414,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": 1.5562409162521362,
      "logits/rejected": 1.552800178527832,
      "logps/chosen": -305.5267333984375,
      "logps/ref_chosen": -305.7247009277344,
      "logps/ref_rejected": -290.16650390625,
      "logps/rejected": -313.7304382324219,
      "loss": 4.7607,
      "margin_dpo/margin_mean": 23.761913299560547,
      "margin_dpo/margin_std": 31.44522476196289,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -0.6658859252929688,
      "KL/mean": -14.412221908569336,
      "KL/rejected_KL_mean": -28.158554077148438,
      "KL/std": 28.874914169311523,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.009983214549720287,
      "fcm_dpo/delta": -0.01689843460917473,
      "fcm_dpo/margin": 27.492666244506836,
      "fcm_dpo/q_t": 0.43380409479141235,
      "grad_norm": 18.64201545715332,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": 1.4434306621551514,
      "logits/rejected": 1.5030688047409058,
      "logps/chosen": -252.83616638183594,
      "logps/ref_chosen": -252.1702880859375,
      "logps/ref_rejected": -233.34750366210938,
      "logps/rejected": -261.50604248046875,
      "loss": 4.643,
      "margin_dpo/margin_mean": 27.492664337158203,
      "margin_dpo/margin_std": 32.53924560546875,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": 0.8674793243408203,
      "KL/mean": -11.202190399169922,
      "KL/rejected_KL_mean": -23.271852493286133,
      "KL/std": 27.436424255371094,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.13933563232422,
      "fcm_dpo/q_t": 0.44238701462745667,
      "grad_norm": 20.811071395874023,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": 1.5320645570755005,
      "logits/rejected": 1.5796838998794556,
      "logps/chosen": -301.61090087890625,
      "logps/ref_chosen": -302.4783935546875,
      "logps/ref_rejected": -259.40777587890625,
      "logps/rejected": -282.67962646484375,
      "loss": 4.7523,
      "margin_dpo/margin_mean": 24.139333724975586,
      "margin_dpo/margin_std": 32.58830261230469,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -0.09467124938964844,
      "KL/mean": -12.243462562561035,
      "KL/rejected_KL_mean": -24.392248153686523,
      "KL/std": 28.17013931274414,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.29757308959961,
      "fcm_dpo/q_t": 0.44209718704223633,
      "grad_norm": 23.26610565185547,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 1.4128228425979614,
      "logits/rejected": 1.5241916179656982,
      "logps/chosen": -250.74136352539062,
      "logps/ref_chosen": -250.646728515625,
      "logps/ref_rejected": -259.1659851074219,
      "logps/rejected": -283.5582275390625,
      "loss": 4.7526,
      "margin_dpo/margin_mean": 24.29757308959961,
      "margin_dpo/margin_std": 31.918514251708984,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": 0.6183185577392578,
      "KL/mean": -7.677464008331299,
      "KL/rejected_KL_mean": -15.973255157470703,
      "KL/std": 28.99441909790039,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.591552734375,
      "fcm_dpo/q_t": 0.4600493907928467,
      "grad_norm": 23.06460189819336,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": 1.9057728052139282,
      "logits/rejected": 1.9437506198883057,
      "logps/chosen": -328.4066162109375,
      "logps/ref_chosen": -329.02490234375,
      "logps/ref_rejected": -293.0215759277344,
      "logps/rejected": -308.99481201171875,
      "loss": 5.0373,
      "margin_dpo/margin_mean": 16.591552734375,
      "margin_dpo/margin_std": 33.967811584472656,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -1.12176513671875,
      "KL/mean": -11.361809730529785,
      "KL/rejected_KL_mean": -21.60186004638672,
      "KL/std": 28.984773635864258,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 20.480083465576172,
      "fcm_dpo/q_t": 0.4511392116546631,
      "grad_norm": 20.0596923828125,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": 1.4742934703826904,
      "logits/rejected": 1.442458152770996,
      "logps/chosen": -291.26239013671875,
      "logps/ref_chosen": -290.140625,
      "logps/ref_rejected": -271.474853515625,
      "logps/rejected": -293.0766906738281,
      "loss": 4.8907,
      "margin_dpo/margin_mean": 20.480083465576172,
      "margin_dpo/margin_std": 32.66584396362305,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -0.11094856262207031,
      "KL/mean": -13.164121627807617,
      "KL/rejected_KL_mean": -26.217281341552734,
      "KL/std": 28.53524398803711,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 26.106342315673828,
      "fcm_dpo/q_t": 0.4378606379032135,
      "grad_norm": 25.08749771118164,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": 1.5505139827728271,
      "logits/rejected": 1.4720882177352905,
      "logps/chosen": -329.418212890625,
      "logps/ref_chosen": -329.3072509765625,
      "logps/ref_rejected": -256.6839599609375,
      "logps/rejected": -282.9012451171875,
      "loss": 4.689,
      "margin_dpo/margin_mean": 26.106342315673828,
      "margin_dpo/margin_std": 32.165592193603516,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -3.353670120239258,
      "KL/mean": -13.670148849487305,
      "KL/rejected_KL_mean": -23.986631393432617,
      "KL/std": 29.47393035888672,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 20.63296127319336,
      "fcm_dpo/q_t": 0.45061200857162476,
      "grad_norm": 31.995128631591797,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": 1.5911957025527954,
      "logits/rejected": 1.5842115879058838,
      "logps/chosen": -279.2559814453125,
      "logps/ref_chosen": -275.9023132324219,
      "logps/ref_rejected": -281.4163513183594,
      "logps/rejected": -305.4029846191406,
      "loss": 4.9003,
      "margin_dpo/margin_mean": 20.63296127319336,
      "margin_dpo/margin_std": 35.473960876464844,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -4.921613693237305,
      "KL/mean": -12.749994277954102,
      "KL/rejected_KL_mean": -20.578369140625,
      "KL/std": 29.14145278930664,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.656757354736328,
      "fcm_dpo/q_t": 0.4624548554420471,
      "grad_norm": 25.854082107543945,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": 1.4492413997650146,
      "logits/rejected": 1.512561321258545,
      "logps/chosen": -248.48583984375,
      "logps/ref_chosen": -243.56423950195312,
      "logps/ref_rejected": -239.12905883789062,
      "logps/rejected": -259.7074279785156,
      "loss": 5.0925,
      "margin_dpo/margin_mean": 15.656758308410645,
      "margin_dpo/margin_std": 35.827396392822266,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -6.214336395263672,
      "KL/mean": -13.698237419128418,
      "KL/rejected_KL_mean": -21.182147979736328,
      "KL/std": 30.196779251098633,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.96782112121582,
      "fcm_dpo/q_t": 0.46427083015441895,
      "grad_norm": 31.362546920776367,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": 1.6652681827545166,
      "logits/rejected": 1.592973232269287,
      "logps/chosen": -299.51800537109375,
      "logps/ref_chosen": -293.30364990234375,
      "logps/ref_rejected": -292.293701171875,
      "logps/rejected": -313.475830078125,
      "loss": 5.1475,
      "margin_dpo/margin_mean": 14.967822074890137,
      "margin_dpo/margin_std": 40.00969696044922,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -1.5465583801269531,
      "KL/mean": -13.978608131408691,
      "KL/rejected_KL_mean": -26.410654067993164,
      "KL/std": 31.933210372924805,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.864099502563477,
      "fcm_dpo/q_t": 0.44132328033447266,
      "grad_norm": 20.848419189453125,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": 1.6186931133270264,
      "logits/rejected": 1.6432543992996216,
      "logps/chosen": -277.67279052734375,
      "logps/ref_chosen": -276.126220703125,
      "logps/ref_rejected": -240.31179809570312,
      "logps/rejected": -266.72247314453125,
      "loss": 4.773,
      "margin_dpo/margin_mean": 24.86409568786621,
      "margin_dpo/margin_std": 38.41860580444336,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -1.8187637329101562,
      "KL/mean": -11.092345237731934,
      "KL/rejected_KL_mean": -20.36593246459961,
      "KL/std": 30.71295738220215,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 18.547176361083984,
      "fcm_dpo/q_t": 0.45589926838874817,
      "grad_norm": 38.345550537109375,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 1.3273789882659912,
      "logits/rejected": 1.4418869018554688,
      "logps/chosen": -251.6811981201172,
      "logps/ref_chosen": -249.8624267578125,
      "logps/ref_rejected": -271.18603515625,
      "logps/rejected": -291.552001953125,
      "loss": 4.9848,
      "margin_dpo/margin_mean": 18.547176361083984,
      "margin_dpo/margin_std": 36.01925277709961,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -2.238039016723633,
      "KL/mean": -14.277326583862305,
      "KL/rejected_KL_mean": -26.31661605834961,
      "KL/std": 30.347381591796875,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.078584671020508,
      "fcm_dpo/q_t": 0.44301968812942505,
      "grad_norm": 23.017675399780273,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": 1.4853309392929077,
      "logits/rejected": 1.5193896293640137,
      "logps/chosen": -262.90716552734375,
      "logps/ref_chosen": -260.66912841796875,
      "logps/ref_rejected": -229.25491333007812,
      "logps/rejected": -255.571533203125,
      "loss": 4.7841,
      "margin_dpo/margin_mean": 24.07858657836914,
      "margin_dpo/margin_std": 35.94736099243164,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -1.4621315002441406,
      "KL/mean": -12.871164321899414,
      "KL/rejected_KL_mean": -24.280193328857422,
      "KL/std": 31.23923683166504,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.818065643310547,
      "fcm_dpo/q_t": 0.44591256976127625,
      "grad_norm": 25.269298553466797,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.7281948328018188,
      "logits/rejected": 1.7212491035461426,
      "logps/chosen": -292.3507080078125,
      "logps/ref_chosen": -290.8885803222656,
      "logps/ref_rejected": -275.1922302246094,
      "logps/rejected": -299.472412109375,
      "loss": 4.8516,
      "margin_dpo/margin_mean": 22.818065643310547,
      "margin_dpo/margin_std": 38.54887008666992,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -3.5140151977539062,
      "KL/mean": -17.96072006225586,
      "KL/rejected_KL_mean": -32.40742492675781,
      "KL/std": 31.149383544921875,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.893415451049805,
      "fcm_dpo/q_t": 0.4309816360473633,
      "grad_norm": 31.725637435913086,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": 1.3367310762405396,
      "logits/rejected": 1.342888593673706,
      "logps/chosen": -274.08087158203125,
      "logps/ref_chosen": -270.56683349609375,
      "logps/ref_rejected": -260.3321533203125,
      "logps/rejected": -292.73956298828125,
      "loss": 4.5981,
      "margin_dpo/margin_mean": 28.893413543701172,
      "margin_dpo/margin_std": 33.13063049316406,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -4.117700576782227,
      "KL/mean": -16.195959091186523,
      "KL/rejected_KL_mean": -28.27422332763672,
      "KL/std": 31.902973175048828,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.009865722618997097,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.156522750854492,
      "fcm_dpo/q_t": 0.4427652955055237,
      "grad_norm": 21.22297477722168,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": 1.5961772203445435,
      "logits/rejected": 1.6451501846313477,
      "logps/chosen": -265.14569091796875,
      "logps/ref_chosen": -261.0279846191406,
      "logps/ref_rejected": -240.72361755371094,
      "logps/rejected": -268.99786376953125,
      "loss": 4.8007,
      "margin_dpo/margin_mean": 24.156522750854492,
      "margin_dpo/margin_std": 39.269142150878906,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -0.5218009948730469,
      "KL/mean": -18.80294418334961,
      "KL/rejected_KL_mean": -37.084083557128906,
      "KL/std": 35.9709587097168,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.009652999229729176,
      "fcm_dpo/delta": -0.032799068838357925,
      "fcm_dpo/margin": 36.56228256225586,
      "fcm_dpo/q_t": 0.41549715399742126,
      "grad_norm": 44.57929611206055,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": 1.613839864730835,
      "logits/rejected": 1.6021215915679932,
      "logps/chosen": -309.60565185546875,
      "logps/ref_chosen": -309.0838317871094,
      "logps/ref_rejected": -263.6435241699219,
      "logps/rejected": -300.72760009765625,
      "loss": 4.4343,
      "margin_dpo/margin_mean": 36.56228256225586,
      "margin_dpo/margin_std": 43.27032470703125,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -9.07380485534668,
      "KL/mean": -20.76129722595215,
      "KL/rejected_KL_mean": -32.44879150390625,
      "KL/std": 33.65535354614258,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.009523202665150166,
      "fcm_dpo/delta": -0.018241863697767258,
      "fcm_dpo/margin": 23.374984741210938,
      "fcm_dpo/q_t": 0.44654351472854614,
      "grad_norm": 23.11496353149414,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": 1.4531886577606201,
      "logits/rejected": 1.5255351066589355,
      "logps/chosen": -274.7755126953125,
      "logps/ref_chosen": -265.70172119140625,
      "logps/ref_rejected": -257.5074157714844,
      "logps/rejected": -289.9561767578125,
      "loss": 4.8593,
      "margin_dpo/margin_mean": 23.374984741210938,
      "margin_dpo/margin_std": 39.85984420776367,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -12.68564224243164,
      "KL/mean": -24.057937622070312,
      "KL/rejected_KL_mean": -35.43023681640625,
      "KL/std": 32.16779708862305,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.744590759277344,
      "fcm_dpo/q_t": 0.448189377784729,
      "grad_norm": 33.90011978149414,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": 1.5855448246002197,
      "logits/rejected": 1.6185015439987183,
      "logps/chosen": -304.4106140136719,
      "logps/ref_chosen": -291.7249755859375,
      "logps/ref_rejected": -273.54010009765625,
      "logps/rejected": -308.9703674316406,
      "loss": 4.8605,
      "margin_dpo/margin_mean": 22.744592666625977,
      "margin_dpo/margin_std": 36.85415267944336,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -13.143836975097656,
      "KL/mean": -24.553598403930664,
      "KL/rejected_KL_mean": -35.96336364746094,
      "KL/std": 34.341915130615234,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.819522857666016,
      "fcm_dpo/q_t": 0.4485433101654053,
      "grad_norm": 23.94179916381836,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": 1.6123979091644287,
      "logits/rejected": 1.6502329111099243,
      "logps/chosen": -334.845703125,
      "logps/ref_chosen": -321.70184326171875,
      "logps/ref_rejected": -261.8386535644531,
      "logps/rejected": -297.80206298828125,
      "loss": 4.9048,
      "margin_dpo/margin_mean": 22.819522857666016,
      "margin_dpo/margin_std": 44.25890350341797,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -15.942193984985352,
      "KL/mean": -27.843666076660156,
      "KL/rejected_KL_mean": -39.745147705078125,
      "KL/std": 33.462158203125,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.80295753479004,
      "fcm_dpo/q_t": 0.4458545446395874,
      "grad_norm": 52.452449798583984,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": 1.4869827032089233,
      "logits/rejected": 1.46756911277771,
      "logps/chosen": -271.85760498046875,
      "logps/ref_chosen": -255.91542053222656,
      "logps/ref_rejected": -245.1119842529297,
      "logps/rejected": -284.85711669921875,
      "loss": 4.8697,
      "margin_dpo/margin_mean": 23.80295753479004,
      "margin_dpo/margin_std": 43.12034225463867,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -8.293607711791992,
      "KL/mean": -22.914295196533203,
      "KL/rejected_KL_mean": -37.53498458862305,
      "KL/std": 34.214263916015625,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.241382598876953,
      "fcm_dpo/q_t": 0.4333791434764862,
      "grad_norm": 30.644752502441406,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": 1.5990720987319946,
      "logits/rejected": 1.6342136859893799,
      "logps/chosen": -274.3121643066406,
      "logps/ref_chosen": -266.0185546875,
      "logps/ref_rejected": -272.862548828125,
      "logps/rejected": -310.39752197265625,
      "loss": 4.642,
      "margin_dpo/margin_mean": 29.241382598876953,
      "margin_dpo/margin_std": 36.42220687866211,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -11.009340286254883,
      "KL/mean": -24.5858154296875,
      "KL/rejected_KL_mean": -38.16228485107422,
      "KL/std": 34.10367202758789,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.1529541015625,
      "fcm_dpo/q_t": 0.4382346272468567,
      "grad_norm": 37.02184295654297,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 1.2953547239303589,
      "logits/rejected": 1.2213633060455322,
      "logps/chosen": -261.1603088378906,
      "logps/ref_chosen": -250.15098571777344,
      "logps/ref_rejected": -223.34393310546875,
      "logps/rejected": -261.5062255859375,
      "loss": 4.748,
      "margin_dpo/margin_mean": 27.152950286865234,
      "margin_dpo/margin_std": 42.05812072753906,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -7.445882797241211,
      "KL/mean": -20.652345657348633,
      "KL/rejected_KL_mean": -33.858802795410156,
      "KL/std": 35.41230010986328,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 26.41290855407715,
      "fcm_dpo/q_t": 0.44032177329063416,
      "grad_norm": 26.261754989624023,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 1.485849380493164,
      "logits/rejected": 1.5585010051727295,
      "logps/chosen": -245.83767700195312,
      "logps/ref_chosen": -238.3917999267578,
      "logps/ref_rejected": -243.70272827148438,
      "logps/rejected": -277.5615234375,
      "loss": 4.7899,
      "margin_dpo/margin_mean": 26.41290855407715,
      "margin_dpo/margin_std": 44.361839294433594,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -13.340320587158203,
      "KL/mean": -23.92660140991211,
      "KL/rejected_KL_mean": -34.51287841796875,
      "KL/std": 35.099281311035156,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 21.17256736755371,
      "fcm_dpo/q_t": 0.45186272263526917,
      "grad_norm": 24.32538414001465,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.6161268949508667,
      "logits/rejected": 1.7023253440856934,
      "logps/chosen": -269.977294921875,
      "logps/ref_chosen": -256.6369934082031,
      "logps/ref_rejected": -256.8894348144531,
      "logps/rejected": -291.40228271484375,
      "loss": 4.985,
      "margin_dpo/margin_mean": 21.172569274902344,
      "margin_dpo/margin_std": 45.12506103515625,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -14.381448745727539,
      "KL/mean": -26.00305938720703,
      "KL/rejected_KL_mean": -37.624656677246094,
      "KL/std": 36.72563171386719,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.243213653564453,
      "fcm_dpo/q_t": 0.44732022285461426,
      "grad_norm": 28.76276206970215,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": 1.5013363361358643,
      "logits/rejected": 1.5983829498291016,
      "logps/chosen": -287.18109130859375,
      "logps/ref_chosen": -272.7996826171875,
      "logps/ref_rejected": -289.2391052246094,
      "logps/rejected": -326.86376953125,
      "loss": 4.8775,
      "margin_dpo/margin_mean": 23.243213653564453,
      "margin_dpo/margin_std": 42.533485412597656,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -12.219499588012695,
      "KL/mean": -21.26104736328125,
      "KL/rejected_KL_mean": -30.302593231201172,
      "KL/std": 36.64418411254883,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 18.083087921142578,
      "fcm_dpo/q_t": 0.4592794179916382,
      "grad_norm": 34.13716506958008,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": 1.198574423789978,
      "logits/rejected": 1.2307274341583252,
      "logps/chosen": -293.5285339355469,
      "logps/ref_chosen": -281.30902099609375,
      "logps/ref_rejected": -246.3243408203125,
      "logps/rejected": -276.6269836425781,
      "loss": 5.0517,
      "margin_dpo/margin_mean": 18.083086013793945,
      "margin_dpo/margin_std": 41.23551940917969,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -12.666908264160156,
      "KL/mean": -21.461347579956055,
      "KL/rejected_KL_mean": -30.255786895751953,
      "KL/std": 36.29999542236328,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.009470991790294647,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.588884353637695,
      "fcm_dpo/q_t": 0.46019425988197327,
      "grad_norm": 30.616416931152344,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": 1.339608073234558,
      "logits/rejected": 1.4290828704833984,
      "logps/chosen": -279.5250244140625,
      "logps/ref_chosen": -266.8581237792969,
      "logps/ref_rejected": -276.0855712890625,
      "logps/rejected": -306.3413391113281,
      "loss": 5.0841,
      "margin_dpo/margin_mean": 17.58888816833496,
      "margin_dpo/margin_std": 43.61368179321289,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -2.591215133666992,
      "KL/mean": -17.346506118774414,
      "KL/rejected_KL_mean": -32.1017951965332,
      "KL/std": 34.620361328125,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.009399140253663063,
      "fcm_dpo/delta": -0.015265772119164467,
      "fcm_dpo/margin": 29.51057243347168,
      "fcm_dpo/q_t": 0.4334163963794708,
      "grad_norm": 23.41695785522461,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": 1.3335576057434082,
      "logits/rejected": 1.4389801025390625,
      "logps/chosen": -269.5861511230469,
      "logps/ref_chosen": -266.99493408203125,
      "logps/ref_rejected": -248.60513305664062,
      "logps/rejected": -280.7069396972656,
      "loss": 4.6527,
      "margin_dpo/margin_mean": 29.51057243347168,
      "margin_dpo/margin_std": 38.5134391784668,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -0.35141563415527344,
      "KL/mean": -19.2672119140625,
      "KL/rejected_KL_mean": -38.183013916015625,
      "KL/std": 37.862403869628906,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.009242142550647259,
      "fcm_dpo/delta": -0.0481281653046608,
      "fcm_dpo/margin": 37.831600189208984,
      "fcm_dpo/q_t": 0.4161595106124878,
      "grad_norm": 25.475967407226562,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": 1.3487699031829834,
      "logits/rejected": 1.3164831399917603,
      "logps/chosen": -303.3412170410156,
      "logps/ref_chosen": -302.98980712890625,
      "logps/ref_rejected": -262.8002624511719,
      "logps/rejected": -300.9832763671875,
      "loss": 4.4311,
      "margin_dpo/margin_mean": 37.831600189208984,
      "margin_dpo/margin_std": 41.24234390258789,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -2.2183990478515625,
      "KL/mean": -16.39239501953125,
      "KL/rejected_KL_mean": -30.566396713256836,
      "KL/std": 37.56085968017578,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.009002645500004292,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.347990036010742,
      "fcm_dpo/q_t": 0.43903133273124695,
      "grad_norm": 24.78966522216797,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": 1.323146939277649,
      "logits/rejected": 1.4067717790603638,
      "logps/chosen": -298.4302062988281,
      "logps/ref_chosen": -296.2117614746094,
      "logps/ref_rejected": -289.8357238769531,
      "logps/rejected": -320.402099609375,
      "loss": 4.7855,
      "margin_dpo/margin_mean": 28.347990036010742,
      "margin_dpo/margin_std": 48.070430755615234,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -2.362947463989258,
      "KL/mean": -16.319074630737305,
      "KL/rejected_KL_mean": -30.27519416809082,
      "KL/std": 35.402198791503906,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.009002645500004292,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.91224479675293,
      "fcm_dpo/q_t": 0.4397514760494232,
      "grad_norm": 30.30669593811035,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": 1.391531229019165,
      "logits/rejected": 1.3360536098480225,
      "logps/chosen": -257.6556091308594,
      "logps/ref_chosen": -255.29263305664062,
      "logps/ref_rejected": -212.69761657714844,
      "logps/rejected": -242.97283935546875,
      "loss": 4.7607,
      "margin_dpo/margin_mean": 27.912246704101562,
      "margin_dpo/margin_std": 43.625221252441406,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -1.3098831176757812,
      "KL/mean": -13.315980911254883,
      "KL/rejected_KL_mean": -25.32207679748535,
      "KL/std": 36.7401123046875,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.009002645500004292,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.0122013092041,
      "fcm_dpo/q_t": 0.4487912654876709,
      "grad_norm": 20.80882453918457,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 1.7209532260894775,
      "logits/rejected": 1.727477788925171,
      "logps/chosen": -314.8463134765625,
      "logps/ref_chosen": -313.5364074707031,
      "logps/ref_rejected": -274.349609375,
      "logps/rejected": -299.671630859375,
      "loss": 4.9448,
      "margin_dpo/margin_mean": 24.0122013092041,
      "margin_dpo/margin_std": 51.121620178222656,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_KL/chosen_KL_mean": -5.696249961853027,
      "eval_KL/mean": -18.915620803833008,
      "eval_KL/rejected_KL_mean": -32.13499069213867,
      "eval_KL/std": 36.57339859008789,
      "eval_fcm_dpo/beta": 0.009002645500004292,
      "eval_logits/chosen": 1.682607650756836,
      "eval_logits/rejected": 1.7371432781219482,
      "eval_logps/chosen": -286.11297607421875,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -296.7804870605469,
      "eval_loss": 0.6026569604873657,
      "eval_margin_dpo/margin_mean": 26.43874168395996,
      "eval_margin_dpo/margin_std": 42.3830451965332,
      "eval_runtime": 87.7477,
      "eval_samples_per_second": 22.793,
      "eval_steps_per_second": 2.849,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -3.9227066040039062,
      "KL/mean": -18.352291107177734,
      "KL/rejected_KL_mean": -32.78187561035156,
      "KL/std": 37.574405670166016,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.008983886800706387,
      "fcm_dpo/delta": -0.010461856611073017,
      "fcm_dpo/margin": 28.859167098999023,
      "fcm_dpo/q_t": 0.43771448731422424,
      "grad_norm": 29.29420280456543,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 1.4531577825546265,
      "logits/rejected": 1.5046379566192627,
      "logps/chosen": -266.6299133300781,
      "logps/ref_chosen": -262.7071838378906,
      "logps/ref_rejected": -243.85879516601562,
      "logps/rejected": -276.6406555175781,
      "loss": 4.7265,
      "margin_dpo/margin_mean": 28.85916519165039,
      "margin_dpo/margin_std": 42.03227615356445,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -5.880809783935547,
      "KL/mean": -19.895978927612305,
      "KL/rejected_KL_mean": -33.911155700683594,
      "KL/std": 37.81553649902344,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.008927612565457821,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.030345916748047,
      "fcm_dpo/q_t": 0.44046592712402344,
      "grad_norm": 23.350282669067383,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": 1.6060550212860107,
      "logits/rejected": 1.5928857326507568,
      "logps/chosen": -270.16754150390625,
      "logps/ref_chosen": -264.2867431640625,
      "logps/ref_rejected": -231.6894989013672,
      "logps/rejected": -265.60064697265625,
      "loss": 4.8024,
      "margin_dpo/margin_mean": 28.030345916748047,
      "margin_dpo/margin_std": 48.10984420776367,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -6.533029556274414,
      "KL/mean": -17.309005737304688,
      "KL/rejected_KL_mean": -28.084976196289062,
      "KL/std": 35.95152282714844,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.008927612565457821,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 21.551944732666016,
      "fcm_dpo/q_t": 0.45352643728256226,
      "grad_norm": 36.110286712646484,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": 1.6227681636810303,
      "logits/rejected": 1.6044498682022095,
      "logps/chosen": -277.8172302246094,
      "logps/ref_chosen": -271.2842102050781,
      "logps/ref_rejected": -247.95462036132812,
      "logps/rejected": -276.03961181640625,
      "loss": 4.9286,
      "margin_dpo/margin_mean": 21.551944732666016,
      "margin_dpo/margin_std": 36.89060592651367,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -3.47100830078125,
      "KL/mean": -19.42017936706543,
      "KL/rejected_KL_mean": -35.36934280395508,
      "KL/std": 37.0676383972168,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.008927612565457821,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.89834213256836,
      "fcm_dpo/q_t": 0.4319766163825989,
      "grad_norm": 30.403507232666016,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": 1.4853579998016357,
      "logits/rejected": 1.4535101652145386,
      "logps/chosen": -313.6014099121094,
      "logps/ref_chosen": -310.13043212890625,
      "logps/ref_rejected": -267.3905334472656,
      "logps/rejected": -302.75982666015625,
      "loss": 4.637,
      "margin_dpo/margin_mean": 31.89834213256836,
      "margin_dpo/margin_std": 43.56296157836914,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -0.7190723419189453,
      "KL/mean": -15.949538230895996,
      "KL/rejected_KL_mean": -31.180002212524414,
      "KL/std": 36.703033447265625,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.008903943933546543,
      "fcm_dpo/delta": -0.013326846063137054,
      "fcm_dpo/margin": 30.460941314697266,
      "fcm_dpo/q_t": 0.43502864241600037,
      "grad_norm": 20.76314926147461,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 1.3922882080078125,
      "logits/rejected": 1.3918850421905518,
      "logps/chosen": -277.8797607421875,
      "logps/ref_chosen": -277.16064453125,
      "logps/ref_rejected": -292.91815185546875,
      "logps/rejected": -324.09820556640625,
      "loss": 4.6986,
      "margin_dpo/margin_mean": 30.4609432220459,
      "margin_dpo/margin_std": 43.59751892089844,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -0.5813255310058594,
      "KL/mean": -14.911848068237305,
      "KL/rejected_KL_mean": -29.242372512817383,
      "KL/std": 36.506736755371094,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.00872974842786789,
      "fcm_dpo/delta": -0.016801370307803154,
      "fcm_dpo/margin": 28.661043167114258,
      "fcm_dpo/q_t": 0.4396204948425293,
      "grad_norm": 25.91968536376953,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": 1.4057502746582031,
      "logits/rejected": 1.5115894079208374,
      "logps/chosen": -287.3639221191406,
      "logps/ref_chosen": -286.7825927734375,
      "logps/ref_rejected": -289.08270263671875,
      "logps/rejected": -318.3251037597656,
      "loss": 4.7485,
      "margin_dpo/margin_mean": 28.661043167114258,
      "margin_dpo/margin_std": 41.56435775756836,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -5.12275505065918,
      "KL/mean": -17.36612319946289,
      "KL/rejected_KL_mean": -29.6094970703125,
      "KL/std": 40.01288986206055,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.008715006522834301,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.486732482910156,
      "fcm_dpo/q_t": 0.4494379460811615,
      "grad_norm": 28.719999313354492,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": 1.563735008239746,
      "logits/rejected": 1.6678171157836914,
      "logps/chosen": -271.6549987792969,
      "logps/ref_chosen": -266.5322570800781,
      "logps/ref_rejected": -275.20672607421875,
      "logps/rejected": -304.81622314453125,
      "loss": 4.9494,
      "margin_dpo/margin_mean": 24.486732482910156,
      "margin_dpo/margin_std": 51.69967269897461,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -4.845132827758789,
      "KL/mean": -18.717267990112305,
      "KL/rejected_KL_mean": -32.58940887451172,
      "KL/std": 39.16844177246094,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.008715006522834301,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.744281768798828,
      "fcm_dpo/q_t": 0.4422115385532379,
      "grad_norm": 38.74478530883789,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": 1.5718224048614502,
      "logits/rejected": 1.5832414627075195,
      "logps/chosen": -277.646484375,
      "logps/ref_chosen": -272.80133056640625,
      "logps/ref_rejected": -288.4167785644531,
      "logps/rejected": -321.0061950683594,
      "loss": 4.7991,
      "margin_dpo/margin_mean": 27.744281768798828,
      "margin_dpo/margin_std": 45.06488800048828,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -10.884679794311523,
      "KL/mean": -22.615802764892578,
      "KL/rejected_KL_mean": -34.34693908691406,
      "KL/std": 39.44410705566406,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.008672904223203659,
      "fcm_dpo/delta": -0.009699449874460697,
      "fcm_dpo/margin": 23.462268829345703,
      "fcm_dpo/q_t": 0.45149001479148865,
      "grad_norm": 25.835397720336914,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": 1.6942284107208252,
      "logits/rejected": 1.5755202770233154,
      "logps/chosen": -321.5641174316406,
      "logps/ref_chosen": -310.679443359375,
      "logps/ref_rejected": -259.3792419433594,
      "logps/rejected": -293.7261657714844,
      "loss": 4.9479,
      "margin_dpo/margin_mean": 23.46226692199707,
      "margin_dpo/margin_std": 45.58243179321289,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -5.384340286254883,
      "KL/mean": -20.452592849731445,
      "KL/rejected_KL_mean": -35.52083969116211,
      "KL/std": 40.40150451660156,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.008540064096450806,
      "fcm_dpo/delta": -0.020907925441861153,
      "fcm_dpo/margin": 30.136493682861328,
      "fcm_dpo/q_t": 0.4382501244544983,
      "grad_norm": 27.46225357055664,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 1.6214678287506104,
      "logits/rejected": 1.6643211841583252,
      "logps/chosen": -280.3525085449219,
      "logps/ref_chosen": -274.9681701660156,
      "logps/ref_rejected": -283.346923828125,
      "logps/rejected": -318.8677978515625,
      "loss": 4.7545,
      "margin_dpo/margin_mean": 30.136493682861328,
      "margin_dpo/margin_std": 46.9984130859375,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -1.5620079040527344,
      "KL/mean": -21.074443817138672,
      "KL/rejected_KL_mean": -40.58688735961914,
      "KL/std": 40.12736511230469,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.008378613740205765,
      "fcm_dpo/delta": -0.03717486932873726,
      "fcm_dpo/margin": 39.024864196777344,
      "fcm_dpo/q_t": 0.4213881492614746,
      "grad_norm": 21.063472747802734,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.3571749925613403,
      "logits/rejected": 1.3506364822387695,
      "logps/chosen": -267.8415832519531,
      "logps/ref_chosen": -266.2795715332031,
      "logps/ref_rejected": -249.7120819091797,
      "logps/rejected": -290.2989807128906,
      "loss": 4.4818,
      "margin_dpo/margin_mean": 39.024864196777344,
      "margin_dpo/margin_std": 42.31978988647461,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -6.819692611694336,
      "KL/mean": -21.275615692138672,
      "KL/rejected_KL_mean": -35.73153305053711,
      "KL/std": 39.7630615234375,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.008255013264715672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.911832809448242,
      "fcm_dpo/q_t": 0.44243213534355164,
      "grad_norm": 54.20003890991211,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": 1.403881311416626,
      "logits/rejected": 1.444274663925171,
      "logps/chosen": -271.572509765625,
      "logps/ref_chosen": -264.7528076171875,
      "logps/ref_rejected": -241.02842712402344,
      "logps/rejected": -276.75994873046875,
      "loss": 4.8108,
      "margin_dpo/margin_mean": 28.911832809448242,
      "margin_dpo/margin_std": 47.68541717529297,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -11.806400299072266,
      "KL/mean": -25.555179595947266,
      "KL/rejected_KL_mean": -39.303958892822266,
      "KL/std": 40.79571533203125,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.008255013264715672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.49757194519043,
      "fcm_dpo/q_t": 0.4462381601333618,
      "grad_norm": 36.11062240600586,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": 1.4388220310211182,
      "logits/rejected": 1.559664011001587,
      "logps/chosen": -305.94500732421875,
      "logps/ref_chosen": -294.13861083984375,
      "logps/ref_rejected": -303.11669921875,
      "logps/rejected": -342.420654296875,
      "loss": 4.8521,
      "margin_dpo/margin_mean": 27.497568130493164,
      "margin_dpo/margin_std": 48.02104568481445,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -23.532047271728516,
      "KL/mean": -35.85136032104492,
      "KL/rejected_KL_mean": -48.170677185058594,
      "KL/std": 38.89045333862305,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.008255013264715672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.638629913330078,
      "fcm_dpo/q_t": 0.4514297842979431,
      "grad_norm": 44.74829864501953,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": 1.6251846551895142,
      "logits/rejected": 1.7800142765045166,
      "logps/chosen": -290.1087951660156,
      "logps/ref_chosen": -266.5767517089844,
      "logps/ref_rejected": -291.2264709472656,
      "logps/rejected": -339.39715576171875,
      "loss": 4.9262,
      "margin_dpo/margin_mean": 24.638629913330078,
      "margin_dpo/margin_std": 45.924705505371094,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -16.73063087463379,
      "KL/mean": -29.7880802154541,
      "KL/rejected_KL_mean": -42.845516204833984,
      "KL/std": 43.39089584350586,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.008255013264715672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 26.11488151550293,
      "fcm_dpo/q_t": 0.44895580410957336,
      "grad_norm": 22.119131088256836,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": 1.4270472526550293,
      "logits/rejected": 1.4248688220977783,
      "logps/chosen": -301.7713928222656,
      "logps/ref_chosen": -285.040771484375,
      "logps/ref_rejected": -258.0254821777344,
      "logps/rejected": -300.8710021972656,
      "loss": 4.9161,
      "margin_dpo/margin_mean": 26.11488151550293,
      "margin_dpo/margin_std": 51.482994079589844,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -17.433639526367188,
      "KL/mean": -32.21119689941406,
      "KL/rejected_KL_mean": -46.988739013671875,
      "KL/std": 44.56312942504883,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.008255013264715672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.555112838745117,
      "fcm_dpo/q_t": 0.44150805473327637,
      "grad_norm": 36.44731140136719,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": 1.5784231424331665,
      "logits/rejected": 1.5953052043914795,
      "logps/chosen": -350.18499755859375,
      "logps/ref_chosen": -332.7513732910156,
      "logps/ref_rejected": -302.9167785644531,
      "logps/rejected": -349.9054870605469,
      "loss": 4.8435,
      "margin_dpo/margin_mean": 29.55511474609375,
      "margin_dpo/margin_std": 54.53816223144531,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -17.990602493286133,
      "KL/mean": -38.02338409423828,
      "KL/rejected_KL_mean": -58.05616760253906,
      "KL/std": 43.427764892578125,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.008191779255867004,
      "fcm_dpo/delta": -0.010991193354129791,
      "fcm_dpo/margin": 40.0655632019043,
      "fcm_dpo/q_t": 0.42250949144363403,
      "grad_norm": 25.941158294677734,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": 1.6569850444793701,
      "logits/rejected": 1.6633639335632324,
      "logps/chosen": -304.45855712890625,
      "logps/ref_chosen": -286.46795654296875,
      "logps/ref_rejected": -253.99298095703125,
      "logps/rejected": -312.0491638183594,
      "loss": 4.5271,
      "margin_dpo/margin_mean": 40.06556701660156,
      "margin_dpo/margin_std": 52.957393646240234,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -13.793067932128906,
      "KL/mean": -30.04142189025879,
      "KL/rejected_KL_mean": -46.289772033691406,
      "KL/std": 43.42946243286133,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.008182745426893234,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 32.4966926574707,
      "fcm_dpo/q_t": 0.43625035881996155,
      "grad_norm": 36.58794403076172,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 1.6050039529800415,
      "logits/rejected": 1.6731162071228027,
      "logps/chosen": -325.171630859375,
      "logps/ref_chosen": -311.3785400390625,
      "logps/ref_rejected": -290.3092956542969,
      "logps/rejected": -336.5990905761719,
      "loss": 4.7039,
      "margin_dpo/margin_mean": 32.4966926574707,
      "margin_dpo/margin_std": 46.86227035522461,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -14.238744735717773,
      "KL/mean": -30.255115509033203,
      "KL/rejected_KL_mean": -46.27147674560547,
      "KL/std": 42.072113037109375,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.008129769004881382,
      "fcm_dpo/delta": -0.010837285779416561,
      "fcm_dpo/margin": 32.03274917602539,
      "fcm_dpo/q_t": 0.4374465346336365,
      "grad_norm": 27.92180061340332,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": 1.3620356321334839,
      "logits/rejected": 1.4264293909072876,
      "logps/chosen": -273.5716857910156,
      "logps/ref_chosen": -259.33294677734375,
      "logps/ref_rejected": -261.7661437988281,
      "logps/rejected": -308.03759765625,
      "loss": 4.7464,
      "margin_dpo/margin_mean": 32.03274917602539,
      "margin_dpo/margin_std": 50.45026779174805,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -15.625926971435547,
      "KL/mean": -33.10492706298828,
      "KL/rejected_KL_mean": -50.58393096923828,
      "KL/std": 44.279354095458984,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.008112109266221523,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.95800018310547,
      "fcm_dpo/q_t": 0.43223506212234497,
      "grad_norm": 32.5186653137207,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 1.623255968093872,
      "logits/rejected": 1.5669981241226196,
      "logps/chosen": -321.8302917480469,
      "logps/ref_chosen": -306.2043762207031,
      "logps/ref_rejected": -256.7926330566406,
      "logps/rejected": -307.3765563964844,
      "loss": 4.6761,
      "margin_dpo/margin_mean": 34.95800018310547,
      "margin_dpo/margin_std": 53.10782241821289,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -16.793703079223633,
      "KL/mean": -30.39249038696289,
      "KL/rejected_KL_mean": -43.99127960205078,
      "KL/std": 44.25605010986328,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.008094197139143944,
      "fcm_dpo/delta": -0.011089268140494823,
      "fcm_dpo/margin": 27.19757080078125,
      "fcm_dpo/q_t": 0.4469802975654602,
      "grad_norm": 24.939477920532227,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 1.1432145833969116,
      "logits/rejected": 1.2596477270126343,
      "logps/chosen": -310.12567138671875,
      "logps/ref_chosen": -293.3319396972656,
      "logps/ref_rejected": -282.7599182128906,
      "logps/rejected": -326.7511901855469,
      "loss": 4.8866,
      "margin_dpo/margin_mean": 27.19757080078125,
      "margin_dpo/margin_std": 49.46776580810547,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -12.735719680786133,
      "KL/mean": -30.299850463867188,
      "KL/rejected_KL_mean": -47.863983154296875,
      "KL/std": 42.577274322509766,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.007977076806128025,
      "fcm_dpo/delta": -0.009893092326819897,
      "fcm_dpo/margin": 35.12825012207031,
      "fcm_dpo/q_t": 0.4327159821987152,
      "grad_norm": 36.90123748779297,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": 1.4058762788772583,
      "logits/rejected": 1.3049651384353638,
      "logps/chosen": -299.74188232421875,
      "logps/ref_chosen": -287.0061340332031,
      "logps/ref_rejected": -233.23130798339844,
      "logps/rejected": -281.09527587890625,
      "loss": 4.6524,
      "margin_dpo/margin_mean": 35.12825012207031,
      "margin_dpo/margin_std": 47.36408233642578,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -14.34689712524414,
      "KL/mean": -33.912696838378906,
      "KL/rejected_KL_mean": -53.47848892211914,
      "KL/std": 46.12067413330078,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.007907740771770477,
      "fcm_dpo/delta": -0.010912401601672173,
      "fcm_dpo/margin": 39.13160705566406,
      "fcm_dpo/q_t": 0.4261813759803772,
      "grad_norm": 26.237770080566406,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": 1.5427178144454956,
      "logits/rejected": 1.4585115909576416,
      "logps/chosen": -290.16217041015625,
      "logps/ref_chosen": -275.8153076171875,
      "logps/ref_rejected": -238.54286193847656,
      "logps/rejected": -292.0213623046875,
      "loss": 4.5856,
      "margin_dpo/margin_mean": 39.13160705566406,
      "margin_dpo/margin_std": 53.13092041015625,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -13.786384582519531,
      "KL/mean": -33.29928207397461,
      "KL/rejected_KL_mean": -52.81217956542969,
      "KL/std": 44.543548583984375,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.00789222028106451,
      "fcm_dpo/delta": -0.01978299394249916,
      "fcm_dpo/margin": 39.025794982910156,
      "fcm_dpo/q_t": 0.4261471927165985,
      "grad_norm": 37.17509460449219,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": 1.416817307472229,
      "logits/rejected": 1.4880239963531494,
      "logps/chosen": -252.8867645263672,
      "logps/ref_chosen": -239.1003875732422,
      "logps/ref_rejected": -230.31900024414062,
      "logps/rejected": -283.13116455078125,
      "loss": 4.5681,
      "margin_dpo/margin_mean": 39.025794982910156,
      "margin_dpo/margin_std": 48.93801498413086,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -23.89708137512207,
      "KL/mean": -37.066368103027344,
      "KL/rejected_KL_mean": -50.235652923583984,
      "KL/std": 45.57200622558594,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.007783575914800167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 26.338563919067383,
      "fcm_dpo/q_t": 0.4517073631286621,
      "grad_norm": 23.56674575805664,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": 1.5550484657287598,
      "logits/rejected": 1.582556962966919,
      "logps/chosen": -324.2264404296875,
      "logps/ref_chosen": -300.3293762207031,
      "logps/ref_rejected": -332.132568359375,
      "logps/rejected": -382.36822509765625,
      "loss": 4.9821,
      "margin_dpo/margin_mean": 26.338563919067383,
      "margin_dpo/margin_std": 58.862796783447266,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -17.146535873413086,
      "KL/mean": -30.871318817138672,
      "KL/rejected_KL_mean": -44.596092224121094,
      "KL/std": 44.23310470581055,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.007783575914800167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.44956398010254,
      "fcm_dpo/q_t": 0.44892972707748413,
      "grad_norm": 26.707029342651367,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": 1.6369954347610474,
      "logits/rejected": 1.7108033895492554,
      "logps/chosen": -262.5586853027344,
      "logps/ref_chosen": -245.41213989257812,
      "logps/ref_rejected": -251.72999572753906,
      "logps/rejected": -296.3260803222656,
      "loss": 4.9085,
      "margin_dpo/margin_mean": 27.449562072753906,
      "margin_dpo/margin_std": 52.702720642089844,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -16.15459442138672,
      "KL/mean": -35.788944244384766,
      "KL/rejected_KL_mean": -55.42328643798828,
      "KL/std": 46.52720260620117,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.007709097117185593,
      "fcm_dpo/delta": -0.013744833879172802,
      "fcm_dpo/margin": 39.268699645996094,
      "fcm_dpo/q_t": 0.42752811312675476,
      "grad_norm": 21.669261932373047,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": 1.4608261585235596,
      "logits/rejected": 1.512335181236267,
      "logps/chosen": -293.0304260253906,
      "logps/ref_chosen": -276.87579345703125,
      "logps/ref_rejected": -259.0455017089844,
      "logps/rejected": -314.4687805175781,
      "loss": 4.6065,
      "margin_dpo/margin_mean": 39.268699645996094,
      "margin_dpo/margin_std": 53.9827880859375,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -13.523141860961914,
      "KL/mean": -33.678550720214844,
      "KL/rejected_KL_mean": -53.833961486816406,
      "KL/std": 46.06224060058594,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.0076585072092711926,
      "fcm_dpo/delta": -0.026219487190246582,
      "fcm_dpo/margin": 40.31081771850586,
      "fcm_dpo/q_t": 0.42624932527542114,
      "grad_norm": 20.67548942565918,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": 1.6385304927825928,
      "logits/rejected": 1.647813081741333,
      "logps/chosen": -317.49420166015625,
      "logps/ref_chosen": -303.9710693359375,
      "logps/ref_rejected": -255.62033081054688,
      "logps/rejected": -309.45428466796875,
      "loss": 4.5771,
      "margin_dpo/margin_mean": 40.31081771850586,
      "margin_dpo/margin_std": 53.02606201171875,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -14.27197265625,
      "KL/mean": -39.3639030456543,
      "KL/rejected_KL_mean": -64.45581817626953,
      "KL/std": 48.13701629638672,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.007356872782111168,
      "fcm_dpo/delta": -0.042223554104566574,
      "fcm_dpo/margin": 50.183841705322266,
      "fcm_dpo/q_t": 0.41162383556365967,
      "grad_norm": 29.298030853271484,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": 1.2834402322769165,
      "logits/rejected": 1.262515902519226,
      "logps/chosen": -295.9678955078125,
      "logps/ref_chosen": -281.6959533691406,
      "logps/ref_rejected": -258.93353271484375,
      "logps/rejected": -323.389404296875,
      "loss": 4.3773,
      "margin_dpo/margin_mean": 50.183841705322266,
      "margin_dpo/margin_std": 54.968387603759766,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -15.751310348510742,
      "KL/mean": -34.85417175292969,
      "KL/rejected_KL_mean": -53.95704650878906,
      "KL/std": 46.25481414794922,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.007288265507668257,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.205726623535156,
      "fcm_dpo/q_t": 0.4332759380340576,
      "grad_norm": 32.35492706298828,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 1.169114112854004,
      "logits/rejected": 1.2590916156768799,
      "logps/chosen": -239.70785522460938,
      "logps/ref_chosen": -223.95655822753906,
      "logps/ref_rejected": -238.95187377929688,
      "logps/rejected": -292.908935546875,
      "loss": 4.6808,
      "margin_dpo/margin_mean": 38.20572280883789,
      "margin_dpo/margin_std": 55.78633499145508,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -24.58745765686035,
      "KL/mean": -39.84353256225586,
      "KL/rejected_KL_mean": -55.09961700439453,
      "KL/std": 46.1339225769043,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.007288265507668257,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 30.512161254882812,
      "fcm_dpo/q_t": 0.44639837741851807,
      "grad_norm": 24.349523544311523,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 1.4888525009155273,
      "logits/rejected": 1.580322265625,
      "logps/chosen": -339.5120544433594,
      "logps/ref_chosen": -314.92462158203125,
      "logps/ref_rejected": -279.6347351074219,
      "logps/rejected": -334.73431396484375,
      "loss": 4.8494,
      "margin_dpo/margin_mean": 30.512165069580078,
      "margin_dpo/margin_std": 50.85298538208008,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -15.180761337280273,
      "KL/mean": -32.79558181762695,
      "KL/rejected_KL_mean": -50.41040802001953,
      "KL/std": 48.46588134765625,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.007288265507668257,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 35.229644775390625,
      "fcm_dpo/q_t": 0.43870431184768677,
      "grad_norm": 26.99349021911621,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": 1.4114093780517578,
      "logits/rejected": 1.4808433055877686,
      "logps/chosen": -278.5003662109375,
      "logps/ref_chosen": -263.3196105957031,
      "logps/ref_rejected": -247.13612365722656,
      "logps/rejected": -297.5465087890625,
      "loss": 4.7496,
      "margin_dpo/margin_mean": 35.22964859008789,
      "margin_dpo/margin_std": 54.46686553955078,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -15.734930038452148,
      "KL/mean": -34.291011810302734,
      "KL/rejected_KL_mean": -52.84709548950195,
      "KL/std": 47.14774703979492,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.007288265507668257,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.11216735839844,
      "fcm_dpo/q_t": 0.43549400568008423,
      "grad_norm": 32.12618637084961,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": 1.321073055267334,
      "logits/rejected": 1.3923301696777344,
      "logps/chosen": -295.09716796875,
      "logps/ref_chosen": -279.36224365234375,
      "logps/ref_rejected": -264.46112060546875,
      "logps/rejected": -317.3082275390625,
      "loss": 4.6947,
      "margin_dpo/margin_mean": 37.11216735839844,
      "margin_dpo/margin_std": 54.00865173339844,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -14.285463333129883,
      "KL/mean": -33.561344146728516,
      "KL/rejected_KL_mean": -52.83723068237305,
      "KL/std": 50.108272552490234,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.007288265507668257,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.551780700683594,
      "fcm_dpo/q_t": 0.43330731987953186,
      "grad_norm": 31.607486724853516,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": 1.5549715757369995,
      "logits/rejected": 1.643933892250061,
      "logps/chosen": -300.7345886230469,
      "logps/ref_chosen": -286.4491271972656,
      "logps/ref_rejected": -294.5469665527344,
      "logps/rejected": -347.38421630859375,
      "loss": 4.7194,
      "margin_dpo/margin_mean": 38.551780700683594,
      "margin_dpo/margin_std": 62.491695404052734,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -15.324605941772461,
      "KL/mean": -33.354000091552734,
      "KL/rejected_KL_mean": -51.38338088989258,
      "KL/std": 50.750667572021484,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.007273453753441572,
      "fcm_dpo/delta": -0.010202908888459206,
      "fcm_dpo/margin": 36.05876922607422,
      "fcm_dpo/q_t": 0.4373144209384918,
      "grad_norm": 39.25311279296875,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": 1.4319047927856445,
      "logits/rejected": 1.4398705959320068,
      "logps/chosen": -297.68463134765625,
      "logps/ref_chosen": -282.36004638671875,
      "logps/ref_rejected": -243.79052734375,
      "logps/rejected": -295.1739196777344,
      "loss": 4.766,
      "margin_dpo/margin_mean": 36.05876541137695,
      "margin_dpo/margin_std": 60.610618591308594,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -8.284879684448242,
      "KL/mean": -28.881607055664062,
      "KL/rejected_KL_mean": -49.47833251953125,
      "KL/std": 48.18082046508789,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.0072290184907615185,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.193450927734375,
      "fcm_dpo/q_t": 0.42868682742118835,
      "grad_norm": 28.814533233642578,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": 1.322727084159851,
      "logits/rejected": 1.3360469341278076,
      "logps/chosen": -287.42431640625,
      "logps/ref_chosen": -279.139404296875,
      "logps/ref_rejected": -240.5850067138672,
      "logps/rejected": -290.0633239746094,
      "loss": 4.6121,
      "margin_dpo/margin_mean": 41.193450927734375,
      "margin_dpo/margin_std": 55.910274505615234,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -11.959070205688477,
      "KL/mean": -29.441879272460938,
      "KL/rejected_KL_mean": -46.924686431884766,
      "KL/std": 45.24892807006836,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.0072290184907615185,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.96562194824219,
      "fcm_dpo/q_t": 0.4398249387741089,
      "grad_norm": 30.645444869995117,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": 1.4219770431518555,
      "logits/rejected": 1.502173900604248,
      "logps/chosen": -270.77056884765625,
      "logps/ref_chosen": -258.8114929199219,
      "logps/ref_rejected": -266.3898010253906,
      "logps/rejected": -313.314453125,
      "loss": 4.7744,
      "margin_dpo/margin_mean": 34.96562576293945,
      "margin_dpo/margin_std": 56.95813751220703,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -12.892793655395508,
      "KL/mean": -32.40087890625,
      "KL/rejected_KL_mean": -51.90896224975586,
      "KL/std": 50.369930267333984,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.0072290184907615185,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.01617431640625,
      "fcm_dpo/q_t": 0.433335542678833,
      "grad_norm": 27.78238296508789,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": 1.3648343086242676,
      "logits/rejected": 1.299009919166565,
      "logps/chosen": -314.958984375,
      "logps/ref_chosen": -302.0661926269531,
      "logps/ref_rejected": -265.5518798828125,
      "logps/rejected": -317.4608154296875,
      "loss": 4.6943,
      "margin_dpo/margin_mean": 39.016170501708984,
      "margin_dpo/margin_std": 59.75685119628906,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -8.714956283569336,
      "KL/mean": -25.09259605407715,
      "KL/rejected_KL_mean": -41.47023391723633,
      "KL/std": 48.819297790527344,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.0072290184907615185,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 32.755279541015625,
      "fcm_dpo/q_t": 0.4438488483428955,
      "grad_norm": 28.894617080688477,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": 1.566054105758667,
      "logits/rejected": 1.6759568452835083,
      "logps/chosen": -267.71331787109375,
      "logps/ref_chosen": -258.99835205078125,
      "logps/ref_rejected": -273.7834777832031,
      "logps/rejected": -315.2536926269531,
      "loss": 4.8579,
      "margin_dpo/margin_mean": 32.755279541015625,
      "margin_dpo/margin_std": 62.38018035888672,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -6.24370002746582,
      "KL/mean": -27.37607765197754,
      "KL/rejected_KL_mean": -48.50846481323242,
      "KL/std": 48.191673278808594,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.007154467981308699,
      "fcm_dpo/delta": -0.012957663275301456,
      "fcm_dpo/margin": 42.2647705078125,
      "fcm_dpo/q_t": 0.4276670515537262,
      "grad_norm": 37.52655792236328,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 1.288834571838379,
      "logits/rejected": 1.3771389722824097,
      "logps/chosen": -281.7667541503906,
      "logps/ref_chosen": -275.5230407714844,
      "logps/ref_rejected": -255.50784301757812,
      "logps/rejected": -304.01629638671875,
      "loss": 4.5933,
      "margin_dpo/margin_mean": 42.2647705078125,
      "margin_dpo/margin_std": 56.21650695800781,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -12.253290176391602,
      "KL/mean": -28.736299514770508,
      "KL/rejected_KL_mean": -45.219322204589844,
      "KL/std": 49.2982063293457,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.007154467981308699,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 32.96603775024414,
      "fcm_dpo/q_t": 0.4440564215183258,
      "grad_norm": 24.134424209594727,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.348827838897705,
      "logits/rejected": 1.3938237428665161,
      "logps/chosen": -265.61676025390625,
      "logps/ref_chosen": -253.36349487304688,
      "logps/ref_rejected": -247.1916961669922,
      "logps/rejected": -292.4110107421875,
      "loss": 4.8488,
      "margin_dpo/margin_mean": 32.966033935546875,
      "margin_dpo/margin_std": 59.99987030029297,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -9.122575759887695,
      "KL/mean": -28.979413986206055,
      "KL/rejected_KL_mean": -48.83624267578125,
      "KL/std": 48.21861267089844,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.007086261175572872,
      "fcm_dpo/delta": -0.011974107474088669,
      "fcm_dpo/margin": 39.71366500854492,
      "fcm_dpo/q_t": 0.4322541058063507,
      "grad_norm": 26.637306213378906,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": 1.4969183206558228,
      "logits/rejected": 1.4512302875518799,
      "logps/chosen": -274.11065673828125,
      "logps/ref_chosen": -264.9880676269531,
      "logps/ref_rejected": -243.07838439941406,
      "logps/rejected": -291.91461181640625,
      "loss": 4.6534,
      "margin_dpo/margin_mean": 39.71366882324219,
      "margin_dpo/margin_std": 54.57093811035156,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -4.471899032592773,
      "KL/mean": -26.982933044433594,
      "KL/rejected_KL_mean": -49.49395751953125,
      "KL/std": 47.800193786621094,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.007009504828602076,
      "fcm_dpo/delta": -0.04008102789521217,
      "fcm_dpo/margin": 45.02206802368164,
      "fcm_dpo/q_t": 0.42434728145599365,
      "grad_norm": 61.56886291503906,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": 1.360253095626831,
      "logits/rejected": 1.4185779094696045,
      "logps/chosen": -320.34259033203125,
      "logps/ref_chosen": -315.8706970214844,
      "logps/ref_rejected": -304.3729248046875,
      "logps/rejected": -353.8668518066406,
      "loss": 4.5323,
      "margin_dpo/margin_mean": 45.022064208984375,
      "margin_dpo/margin_std": 51.687217712402344,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -8.347213745117188,
      "KL/mean": -31.14093780517578,
      "KL/rejected_KL_mean": -53.9346809387207,
      "KL/std": 50.61368179321289,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.00680368859320879,
      "fcm_dpo/delta": -0.012333443388342857,
      "fcm_dpo/margin": 45.58747100830078,
      "fcm_dpo/q_t": 0.4259068965911865,
      "grad_norm": 23.71196937561035,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 1.3988443613052368,
      "logits/rejected": 1.3694525957107544,
      "logps/chosen": -319.4443664550781,
      "logps/ref_chosen": -311.09716796875,
      "logps/ref_rejected": -266.01959228515625,
      "logps/rejected": -319.9542541503906,
      "loss": 4.571,
      "margin_dpo/margin_mean": 45.587467193603516,
      "margin_dpo/margin_std": 58.72907257080078,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -8.51583480834961,
      "KL/mean": -31.536964416503906,
      "KL/rejected_KL_mean": -54.558082580566406,
      "KL/std": 50.38486862182617,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.0067716557532548904,
      "fcm_dpo/delta": -0.01749473065137863,
      "fcm_dpo/margin": 46.042259216308594,
      "fcm_dpo/q_t": 0.4253847599029541,
      "grad_norm": 30.4219970703125,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": 1.488696575164795,
      "logits/rejected": 1.3850308656692505,
      "logps/chosen": -293.25628662109375,
      "logps/ref_chosen": -284.740478515625,
      "logps/ref_rejected": -237.64117431640625,
      "logps/rejected": -292.1992492675781,
      "loss": 4.5677,
      "margin_dpo/margin_mean": 46.042259216308594,
      "margin_dpo/margin_std": 58.681640625,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -20.66814613342285,
      "KL/mean": -35.503013610839844,
      "KL/rejected_KL_mean": -50.33789825439453,
      "KL/std": 49.46540451049805,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.0067008244805037975,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.669742584228516,
      "fcm_dpo/q_t": 0.4528736472129822,
      "grad_norm": 46.56711959838867,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": 1.4676657915115356,
      "logits/rejected": 1.4960618019104004,
      "logps/chosen": -293.2810363769531,
      "logps/ref_chosen": -272.6128845214844,
      "logps/ref_rejected": -249.99725341796875,
      "logps/rejected": -300.3351745605469,
      "loss": 4.9753,
      "margin_dpo/margin_mean": 29.66974449157715,
      "margin_dpo/margin_std": 61.60385513305664,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -10.492286682128906,
      "KL/mean": -30.996910095214844,
      "KL/rejected_KL_mean": -51.50152587890625,
      "KL/std": 50.43907928466797,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.0067008244805037975,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.009239196777344,
      "fcm_dpo/q_t": 0.434203565120697,
      "grad_norm": 26.606531143188477,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": 1.4750033617019653,
      "logits/rejected": 1.4922953844070435,
      "logps/chosen": -284.9281005859375,
      "logps/ref_chosen": -274.435791015625,
      "logps/ref_rejected": -257.8125305175781,
      "logps/rejected": -309.31402587890625,
      "loss": 4.673,
      "margin_dpo/margin_mean": 41.009239196777344,
      "margin_dpo/margin_std": 57.429317474365234,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -16.942001342773438,
      "KL/mean": -33.00630187988281,
      "KL/rejected_KL_mean": -49.070594787597656,
      "KL/std": 49.50682067871094,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.006659214850515127,
      "fcm_dpo/delta": -0.01562135573476553,
      "fcm_dpo/margin": 32.12859344482422,
      "fcm_dpo/q_t": 0.44862914085388184,
      "grad_norm": 40.08014678955078,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": 1.2702447175979614,
      "logits/rejected": 1.3093323707580566,
      "logps/chosen": -278.2533264160156,
      "logps/ref_chosen": -261.3113098144531,
      "logps/ref_rejected": -231.152587890625,
      "logps/rejected": -280.22320556640625,
      "loss": 4.9064,
      "margin_dpo/margin_mean": 32.12858963012695,
      "margin_dpo/margin_std": 59.875526428222656,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -17.84748077392578,
      "KL/mean": -37.48646545410156,
      "KL/rejected_KL_mean": -57.12546157836914,
      "KL/std": 52.798133850097656,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.0066176047548651695,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.27798080444336,
      "fcm_dpo/q_t": 0.4379235506057739,
      "grad_norm": 43.79726028442383,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": 1.3724814653396606,
      "logits/rejected": 1.3295234441757202,
      "logps/chosen": -286.3147277832031,
      "logps/ref_chosen": -268.46722412109375,
      "logps/ref_rejected": -240.27597045898438,
      "logps/rejected": -297.4013977050781,
      "loss": 4.744,
      "margin_dpo/margin_mean": 39.27798080444336,
      "margin_dpo/margin_std": 60.72298812866211,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -19.687164306640625,
      "KL/mean": -39.604488372802734,
      "KL/rejected_KL_mean": -59.52181625366211,
      "KL/std": 54.42945861816406,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.006543540861457586,
      "fcm_dpo/delta": -0.01609138771891594,
      "fcm_dpo/margin": 39.83466720581055,
      "fcm_dpo/q_t": 0.43763238191604614,
      "grad_norm": 31.380090713500977,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 1.4238722324371338,
      "logits/rejected": 1.5419538021087646,
      "logps/chosen": -304.7923583984375,
      "logps/ref_chosen": -285.1051940917969,
      "logps/ref_rejected": -281.2331848144531,
      "logps/rejected": -340.7550048828125,
      "loss": 4.7564,
      "margin_dpo/margin_mean": 39.83466339111328,
      "margin_dpo/margin_std": 64.36811828613281,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -17.592878341674805,
      "KL/mean": -37.08160400390625,
      "KL/rejected_KL_mean": -56.570335388183594,
      "KL/std": 52.140403747558594,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.006520876660943031,
      "fcm_dpo/delta": -0.009282995015382767,
      "fcm_dpo/margin": 38.97745895385742,
      "fcm_dpo/q_t": 0.4395999312400818,
      "grad_norm": 35.60149002075195,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 1.4641497135162354,
      "logits/rejected": 1.4251803159713745,
      "logps/chosen": -291.70550537109375,
      "logps/ref_chosen": -274.1126403808594,
      "logps/ref_rejected": -256.7730407714844,
      "logps/rejected": -313.3433837890625,
      "loss": 4.7597,
      "margin_dpo/margin_mean": 38.977455139160156,
      "margin_dpo/margin_std": 60.086788177490234,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -26.138675689697266,
      "KL/mean": -47.778778076171875,
      "KL/rejected_KL_mean": -69.41888427734375,
      "KL/std": 54.59175109863281,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.006337874568998814,
      "fcm_dpo/delta": -0.028612853959202766,
      "fcm_dpo/margin": 43.28020477294922,
      "fcm_dpo/q_t": 0.4342646598815918,
      "grad_norm": 39.25895309448242,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": 1.4168802499771118,
      "logits/rejected": 1.3854378461837769,
      "logps/chosen": -315.70440673828125,
      "logps/ref_chosen": -289.56573486328125,
      "logps/ref_rejected": -277.9891662597656,
      "logps/rejected": -347.4080505371094,
      "loss": 4.7167,
      "margin_dpo/margin_mean": 43.28020477294922,
      "margin_dpo/margin_std": 67.19970703125,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -23.41852378845215,
      "KL/mean": -39.98757553100586,
      "KL/rejected_KL_mean": -56.556617736816406,
      "KL/std": 53.967044830322266,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.006337874568998814,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 33.13809585571289,
      "fcm_dpo/q_t": 0.44977810978889465,
      "grad_norm": 27.31587028503418,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": 1.3723818063735962,
      "logits/rejected": 1.3635538816452026,
      "logps/chosen": -293.60247802734375,
      "logps/ref_chosen": -270.1839294433594,
      "logps/ref_rejected": -245.86949157714844,
      "logps/rejected": -302.42608642578125,
      "loss": 4.9169,
      "margin_dpo/margin_mean": 33.138092041015625,
      "margin_dpo/margin_std": 62.7706298828125,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -19.83953285217285,
      "KL/mean": -40.91039276123047,
      "KL/rejected_KL_mean": -61.981239318847656,
      "KL/std": 54.39976119995117,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.006322373636066914,
      "fcm_dpo/delta": -0.02470031939446926,
      "fcm_dpo/margin": 42.141700744628906,
      "fcm_dpo/q_t": 0.4358440041542053,
      "grad_norm": 32.039161682128906,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": 1.2940306663513184,
      "logits/rejected": 1.2854132652282715,
      "logps/chosen": -280.5259704589844,
      "logps/ref_chosen": -260.6864318847656,
      "logps/ref_rejected": -245.99508666992188,
      "logps/rejected": -307.976318359375,
      "loss": 4.6927,
      "margin_dpo/margin_mean": 42.141700744628906,
      "margin_dpo/margin_std": 55.31855392456055,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -19.543973922729492,
      "KL/mean": -42.607730865478516,
      "KL/rejected_KL_mean": -65.67147827148438,
      "KL/std": 54.51893615722656,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.006213866639882326,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.12752151489258,
      "fcm_dpo/q_t": 0.4317150115966797,
      "grad_norm": 35.053001403808594,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": 1.3713979721069336,
      "logits/rejected": 1.2654247283935547,
      "logps/chosen": -317.45782470703125,
      "logps/ref_chosen": -297.9138488769531,
      "logps/ref_rejected": -253.39952087402344,
      "logps/rejected": -319.0710144042969,
      "loss": 4.6684,
      "margin_dpo/margin_mean": 46.12752151489258,
      "margin_dpo/margin_std": 69.46429443359375,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -17.744577407836914,
      "KL/mean": -40.88645553588867,
      "KL/rejected_KL_mean": -64.02833557128906,
      "KL/std": 55.532798767089844,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.006161234341561794,
      "fcm_dpo/delta": -0.029833219945430756,
      "fcm_dpo/margin": 46.28375244140625,
      "fcm_dpo/q_t": 0.43170052766799927,
      "grad_norm": 22.648588180541992,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": 1.3604702949523926,
      "logits/rejected": 1.311057209968567,
      "logps/chosen": -327.67095947265625,
      "logps/ref_chosen": -309.9263916015625,
      "logps/ref_rejected": -253.452392578125,
      "logps/rejected": -317.480712890625,
      "loss": 4.6889,
      "margin_dpo/margin_mean": 46.28375244140625,
      "margin_dpo/margin_std": 70.31912994384766,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -23.124380111694336,
      "KL/mean": -44.467185974121094,
      "KL/rejected_KL_mean": -65.80999755859375,
      "KL/std": 56.13432312011719,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.006013539619743824,
      "fcm_dpo/delta": -0.01272722240537405,
      "fcm_dpo/margin": 42.685611724853516,
      "fcm_dpo/q_t": 0.4386219382286072,
      "grad_norm": 44.26255798339844,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 1.129820466041565,
      "logits/rejected": 1.1621758937835693,
      "logps/chosen": -282.9380187988281,
      "logps/ref_chosen": -259.8136291503906,
      "logps/ref_rejected": -245.84947204589844,
      "logps/rejected": -311.65948486328125,
      "loss": 4.7371,
      "margin_dpo/margin_mean": 42.68560791015625,
      "margin_dpo/margin_std": 63.72529602050781,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -20.80038070678711,
      "KL/mean": -44.611083984375,
      "KL/rejected_KL_mean": -68.42179870605469,
      "KL/std": 56.48371124267578,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.62141036987305,
      "fcm_dpo/q_t": 0.43102967739105225,
      "grad_norm": 34.5673828125,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": 1.494371771812439,
      "logits/rejected": 1.5827891826629639,
      "logps/chosen": -286.154052734375,
      "logps/ref_chosen": -265.35369873046875,
      "logps/ref_rejected": -247.9174346923828,
      "logps/rejected": -316.3392028808594,
      "loss": 4.6156,
      "margin_dpo/margin_mean": 47.62141036987305,
      "margin_dpo/margin_std": 59.76416015625,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -20.941368103027344,
      "KL/mean": -41.9123420715332,
      "KL/rejected_KL_mean": -62.88330078125,
      "KL/std": 55.87968444824219,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.941925048828125,
      "fcm_dpo/q_t": 0.43941792845726013,
      "grad_norm": 32.7757453918457,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": 1.5557620525360107,
      "logits/rejected": 1.5598646402359009,
      "logps/chosen": -309.16461181640625,
      "logps/ref_chosen": -288.2232360839844,
      "logps/ref_rejected": -270.7300109863281,
      "logps/rejected": -333.61334228515625,
      "loss": 4.7557,
      "margin_dpo/margin_mean": 41.941932678222656,
      "margin_dpo/margin_std": 65.73947143554688,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -24.936317443847656,
      "KL/mean": -34.90322494506836,
      "KL/rejected_KL_mean": -44.8701286315918,
      "KL/std": 54.15550994873047,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 19.933805465698242,
      "fcm_dpo/q_t": 0.4712284207344055,
      "grad_norm": 25.475360870361328,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 1.305245280265808,
      "logits/rejected": 1.342010498046875,
      "logps/chosen": -306.5560302734375,
      "logps/ref_chosen": -281.61968994140625,
      "logps/ref_rejected": -271.03973388671875,
      "logps/rejected": -315.9098815917969,
      "loss": 5.2371,
      "margin_dpo/margin_mean": 19.933805465698242,
      "margin_dpo/margin_std": 64.28247833251953,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -31.55419921875,
      "KL/mean": -49.88814926147461,
      "KL/rejected_KL_mean": -68.22209930419922,
      "KL/std": 55.3823127746582,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 36.66789245605469,
      "fcm_dpo/q_t": 0.44722604751586914,
      "grad_norm": 28.218448638916016,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 1.0119960308074951,
      "logits/rejected": 1.1511033773422241,
      "logps/chosen": -274.0173645019531,
      "logps/ref_chosen": -242.4631805419922,
      "logps/ref_rejected": -244.0020751953125,
      "logps/rejected": -312.22418212890625,
      "loss": 4.8884,
      "margin_dpo/margin_mean": 36.66789627075195,
      "margin_dpo/margin_std": 69.21680450439453,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -22.839426040649414,
      "KL/mean": -45.56513214111328,
      "KL/rejected_KL_mean": -68.29083251953125,
      "KL/std": 57.05026626586914,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.451416015625,
      "fcm_dpo/q_t": 0.4344841539859772,
      "grad_norm": 51.030006408691406,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": 1.2438616752624512,
      "logits/rejected": 1.260660171508789,
      "logps/chosen": -314.8280334472656,
      "logps/ref_chosen": -291.98858642578125,
      "logps/ref_rejected": -273.35955810546875,
      "logps/rejected": -341.6504211425781,
      "loss": 4.6762,
      "margin_dpo/margin_mean": 45.451412200927734,
      "margin_dpo/margin_std": 63.52383041381836,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -31.096771240234375,
      "KL/mean": -49.98383331298828,
      "KL/rejected_KL_mean": -68.87091064453125,
      "KL/std": 57.57355499267578,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.77413558959961,
      "fcm_dpo/q_t": 0.44595223665237427,
      "grad_norm": 45.51173782348633,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": 1.0950443744659424,
      "logits/rejected": 1.0684353113174438,
      "logps/chosen": -306.08526611328125,
      "logps/ref_chosen": -274.98846435546875,
      "logps/ref_rejected": -254.75782775878906,
      "logps/rejected": -323.6287536621094,
      "loss": 4.8603,
      "margin_dpo/margin_mean": 37.77413558959961,
      "margin_dpo/margin_std": 67.39826965332031,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -20.749170303344727,
      "KL/mean": -45.08157730102539,
      "KL/rejected_KL_mean": -69.41399383544922,
      "KL/std": 57.01692199707031,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.66482162475586,
      "fcm_dpo/q_t": 0.43012499809265137,
      "grad_norm": 25.108314514160156,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": 1.064978837966919,
      "logits/rejected": 1.1277459859848022,
      "logps/chosen": -272.3753356933594,
      "logps/ref_chosen": -251.62615966796875,
      "logps/ref_rejected": -251.9549560546875,
      "logps/rejected": -321.36895751953125,
      "loss": 4.6033,
      "margin_dpo/margin_mean": 48.66482162475586,
      "margin_dpo/margin_std": 61.96724319458008,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -28.71128273010254,
      "KL/mean": -51.01698684692383,
      "KL/rejected_KL_mean": -73.32269287109375,
      "KL/std": 59.98270797729492,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.006005856674164534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.61140823364258,
      "fcm_dpo/q_t": 0.4361339807510376,
      "grad_norm": 27.3245849609375,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": 1.1880643367767334,
      "logits/rejected": 1.2265613079071045,
      "logps/chosen": -306.64788818359375,
      "logps/ref_chosen": -277.9366149902344,
      "logps/ref_rejected": -262.689453125,
      "logps/rejected": -336.01214599609375,
      "loss": 4.7224,
      "margin_dpo/margin_mean": 44.611412048339844,
      "margin_dpo/margin_std": 68.27713012695312,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -14.663717269897461,
      "KL/mean": -38.8934440612793,
      "KL/rejected_KL_mean": -63.123172760009766,
      "KL/std": 57.958457946777344,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.005946532357484102,
      "fcm_dpo/delta": -0.029838802292943,
      "fcm_dpo/margin": 48.459449768066406,
      "fcm_dpo/q_t": 0.43051955103874207,
      "grad_norm": 21.11540412902832,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": 1.6155638694763184,
      "logits/rejected": 1.615441083908081,
      "logps/chosen": -285.1086120605469,
      "logps/ref_chosen": -270.4449157714844,
      "logps/ref_rejected": -259.5942077636719,
      "logps/rejected": -322.7173767089844,
      "loss": 4.6255,
      "margin_dpo/margin_mean": 48.459449768066406,
      "margin_dpo/margin_std": 61.960052490234375,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -25.1865177154541,
      "KL/mean": -45.622703552246094,
      "KL/rejected_KL_mean": -66.05889892578125,
      "KL/std": 59.287994384765625,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.005864188075065613,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 40.872371673583984,
      "fcm_dpo/q_t": 0.4427345097064972,
      "grad_norm": 45.67393112182617,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": 1.2007468938827515,
      "logits/rejected": 1.2680995464324951,
      "logps/chosen": -295.02642822265625,
      "logps/ref_chosen": -269.83990478515625,
      "logps/ref_rejected": -246.33746337890625,
      "logps/rejected": -312.3963317871094,
      "loss": 4.8274,
      "margin_dpo/margin_mean": 40.872371673583984,
      "margin_dpo/margin_std": 72.4842529296875,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -19.33561134338379,
      "KL/mean": -41.0200309753418,
      "KL/rejected_KL_mean": -62.70444869995117,
      "KL/std": 57.63920974731445,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.005864188075065613,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.368831634521484,
      "fcm_dpo/q_t": 0.4391006529331207,
      "grad_norm": 27.378725051879883,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": 1.3269283771514893,
      "logits/rejected": 1.3103455305099487,
      "logps/chosen": -316.86956787109375,
      "logps/ref_chosen": -297.533935546875,
      "logps/ref_rejected": -270.3407897949219,
      "logps/rejected": -333.04522705078125,
      "loss": 4.7348,
      "margin_dpo/margin_mean": 43.36882781982422,
      "margin_dpo/margin_std": 63.29307174682617,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -22.47580337524414,
      "KL/mean": -43.593929290771484,
      "KL/rejected_KL_mean": -64.71205139160156,
      "KL/std": 58.90170669555664,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.005864188075065613,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.236236572265625,
      "fcm_dpo/q_t": 0.4404001235961914,
      "grad_norm": 31.401159286499023,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": 0.9612157940864563,
      "logits/rejected": 0.9943892359733582,
      "logps/chosen": -262.8146667480469,
      "logps/ref_chosen": -240.33883666992188,
      "logps/ref_rejected": -220.71717834472656,
      "logps/rejected": -285.4292297363281,
      "loss": 4.7918,
      "margin_dpo/margin_mean": 42.23624038696289,
      "margin_dpo/margin_std": 71.0365982055664,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -19.979633331298828,
      "KL/mean": -44.83853530883789,
      "KL/rejected_KL_mean": -69.69743347167969,
      "KL/std": 61.59584045410156,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.005864188075065613,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.71778869628906,
      "fcm_dpo/q_t": 0.4301414489746094,
      "grad_norm": 34.517208099365234,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 1.6218408346176147,
      "logits/rejected": 1.6305378675460815,
      "logps/chosen": -328.87274169921875,
      "logps/ref_chosen": -308.8930969238281,
      "logps/ref_rejected": -278.47869873046875,
      "logps/rejected": -348.1761169433594,
      "loss": 4.6664,
      "margin_dpo/margin_mean": 49.71778869628906,
      "margin_dpo/margin_std": 75.52421569824219,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -20.789073944091797,
      "KL/mean": -44.05741500854492,
      "KL/rejected_KL_mean": -67.32574462890625,
      "KL/std": 57.98163604736328,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.005836832337081432,
      "fcm_dpo/delta": -0.015647105872631073,
      "fcm_dpo/margin": 46.536678314208984,
      "fcm_dpo/q_t": 0.4349617063999176,
      "grad_norm": 39.25949478149414,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.2480366230010986,
      "logits/rejected": 1.2579851150512695,
      "logps/chosen": -296.21728515625,
      "logps/ref_chosen": -275.42822265625,
      "logps/ref_rejected": -251.50906372070312,
      "logps/rejected": -318.8348083496094,
      "loss": 4.6808,
      "margin_dpo/margin_mean": 46.53667068481445,
      "margin_dpo/margin_std": 64.66627502441406,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -18.950950622558594,
      "KL/mean": -40.6681022644043,
      "KL/rejected_KL_mean": -62.3852653503418,
      "KL/std": 58.65740203857422,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.005791239440441132,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.434303283691406,
      "fcm_dpo/q_t": 0.43974122405052185,
      "grad_norm": 41.3842887878418,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": 1.3018513917922974,
      "logits/rejected": 1.3098012208938599,
      "logps/chosen": -308.3337097167969,
      "logps/ref_chosen": -289.38275146484375,
      "logps/ref_rejected": -274.895751953125,
      "logps/rejected": -337.281005859375,
      "loss": 4.758,
      "margin_dpo/margin_mean": 43.434303283691406,
      "margin_dpo/margin_std": 65.42008209228516,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -19.592041015625,
      "KL/mean": -43.52835464477539,
      "KL/rejected_KL_mean": -67.46467590332031,
      "KL/std": 59.04290008544922,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.005741673056036234,
      "fcm_dpo/delta": -0.012287174351513386,
      "fcm_dpo/margin": 47.87264633178711,
      "fcm_dpo/q_t": 0.4342363774776459,
      "grad_norm": 37.30966567993164,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": 1.5419025421142578,
      "logits/rejected": 1.6667585372924805,
      "logps/chosen": -304.58050537109375,
      "logps/ref_chosen": -284.98846435546875,
      "logps/ref_rejected": -273.5077819824219,
      "logps/rejected": -340.97247314453125,
      "loss": 4.6966,
      "margin_dpo/margin_mean": 47.87264633178711,
      "margin_dpo/margin_std": 71.67526245117188,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -21.081722259521484,
      "KL/mean": -46.71761703491211,
      "KL/rejected_KL_mean": -72.353515625,
      "KL/std": 62.425743103027344,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.005734592210501432,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.27178955078125,
      "fcm_dpo/q_t": 0.42960143089294434,
      "grad_norm": 29.328458786010742,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": 1.4181828498840332,
      "logits/rejected": 1.486657977104187,
      "logps/chosen": -283.7613220214844,
      "logps/ref_chosen": -262.6795959472656,
      "logps/ref_rejected": -236.13792419433594,
      "logps/rejected": -308.491455078125,
      "loss": 4.6218,
      "margin_dpo/margin_mean": 51.27178955078125,
      "margin_dpo/margin_std": 69.71764373779297,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -17.058387756347656,
      "KL/mean": -45.88899230957031,
      "KL/rejected_KL_mean": -74.71958923339844,
      "KL/std": 60.04632568359375,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.005677453707903624,
      "fcm_dpo/delta": -0.012517203576862812,
      "fcm_dpo/margin": 57.66119384765625,
      "fcm_dpo/q_t": 0.4215719997882843,
      "grad_norm": 21.834104537963867,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": 1.342179536819458,
      "logits/rejected": 1.456270456314087,
      "logps/chosen": -263.4578857421875,
      "logps/ref_chosen": -246.39950561523438,
      "logps/ref_rejected": -247.4836883544922,
      "logps/rejected": -322.2032775878906,
      "loss": 4.4947,
      "margin_dpo/margin_mean": 57.66119384765625,
      "margin_dpo/margin_std": 68.3296127319336,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -15.085214614868164,
      "KL/mean": -42.911338806152344,
      "KL/rejected_KL_mean": -70.73747253417969,
      "KL/std": 61.15802001953125,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.005677453707903624,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.652259826660156,
      "fcm_dpo/q_t": 0.42475026845932007,
      "grad_norm": 26.14124298095703,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": 1.5112122297286987,
      "logits/rejected": 1.631365418434143,
      "logps/chosen": -304.0052795410156,
      "logps/ref_chosen": -288.9200439453125,
      "logps/ref_rejected": -291.69305419921875,
      "logps/rejected": -362.4305114746094,
      "loss": 4.5492,
      "margin_dpo/margin_mean": 55.652252197265625,
      "margin_dpo/margin_std": 71.18743133544922,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -27.990076065063477,
      "KL/mean": -46.711143493652344,
      "KL/rejected_KL_mean": -65.43218994140625,
      "KL/std": 59.580055236816406,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.005677453707903624,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.4421272277832,
      "fcm_dpo/q_t": 0.4489213228225708,
      "grad_norm": 51.173431396484375,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": 1.1170066595077515,
      "logits/rejected": 1.1574803590774536,
      "logps/chosen": -285.0749816894531,
      "logps/ref_chosen": -257.08489990234375,
      "logps/ref_rejected": -251.61532592773438,
      "logps/rejected": -317.0474853515625,
      "loss": 4.9336,
      "margin_dpo/margin_mean": 37.4421272277832,
      "margin_dpo/margin_std": 76.60790252685547,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -32.10133361816406,
      "KL/mean": -49.30353546142578,
      "KL/rejected_KL_mean": -66.5057373046875,
      "KL/std": 61.6645622253418,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.005677453707903624,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.40441131591797,
      "fcm_dpo/q_t": 0.45329347252845764,
      "grad_norm": 33.94520950317383,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": 1.2508306503295898,
      "logits/rejected": 1.2060761451721191,
      "logps/chosen": -292.84271240234375,
      "logps/ref_chosen": -260.7413635253906,
      "logps/ref_rejected": -227.4325408935547,
      "logps/rejected": -293.93829345703125,
      "loss": 5.0057,
      "margin_dpo/margin_mean": 34.40441131591797,
      "margin_dpo/margin_std": 76.660888671875,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -21.609275817871094,
      "KL/mean": -48.747894287109375,
      "KL/rejected_KL_mean": -75.88652038574219,
      "KL/std": 63.80519104003906,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.005626695696264505,
      "fcm_dpo/delta": -0.014989969320595264,
      "fcm_dpo/margin": 54.277244567871094,
      "fcm_dpo/q_t": 0.42698442935943604,
      "grad_norm": 34.64128875732422,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": 1.2201496362686157,
      "logits/rejected": 1.3139028549194336,
      "logps/chosen": -324.5567932128906,
      "logps/ref_chosen": -302.947509765625,
      "logps/ref_rejected": -294.1771545410156,
      "logps/rejected": -370.06365966796875,
      "loss": 4.578,
      "margin_dpo/margin_mean": 54.277244567871094,
      "margin_dpo/margin_std": 69.78999328613281,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -23.073413848876953,
      "KL/mean": -46.933101654052734,
      "KL/rejected_KL_mean": -70.79279327392578,
      "KL/std": 60.593101501464844,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.005609776359051466,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.71937942504883,
      "fcm_dpo/q_t": 0.4360616207122803,
      "grad_norm": 34.35310745239258,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 1.196717619895935,
      "logits/rejected": 1.2713780403137207,
      "logps/chosen": -280.9715270996094,
      "logps/ref_chosen": -257.8981018066406,
      "logps/ref_rejected": -244.9084014892578,
      "logps/rejected": -315.701171875,
      "loss": 4.725,
      "margin_dpo/margin_mean": 47.71937942504883,
      "margin_dpo/margin_std": 74.8751449584961,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -32.91215896606445,
      "KL/mean": -57.46163558959961,
      "KL/rejected_KL_mean": -82.0111083984375,
      "KL/std": 63.165016174316406,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.005609776359051466,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.09895324707031,
      "fcm_dpo/q_t": 0.4343499541282654,
      "grad_norm": 28.464889526367188,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 1.3321528434753418,
      "logits/rejected": 1.3985906839370728,
      "logps/chosen": -323.10687255859375,
      "logps/ref_chosen": -290.1946716308594,
      "logps/ref_rejected": -290.98883056640625,
      "logps/rejected": -372.9999084472656,
      "loss": 4.6721,
      "margin_dpo/margin_mean": 49.09895324707031,
      "margin_dpo/margin_std": 67.05542755126953,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -30.710840225219727,
      "KL/mean": -52.37788391113281,
      "KL/rejected_KL_mean": -74.04493713378906,
      "KL/std": 64.22257232666016,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.005609776359051466,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.3341178894043,
      "fcm_dpo/q_t": 0.4422397017478943,
      "grad_norm": 39.205135345458984,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": 1.315058946609497,
      "logits/rejected": 1.3389642238616943,
      "logps/chosen": -339.6282958984375,
      "logps/ref_chosen": -308.91741943359375,
      "logps/ref_rejected": -272.2177734375,
      "logps/rejected": -346.2626953125,
      "loss": 4.8314,
      "margin_dpo/margin_mean": 43.334110260009766,
      "margin_dpo/margin_std": 78.94495391845703,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -31.399980545043945,
      "KL/mean": -55.858741760253906,
      "KL/rejected_KL_mean": -80.31752014160156,
      "KL/std": 60.98528289794922,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.005579269025474787,
      "fcm_dpo/delta": -0.018260065466165543,
      "fcm_dpo/margin": 48.917545318603516,
      "fcm_dpo/q_t": 0.434812992811203,
      "grad_norm": 33.92677307128906,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 1.477236270904541,
      "logits/rejected": 1.5440239906311035,
      "logps/chosen": -279.4219970703125,
      "logps/ref_chosen": -248.02200317382812,
      "logps/ref_rejected": -234.78541564941406,
      "logps/rejected": -315.10296630859375,
      "loss": 4.7093,
      "margin_dpo/margin_mean": 48.91754913330078,
      "margin_dpo/margin_std": 74.09596252441406,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -34.34777069091797,
      "KL/mean": -58.19141387939453,
      "KL/rejected_KL_mean": -82.0350341796875,
      "KL/std": 65.47592163085938,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.0054787700064480305,
      "fcm_dpo/delta": -0.012896896339952946,
      "fcm_dpo/margin": 47.687259674072266,
      "fcm_dpo/q_t": 0.43810999393463135,
      "grad_norm": 28.981847763061523,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": 1.1771256923675537,
      "logits/rejected": 1.1762566566467285,
      "logps/chosen": -298.8370361328125,
      "logps/ref_chosen": -264.4892578125,
      "logps/ref_rejected": -255.43092346191406,
      "logps/rejected": -337.4659423828125,
      "loss": 4.7897,
      "margin_dpo/margin_mean": 47.687259674072266,
      "margin_dpo/margin_std": 83.66376495361328,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -32.86461639404297,
      "KL/mean": -51.0294303894043,
      "KL/rejected_KL_mean": -69.19422912597656,
      "KL/std": 61.58562088012695,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.005457767751067877,
      "fcm_dpo/delta": -0.012775721028447151,
      "fcm_dpo/margin": 36.32962417602539,
      "fcm_dpo/q_t": 0.4523497223854065,
      "grad_norm": 46.78046798706055,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": 1.5635062456130981,
      "logits/rejected": 1.5916424989700317,
      "logps/chosen": -334.33331298828125,
      "logps/ref_chosen": -301.46868896484375,
      "logps/ref_rejected": -274.2380676269531,
      "logps/rejected": -343.4323425292969,
      "loss": 4.9593,
      "margin_dpo/margin_mean": 36.32962417602539,
      "margin_dpo/margin_std": 70.66097259521484,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -35.857757568359375,
      "KL/mean": -54.82905197143555,
      "KL/rejected_KL_mean": -73.80032348632812,
      "KL/std": 61.98818588256836,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.005416038446128368,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.94257354736328,
      "fcm_dpo/q_t": 0.4508640170097351,
      "grad_norm": 43.17778778076172,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": 1.209406852722168,
      "logits/rejected": 1.2768983840942383,
      "logps/chosen": -302.41680908203125,
      "logps/ref_chosen": -266.5590515136719,
      "logps/ref_rejected": -280.12066650390625,
      "logps/rejected": -353.9210205078125,
      "loss": 4.9457,
      "margin_dpo/margin_mean": 37.94257354736328,
      "margin_dpo/margin_std": 77.65943908691406,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -29.374303817749023,
      "KL/mean": -51.704689025878906,
      "KL/rejected_KL_mean": -74.03507232666016,
      "KL/std": 59.30611038208008,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.005416038446128368,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.6607666015625,
      "fcm_dpo/q_t": 0.44177764654159546,
      "grad_norm": 27.379301071166992,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": 1.2560970783233643,
      "logits/rejected": 1.317993402481079,
      "logps/chosen": -299.31060791015625,
      "logps/ref_chosen": -269.93634033203125,
      "logps/ref_rejected": -232.32330322265625,
      "logps/rejected": -306.3583679199219,
      "loss": 4.8023,
      "margin_dpo/margin_mean": 44.66077423095703,
      "margin_dpo/margin_std": 73.94287872314453,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -32.16948318481445,
      "KL/mean": -53.21517562866211,
      "KL/rejected_KL_mean": -74.26087188720703,
      "KL/std": 62.1978759765625,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.005416038446128368,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.09139633178711,
      "fcm_dpo/q_t": 0.4456709027290344,
      "grad_norm": 33.85889434814453,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": 1.3180675506591797,
      "logits/rejected": 1.2743793725967407,
      "logps/chosen": -316.2149658203125,
      "logps/ref_chosen": -284.04547119140625,
      "logps/ref_rejected": -261.19378662109375,
      "logps/rejected": -335.45465087890625,
      "loss": 4.8586,
      "margin_dpo/margin_mean": 42.09139633178711,
      "margin_dpo/margin_std": 76.00890350341797,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -41.040733337402344,
      "KL/mean": -56.15140914916992,
      "KL/rejected_KL_mean": -71.2620849609375,
      "KL/std": 63.549400329589844,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.005416038446128368,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 30.22136116027832,
      "fcm_dpo/q_t": 0.46069738268852234,
      "grad_norm": 32.51848602294922,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": 1.2264560461044312,
      "logits/rejected": 1.1854907274246216,
      "logps/chosen": -364.3219909667969,
      "logps/ref_chosen": -323.28125,
      "logps/ref_rejected": -274.2093200683594,
      "logps/rejected": -345.471435546875,
      "loss": 5.0952,
      "margin_dpo/margin_mean": 30.22136116027832,
      "margin_dpo/margin_std": 76.22372436523438,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -31.211227416992188,
      "KL/mean": -50.22416687011719,
      "KL/rejected_KL_mean": -69.23710632324219,
      "KL/std": 62.107967376708984,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.005416038446128368,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.02587890625,
      "fcm_dpo/q_t": 0.4506572186946869,
      "grad_norm": 34.05064392089844,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 1.3549402952194214,
      "logits/rejected": 1.324060320854187,
      "logps/chosen": -348.6466064453125,
      "logps/ref_chosen": -317.4353942871094,
      "logps/ref_rejected": -309.32403564453125,
      "logps/rejected": -378.5611572265625,
      "loss": 4.9208,
      "margin_dpo/margin_mean": 38.025882720947266,
      "margin_dpo/margin_std": 71.37504577636719,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -28.189353942871094,
      "KL/mean": -56.78004455566406,
      "KL/rejected_KL_mean": -85.37073516845703,
      "KL/std": 67.38497924804688,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.005391877610236406,
      "fcm_dpo/delta": -0.014958597719669342,
      "fcm_dpo/margin": 57.18138122558594,
      "fcm_dpo/q_t": 0.4261842370033264,
      "grad_norm": 26.62029266357422,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 1.0836284160614014,
      "logits/rejected": 1.2414697408676147,
      "logps/chosen": -293.6976013183594,
      "logps/ref_chosen": -265.50823974609375,
      "logps/ref_rejected": -275.1380310058594,
      "logps/rejected": -360.50872802734375,
      "loss": 4.5805,
      "margin_dpo/margin_mean": 57.1813850402832,
      "margin_dpo/margin_std": 75.61519622802734,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -22.97232437133789,
      "KL/mean": -44.694576263427734,
      "KL/rejected_KL_mean": -66.41683197021484,
      "KL/std": 62.39437484741211,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.00532128568738699,
      "fcm_dpo/delta": -0.009480083361268044,
      "fcm_dpo/margin": 43.44450759887695,
      "fcm_dpo/q_t": 0.44430240988731384,
      "grad_norm": 30.72490119934082,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": 1.237911581993103,
      "logits/rejected": 1.3639124631881714,
      "logps/chosen": -272.8736572265625,
      "logps/ref_chosen": -249.9013214111328,
      "logps/ref_rejected": -274.58245849609375,
      "logps/rejected": -340.9992980957031,
      "loss": 4.8312,
      "margin_dpo/margin_mean": 43.44450378417969,
      "margin_dpo/margin_std": 71.38679504394531,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -33.13412857055664,
      "KL/mean": -56.526268005371094,
      "KL/rejected_KL_mean": -79.91839599609375,
      "KL/std": 67.37076568603516,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.005311177112162113,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.784263610839844,
      "fcm_dpo/q_t": 0.4406266510486603,
      "grad_norm": 39.27650833129883,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": 1.5158920288085938,
      "logits/rejected": 1.4975374937057495,
      "logps/chosen": -329.4400634765625,
      "logps/ref_chosen": -296.30596923828125,
      "logps/ref_rejected": -268.0640869140625,
      "logps/rejected": -347.98248291015625,
      "loss": 4.7767,
      "margin_dpo/margin_mean": 46.784263610839844,
      "margin_dpo/margin_std": 74.95236206054688,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -28.611722946166992,
      "KL/mean": -49.32271957397461,
      "KL/rejected_KL_mean": -70.03370666503906,
      "KL/std": 64.34965515136719,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.005311177112162113,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.42200469970703,
      "fcm_dpo/q_t": 0.44721347093582153,
      "grad_norm": 34.313045501708984,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": 1.3445448875427246,
      "logits/rejected": 1.3951084613800049,
      "logps/chosen": -262.49468994140625,
      "logps/ref_chosen": -233.8829803466797,
      "logps/ref_rejected": -224.14334106445312,
      "logps/rejected": -294.17706298828125,
      "loss": 4.8782,
      "margin_dpo/margin_mean": 41.42200469970703,
      "margin_dpo/margin_std": 74.72012329101562,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -31.742862701416016,
      "KL/mean": -54.817543029785156,
      "KL/rejected_KL_mean": -77.89221954345703,
      "KL/std": 64.08187866210938,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.005311177112162113,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.149349212646484,
      "fcm_dpo/q_t": 0.4415561556816101,
      "grad_norm": 26.78761863708496,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": 1.3500956296920776,
      "logits/rejected": 1.3645298480987549,
      "logps/chosen": -275.85418701171875,
      "logps/ref_chosen": -244.111328125,
      "logps/ref_rejected": -240.18373107910156,
      "logps/rejected": -318.075927734375,
      "loss": 4.8057,
      "margin_dpo/margin_mean": 46.149349212646484,
      "margin_dpo/margin_std": 79.11995697021484,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -26.14333724975586,
      "KL/mean": -56.82640075683594,
      "KL/rejected_KL_mean": -87.50946044921875,
      "KL/std": 66.20918273925781,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.005251799710094929,
      "fcm_dpo/delta": -0.020449630916118622,
      "fcm_dpo/margin": 61.36613464355469,
      "fcm_dpo/q_t": 0.42272597551345825,
      "grad_norm": 29.07663917541504,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 1.2707390785217285,
      "logits/rejected": 1.4382139444351196,
      "logps/chosen": -310.8238830566406,
      "logps/ref_chosen": -284.6805725097656,
      "logps/ref_rejected": -282.1923828125,
      "logps/rejected": -369.7018127441406,
      "loss": 4.5354,
      "margin_dpo/margin_mean": 61.36613464355469,
      "margin_dpo/margin_std": 78.68840026855469,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -28.409433364868164,
      "KL/mean": -53.57111740112305,
      "KL/rejected_KL_mean": -78.73281860351562,
      "KL/std": 64.41624450683594,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.005189407151192427,
      "fcm_dpo/delta": -0.009768201969563961,
      "fcm_dpo/margin": 50.323368072509766,
      "fcm_dpo/q_t": 0.4374025762081146,
      "grad_norm": 33.980499267578125,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": 1.4532768726348877,
      "logits/rejected": 1.4592406749725342,
      "logps/chosen": -308.6380615234375,
      "logps/ref_chosen": -280.2286376953125,
      "logps/ref_rejected": -280.8172912597656,
      "logps/rejected": -359.55010986328125,
      "loss": 4.7239,
      "margin_dpo/margin_mean": 50.323368072509766,
      "margin_dpo/margin_std": 74.42920684814453,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -22.42514991760254,
      "KL/mean": -50.64270782470703,
      "KL/rejected_KL_mean": -78.86026000976562,
      "KL/std": 67.84736633300781,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.005161489360034466,
      "fcm_dpo/delta": -0.014768172055482864,
      "fcm_dpo/margin": 56.43511199951172,
      "fcm_dpo/q_t": 0.4309054911136627,
      "grad_norm": 21.244556427001953,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": 1.1723721027374268,
      "logits/rejected": 1.1708648204803467,
      "logps/chosen": -319.42547607421875,
      "logps/ref_chosen": -297.0003356933594,
      "logps/ref_rejected": -279.1561279296875,
      "logps/rejected": -358.0163879394531,
      "loss": 4.6494,
      "margin_dpo/margin_mean": 56.43511199951172,
      "margin_dpo/margin_std": 81.22370147705078,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -34.386592864990234,
      "KL/mean": -56.60679626464844,
      "KL/rejected_KL_mean": -78.82699584960938,
      "KL/std": 66.06631469726562,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.005123433191329241,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.44041442871094,
      "fcm_dpo/q_t": 0.44512203335762024,
      "grad_norm": 35.72551345825195,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": 1.2774916887283325,
      "logits/rejected": 1.3685493469238281,
      "logps/chosen": -305.8580322265625,
      "logps/ref_chosen": -271.471435546875,
      "logps/ref_rejected": -268.0870056152344,
      "logps/rejected": -346.91400146484375,
      "loss": 4.8207,
      "margin_dpo/margin_mean": 44.44041061401367,
      "margin_dpo/margin_std": 68.04814910888672,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -37.048614501953125,
      "KL/mean": -59.78719711303711,
      "KL/rejected_KL_mean": -82.52578735351562,
      "KL/std": 66.19276428222656,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.005123433191329241,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.47718048095703,
      "fcm_dpo/q_t": 0.4444907009601593,
      "grad_norm": 36.775123596191406,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 1.4403003454208374,
      "logits/rejected": 1.4420623779296875,
      "logps/chosen": -343.4065246582031,
      "logps/ref_chosen": -306.35784912109375,
      "logps/ref_rejected": -268.64776611328125,
      "logps/rejected": -351.17352294921875,
      "loss": 4.8503,
      "margin_dpo/margin_mean": 45.477176666259766,
      "margin_dpo/margin_std": 82.59898376464844,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -36.03577423095703,
      "KL/mean": -58.29108428955078,
      "KL/rejected_KL_mean": -80.54641723632812,
      "KL/std": 66.1821517944336,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.005123433191329241,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.51064682006836,
      "fcm_dpo/q_t": 0.44533318281173706,
      "grad_norm": 39.30177688598633,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.2909184694290161,
      "logits/rejected": 1.3548340797424316,
      "logps/chosen": -306.02825927734375,
      "logps/ref_chosen": -269.9924621582031,
      "logps/ref_rejected": -271.3135681152344,
      "logps/rejected": -351.8600158691406,
      "loss": 4.8458,
      "margin_dpo/margin_mean": 44.51064682006836,
      "margin_dpo/margin_std": 77.7679443359375,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -33.696937561035156,
      "KL/mean": -60.94989776611328,
      "KL/rejected_KL_mean": -88.20285034179688,
      "KL/std": 65.50361633300781,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.005087494384497404,
      "fcm_dpo/delta": -0.008799251168966293,
      "fcm_dpo/margin": 54.50590133666992,
      "fcm_dpo/q_t": 0.43326014280319214,
      "grad_norm": 27.54530143737793,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": 1.1357678174972534,
      "logits/rejected": 1.2110532522201538,
      "logps/chosen": -294.1576232910156,
      "logps/ref_chosen": -260.460693359375,
      "logps/ref_rejected": -254.2005615234375,
      "logps/rejected": -342.40338134765625,
      "loss": 4.6788,
      "margin_dpo/margin_mean": 54.50590133666992,
      "margin_dpo/margin_std": 78.28961181640625,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -33.242431640625,
      "KL/mean": -58.325191497802734,
      "KL/rejected_KL_mean": -83.4079360961914,
      "KL/std": 65.34207916259766,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.005087494384497404,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.16551971435547,
      "fcm_dpo/q_t": 0.43847253918647766,
      "grad_norm": 34.07196044921875,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": 1.091488003730774,
      "logits/rejected": 1.1286163330078125,
      "logps/chosen": -329.33868408203125,
      "logps/ref_chosen": -296.09625244140625,
      "logps/ref_rejected": -264.7806701660156,
      "logps/rejected": -348.1885681152344,
      "loss": 4.765,
      "margin_dpo/margin_mean": 50.16551971435547,
      "margin_dpo/margin_std": 79.58503723144531,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -46.218353271484375,
      "KL/mean": -68.04375457763672,
      "KL/rejected_KL_mean": -89.86915588378906,
      "KL/std": 69.38544464111328,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.005087494384497404,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.65080261230469,
      "fcm_dpo/q_t": 0.44662004709243774,
      "grad_norm": 27.89470863342285,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": 1.2778105735778809,
      "logits/rejected": 1.2839608192443848,
      "logps/chosen": -328.06781005859375,
      "logps/ref_chosen": -281.8494873046875,
      "logps/ref_rejected": -234.3673858642578,
      "logps/rejected": -324.2365417480469,
      "loss": 4.8674,
      "margin_dpo/margin_mean": 43.65080261230469,
      "margin_dpo/margin_std": 78.105224609375,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -28.081260681152344,
      "KL/mean": -59.97367477416992,
      "KL/rejected_KL_mean": -91.86610412597656,
      "KL/std": 69.35157012939453,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.005011059809476137,
      "fcm_dpo/delta": -0.029669493436813354,
      "fcm_dpo/margin": 63.78485107421875,
      "fcm_dpo/q_t": 0.42336389422416687,
      "grad_norm": 32.06255340576172,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": 1.252783179283142,
      "logits/rejected": 1.3579366207122803,
      "logps/chosen": -276.1240234375,
      "logps/ref_chosen": -248.0427703857422,
      "logps/ref_rejected": -263.464599609375,
      "logps/rejected": -355.3306884765625,
      "loss": 4.5363,
      "margin_dpo/margin_mean": 63.78484344482422,
      "margin_dpo/margin_std": 78.99354553222656,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -27.891752243041992,
      "KL/mean": -55.420413970947266,
      "KL/rejected_KL_mean": -82.94906616210938,
      "KL/std": 66.6956558227539,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.004968161229044199,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.05733108520508,
      "fcm_dpo/q_t": 0.43421506881713867,
      "grad_norm": 31.329551696777344,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": 1.3423004150390625,
      "logits/rejected": 1.2622957229614258,
      "logps/chosen": -339.1776428222656,
      "logps/ref_chosen": -311.2859191894531,
      "logps/ref_rejected": -268.4949645996094,
      "logps/rejected": -351.44403076171875,
      "loss": 4.6803,
      "margin_dpo/margin_mean": 55.05733108520508,
      "margin_dpo/margin_std": 76.90007019042969,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -34.48876953125,
      "KL/mean": -64.31768035888672,
      "KL/rejected_KL_mean": -94.1466064453125,
      "KL/std": 68.76074981689453,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.004962650593370199,
      "fcm_dpo/delta": -0.011140748858451843,
      "fcm_dpo/margin": 59.6578483581543,
      "fcm_dpo/q_t": 0.4284789562225342,
      "grad_norm": 42.2656135559082,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": 1.3902316093444824,
      "logits/rejected": 1.4046940803527832,
      "logps/chosen": -304.70123291015625,
      "logps/ref_chosen": -270.21246337890625,
      "logps/ref_rejected": -239.1932830810547,
      "logps/rejected": -333.33990478515625,
      "loss": 4.5946,
      "margin_dpo/margin_mean": 59.6578483581543,
      "margin_dpo/margin_std": 74.0295639038086,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -41.20484161376953,
      "KL/mean": -62.06786346435547,
      "KL/rejected_KL_mean": -82.93087005615234,
      "KL/std": 66.6594467163086,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.0049240789376199245,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.72602844238281,
      "fcm_dpo/q_t": 0.4505127966403961,
      "grad_norm": 23.306236267089844,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": 1.347574234008789,
      "logits/rejected": 1.3503193855285645,
      "logps/chosen": -329.7276611328125,
      "logps/ref_chosen": -288.5228271484375,
      "logps/ref_rejected": -253.90597534179688,
      "logps/rejected": -336.83685302734375,
      "loss": 4.9235,
      "margin_dpo/margin_mean": 41.72602462768555,
      "margin_dpo/margin_std": 78.51648712158203,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -39.73585510253906,
      "KL/mean": -69.02242279052734,
      "KL/rejected_KL_mean": -98.30899047851562,
      "KL/std": 70.30194091796875,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.004919067025184631,
      "fcm_dpo/delta": -0.01021986361593008,
      "fcm_dpo/margin": 58.57312774658203,
      "fcm_dpo/q_t": 0.43143999576568604,
      "grad_norm": 61.56427001953125,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 1.3287395238876343,
      "logits/rejected": 1.4567824602127075,
      "logps/chosen": -311.6305847167969,
      "logps/ref_chosen": -271.8947448730469,
      "logps/ref_rejected": -260.5709228515625,
      "logps/rejected": -358.8798828125,
      "loss": 4.6558,
      "margin_dpo/margin_mean": 58.57312774658203,
      "margin_dpo/margin_std": 85.02222442626953,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -44.27870178222656,
      "KL/mean": -68.59642791748047,
      "KL/rejected_KL_mean": -92.91413879394531,
      "KL/std": 65.83905792236328,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.004883984103798866,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.63543701171875,
      "fcm_dpo/q_t": 0.4428134560585022,
      "grad_norm": 30.54885482788086,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 1.2778282165527344,
      "logits/rejected": 1.3932183980941772,
      "logps/chosen": -314.4541931152344,
      "logps/ref_chosen": -270.17547607421875,
      "logps/ref_rejected": -257.6781311035156,
      "logps/rejected": -350.59228515625,
      "loss": 4.7807,
      "margin_dpo/margin_mean": 48.63543701171875,
      "margin_dpo/margin_std": 71.989501953125,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -41.23306655883789,
      "KL/mean": -67.23724365234375,
      "KL/rejected_KL_mean": -93.24141693115234,
      "KL/std": 69.92778015136719,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.004883984103798866,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.00833511352539,
      "fcm_dpo/q_t": 0.4390377402305603,
      "grad_norm": 24.967208862304688,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 1.3749592304229736,
      "logits/rejected": 1.5341527462005615,
      "logps/chosen": -304.99530029296875,
      "logps/ref_chosen": -263.76226806640625,
      "logps/ref_rejected": -263.8758850097656,
      "logps/rejected": -357.1172790527344,
      "loss": 4.7802,
      "margin_dpo/margin_mean": 52.00833511352539,
      "margin_dpo/margin_std": 86.27284240722656,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -37.970787048339844,
      "KL/mean": -64.60653686523438,
      "KL/rejected_KL_mean": -91.24229431152344,
      "KL/std": 66.20455932617188,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.004842543043196201,
      "fcm_dpo/delta": -0.01708666794002056,
      "fcm_dpo/margin": 53.27150344848633,
      "fcm_dpo/q_t": 0.43771541118621826,
      "grad_norm": 52.56047058105469,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": 1.2216007709503174,
      "logits/rejected": 1.1850037574768066,
      "logps/chosen": -312.6431884765625,
      "logps/ref_chosen": -274.67242431640625,
      "logps/ref_rejected": -242.1553955078125,
      "logps/rejected": -333.397705078125,
      "loss": 4.7382,
      "margin_dpo/margin_mean": 53.27151107788086,
      "margin_dpo/margin_std": 79.23887634277344,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -40.85597229003906,
      "KL/mean": -65.85124206542969,
      "KL/rejected_KL_mean": -90.84651947021484,
      "KL/std": 70.18658447265625,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.004805717151612043,
      "fcm_dpo/delta": -0.025075441226363182,
      "fcm_dpo/margin": 49.99053955078125,
      "fcm_dpo/q_t": 0.4422350823879242,
      "grad_norm": 33.14799499511719,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": 1.4759411811828613,
      "logits/rejected": 1.5708470344543457,
      "logps/chosen": -336.0438537597656,
      "logps/ref_chosen": -295.1878662109375,
      "logps/ref_rejected": -271.3431091308594,
      "logps/rejected": -362.1896667480469,
      "loss": 4.8099,
      "margin_dpo/margin_mean": 49.99053955078125,
      "margin_dpo/margin_std": 81.45989990234375,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -39.617103576660156,
      "KL/mean": -64.43018341064453,
      "KL/rejected_KL_mean": -89.24327087402344,
      "KL/std": 68.8984375,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.62617492675781,
      "fcm_dpo/q_t": 0.4434923827648163,
      "grad_norm": 43.26030731201172,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": 1.1376690864562988,
      "logits/rejected": 1.1755272150039673,
      "logps/chosen": -329.38494873046875,
      "logps/ref_chosen": -289.767822265625,
      "logps/ref_rejected": -267.0824279785156,
      "logps/rejected": -356.3257141113281,
      "loss": 4.825,
      "margin_dpo/margin_mean": 49.62617492675781,
      "margin_dpo/margin_std": 83.20230102539062,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -34.50761795043945,
      "KL/mean": -63.04988098144531,
      "KL/rejected_KL_mean": -91.59214782714844,
      "KL/std": 69.68960571289062,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.084537506103516,
      "fcm_dpo/q_t": 0.435499906539917,
      "grad_norm": 27.517866134643555,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": 1.5873266458511353,
      "logits/rejected": 1.6092138290405273,
      "logps/chosen": -332.9266662597656,
      "logps/ref_chosen": -298.4190673828125,
      "logps/ref_rejected": -266.64447021484375,
      "logps/rejected": -358.2366027832031,
      "loss": 4.6906,
      "margin_dpo/margin_mean": 57.084537506103516,
      "margin_dpo/margin_std": 82.00359344482422,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -41.58988952636719,
      "KL/mean": -71.07437133789062,
      "KL/rejected_KL_mean": -100.55885314941406,
      "KL/std": 71.46343231201172,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.96895980834961,
      "fcm_dpo/q_t": 0.43340644240379333,
      "grad_norm": 24.703933715820312,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": 1.307612419128418,
      "logits/rejected": 1.3023402690887451,
      "logps/chosen": -343.81976318359375,
      "logps/ref_chosen": -302.2298583984375,
      "logps/ref_rejected": -272.7162170410156,
      "logps/rejected": -373.2750549316406,
      "loss": 4.6629,
      "margin_dpo/margin_mean": 58.96895980834961,
      "margin_dpo/margin_std": 82.4745101928711,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -32.971656799316406,
      "KL/mean": -60.05366134643555,
      "KL/rejected_KL_mean": -87.13566589355469,
      "KL/std": 68.11445617675781,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.16400909423828,
      "fcm_dpo/q_t": 0.4384688138961792,
      "grad_norm": 38.4249153137207,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": 1.3371608257293701,
      "logits/rejected": 1.309743881225586,
      "logps/chosen": -279.8250732421875,
      "logps/ref_chosen": -246.85342407226562,
      "logps/ref_rejected": -216.5264892578125,
      "logps/rejected": -303.6621398925781,
      "loss": 4.7333,
      "margin_dpo/margin_mean": 54.16400909423828,
      "margin_dpo/margin_std": 79.09444427490234,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -36.67499542236328,
      "KL/mean": -64.18128967285156,
      "KL/rejected_KL_mean": -91.68759155273438,
      "KL/std": 68.72068786621094,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.01261901855469,
      "fcm_dpo/q_t": 0.4370807409286499,
      "grad_norm": 48.862815856933594,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": 1.2669583559036255,
      "logits/rejected": 1.3504135608673096,
      "logps/chosen": -299.3246154785156,
      "logps/ref_chosen": -262.649658203125,
      "logps/ref_rejected": -251.24945068359375,
      "logps/rejected": -342.9370422363281,
      "loss": 4.7059,
      "margin_dpo/margin_mean": 55.012611389160156,
      "margin_dpo/margin_std": 74.97834777832031,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -33.48829650878906,
      "KL/mean": -63.14329528808594,
      "KL/rejected_KL_mean": -92.79830932617188,
      "KL/std": 68.12496185302734,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.31001281738281,
      "fcm_dpo/q_t": 0.4322381019592285,
      "grad_norm": 31.100749969482422,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": 1.2668622732162476,
      "logits/rejected": 1.2969938516616821,
      "logps/chosen": -326.8746337890625,
      "logps/ref_chosen": -293.3863220214844,
      "logps/ref_rejected": -238.86026000976562,
      "logps/rejected": -331.6585998535156,
      "loss": 4.6134,
      "margin_dpo/margin_mean": 59.31001281738281,
      "margin_dpo/margin_std": 69.085693359375,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -32.64850616455078,
      "KL/mean": -59.90916061401367,
      "KL/rejected_KL_mean": -87.16979217529297,
      "KL/std": 67.87783813476562,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.521278381347656,
      "fcm_dpo/q_t": 0.43799668550491333,
      "grad_norm": 38.644012451171875,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 1.2786717414855957,
      "logits/rejected": 1.2996244430541992,
      "logps/chosen": -328.1523132324219,
      "logps/ref_chosen": -295.5038146972656,
      "logps/ref_rejected": -278.524169921875,
      "logps/rejected": -365.6939697265625,
      "loss": 4.7484,
      "margin_dpo/margin_mean": 54.521278381347656,
      "margin_dpo/margin_std": 84.7626953125,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -35.098060607910156,
      "KL/mean": -58.832149505615234,
      "KL/rejected_KL_mean": -82.56622314453125,
      "KL/std": 68.40106964111328,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.46817398071289,
      "fcm_dpo/q_t": 0.4459683299064636,
      "grad_norm": 24.014453887939453,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 1.5275425910949707,
      "logits/rejected": 1.4591090679168701,
      "logps/chosen": -315.8534240722656,
      "logps/ref_chosen": -280.7553405761719,
      "logps/ref_rejected": -237.47286987304688,
      "logps/rejected": -320.03912353515625,
      "loss": 4.8414,
      "margin_dpo/margin_mean": 47.46817398071289,
      "margin_dpo/margin_std": 79.49259948730469,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -35.69960021972656,
      "KL/mean": -64.3497314453125,
      "KL/rejected_KL_mean": -92.9998550415039,
      "KL/std": 69.88575744628906,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.300254821777344,
      "fcm_dpo/q_t": 0.4346883296966553,
      "grad_norm": 37.476627349853516,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 1.3935264348983765,
      "logits/rejected": 1.4029306173324585,
      "logps/chosen": -334.7908935546875,
      "logps/ref_chosen": -299.09130859375,
      "logps/ref_rejected": -268.9637756347656,
      "logps/rejected": -361.9635925292969,
      "loss": 4.6741,
      "margin_dpo/margin_mean": 57.300254821777344,
      "margin_dpo/margin_std": 77.4378433227539,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -44.19821548461914,
      "KL/mean": -63.07315444946289,
      "KL/rejected_KL_mean": -81.94807434082031,
      "KL/std": 67.72361755371094,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.004721995908766985,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.749839782714844,
      "fcm_dpo/q_t": 0.4570227265357971,
      "grad_norm": 25.32196807861328,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": 1.5068625211715698,
      "logits/rejected": 1.4574016332626343,
      "logps/chosen": -320.8942565917969,
      "logps/ref_chosen": -276.696044921875,
      "logps/ref_rejected": -225.78790283203125,
      "logps/rejected": -307.7359924316406,
      "loss": 5.0376,
      "margin_dpo/margin_mean": 37.749839782714844,
      "margin_dpo/margin_std": 85.76554870605469,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -28.938915252685547,
      "KL/mean": -61.202972412109375,
      "KL/rejected_KL_mean": -93.46702575683594,
      "KL/std": 68.79322814941406,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.004644542932510376,
      "fcm_dpo/delta": -0.02067340537905693,
      "fcm_dpo/margin": 64.52811431884766,
      "fcm_dpo/q_t": 0.42761677503585815,
      "grad_norm": 33.198177337646484,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": 1.356738805770874,
      "logits/rejected": 1.4426038265228271,
      "logps/chosen": -259.52392578125,
      "logps/ref_chosen": -230.58499145507812,
      "logps/ref_rejected": -229.8074493408203,
      "logps/rejected": -323.27447509765625,
      "loss": 4.5577,
      "margin_dpo/margin_mean": 64.52811431884766,
      "margin_dpo/margin_std": 71.78933715820312,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -32.868316650390625,
      "KL/mean": -59.77250289916992,
      "KL/rejected_KL_mean": -86.67668151855469,
      "KL/std": 72.78128814697266,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.004644542932510376,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.80838394165039,
      "fcm_dpo/q_t": 0.440575510263443,
      "grad_norm": 19.141433715820312,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": 1.3066699504852295,
      "logits/rejected": 1.3442431688308716,
      "logps/chosen": -303.42431640625,
      "logps/ref_chosen": -270.5559997558594,
      "logps/ref_rejected": -266.328369140625,
      "logps/rejected": -353.00506591796875,
      "loss": 4.7967,
      "margin_dpo/margin_mean": 53.80838394165039,
      "margin_dpo/margin_std": 92.47572326660156,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -31.08609390258789,
      "KL/mean": -58.03746795654297,
      "KL/rejected_KL_mean": -84.98884582519531,
      "KL/std": 67.5262680053711,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.004644542932510376,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.90275573730469,
      "fcm_dpo/q_t": 0.43950334191322327,
      "grad_norm": 33.44983673095703,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": 1.1862235069274902,
      "logits/rejected": 1.274574875831604,
      "logps/chosen": -264.32470703125,
      "logps/ref_chosen": -233.2386016845703,
      "logps/ref_rejected": -258.154541015625,
      "logps/rejected": -343.14337158203125,
      "loss": 4.7773,
      "margin_dpo/margin_mean": 53.90275573730469,
      "margin_dpo/margin_std": 88.34896087646484,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -19.779891967773438,
      "KL/mean": -50.684783935546875,
      "KL/rejected_KL_mean": -81.58969116210938,
      "KL/std": 67.52014923095703,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.004593229852616787,
      "fcm_dpo/delta": -0.025066539645195007,
      "fcm_dpo/margin": 61.80979537963867,
      "fcm_dpo/q_t": 0.4314787685871124,
      "grad_norm": 25.525386810302734,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": 1.3255935907363892,
      "logits/rejected": 1.3555545806884766,
      "logps/chosen": -288.3260803222656,
      "logps/ref_chosen": -268.5461730957031,
      "logps/ref_rejected": -266.5316467285156,
      "logps/rejected": -348.1213073730469,
      "loss": 4.6473,
      "margin_dpo/margin_mean": 61.80979537963867,
      "margin_dpo/margin_std": 83.41767120361328,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -32.920440673828125,
      "KL/mean": -60.78913879394531,
      "KL/rejected_KL_mean": -88.65784454345703,
      "KL/std": 69.59205627441406,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.73741149902344,
      "fcm_dpo/q_t": 0.4388250410556793,
      "grad_norm": 29.151493072509766,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": 1.2071354389190674,
      "logits/rejected": 1.231052041053772,
      "logps/chosen": -327.7972412109375,
      "logps/ref_chosen": -294.8768005371094,
      "logps/ref_rejected": -281.09771728515625,
      "logps/rejected": -369.7555847167969,
      "loss": 4.7289,
      "margin_dpo/margin_mean": 55.73740768432617,
      "margin_dpo/margin_std": 78.56649780273438,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -42.50136184692383,
      "KL/mean": -65.77798461914062,
      "KL/rejected_KL_mean": -89.05459594726562,
      "KL/std": 74.32433319091797,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.553245544433594,
      "fcm_dpo/q_t": 0.4491909146308899,
      "grad_norm": 27.105268478393555,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": 1.4786839485168457,
      "logits/rejected": 1.4980887174606323,
      "logps/chosen": -354.7991027832031,
      "logps/ref_chosen": -312.2977600097656,
      "logps/ref_rejected": -304.6536560058594,
      "logps/rejected": -393.7082214355469,
      "loss": 4.9185,
      "margin_dpo/margin_mean": 46.55324935913086,
      "margin_dpo/margin_std": 90.4580307006836,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -40.596466064453125,
      "KL/mean": -60.15705108642578,
      "KL/rejected_KL_mean": -79.7176513671875,
      "KL/std": 70.71412658691406,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.121177673339844,
      "fcm_dpo/q_t": 0.4572288691997528,
      "grad_norm": 23.201614379882812,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 1.201817274093628,
      "logits/rejected": 1.265385389328003,
      "logps/chosen": -334.8006896972656,
      "logps/ref_chosen": -294.2042236328125,
      "logps/ref_rejected": -296.2102355957031,
      "logps/rejected": -375.92791748046875,
      "loss": 5.0162,
      "margin_dpo/margin_mean": 39.121177673339844,
      "margin_dpo/margin_std": 84.69398498535156,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -30.749290466308594,
      "KL/mean": -55.58964920043945,
      "KL/rejected_KL_mean": -80.43001556396484,
      "KL/std": 69.65440368652344,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.68073272705078,
      "fcm_dpo/q_t": 0.4455479085445404,
      "grad_norm": 26.325618743896484,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.3188114166259766,
      "logits/rejected": 1.328347086906433,
      "logps/chosen": -299.06195068359375,
      "logps/ref_chosen": -268.3126525878906,
      "logps/ref_rejected": -255.5543212890625,
      "logps/rejected": -335.98431396484375,
      "loss": 4.8332,
      "margin_dpo/margin_mean": 49.68073272705078,
      "margin_dpo/margin_std": 80.59165954589844,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -31.572696685791016,
      "KL/mean": -62.80278396606445,
      "KL/rejected_KL_mean": -94.0328598022461,
      "KL/std": 70.67665100097656,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.46017837524414,
      "fcm_dpo/q_t": 0.43160247802734375,
      "grad_norm": 32.71507263183594,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": 1.3224992752075195,
      "logits/rejected": 1.3591933250427246,
      "logps/chosen": -295.3231201171875,
      "logps/ref_chosen": -263.75042724609375,
      "logps/ref_rejected": -234.5144500732422,
      "logps/rejected": -328.54730224609375,
      "loss": 4.6488,
      "margin_dpo/margin_mean": 62.46017837524414,
      "margin_dpo/margin_std": 86.32286834716797,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -30.18381690979004,
      "KL/mean": -53.08464050292969,
      "KL/rejected_KL_mean": -75.98546600341797,
      "KL/std": 67.27308654785156,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.0045523326843976974,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.8016471862793,
      "fcm_dpo/q_t": 0.4495130181312561,
      "grad_norm": 27.931896209716797,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": 1.3063247203826904,
      "logits/rejected": 1.278424620628357,
      "logps/chosen": -317.0002746582031,
      "logps/ref_chosen": -286.81646728515625,
      "logps/ref_rejected": -260.4169616699219,
      "logps/rejected": -336.40240478515625,
      "loss": 4.8829,
      "margin_dpo/margin_mean": 45.80165100097656,
      "margin_dpo/margin_std": 76.70598602294922,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -27.739145278930664,
      "KL/mean": -54.98110580444336,
      "KL/rejected_KL_mean": -82.22306823730469,
      "KL/std": 70.27132415771484,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.004527419339865446,
      "fcm_dpo/delta": -0.01837669312953949,
      "fcm_dpo/margin": 54.483917236328125,
      "fcm_dpo/q_t": 0.44068166613578796,
      "grad_norm": 23.294628143310547,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": 1.2238651514053345,
      "logits/rejected": 1.223275899887085,
      "logps/chosen": -321.050048828125,
      "logps/ref_chosen": -293.3109130859375,
      "logps/ref_rejected": -264.1449279785156,
      "logps/rejected": -346.36798095703125,
      "loss": 4.7891,
      "margin_dpo/margin_mean": 54.48392105102539,
      "margin_dpo/margin_std": 87.3408203125,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -42.71800994873047,
      "KL/mean": -67.65845489501953,
      "KL/rejected_KL_mean": -92.5988998413086,
      "KL/std": 68.51226043701172,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.00445386441424489,
      "fcm_dpo/delta": -0.011958310380578041,
      "fcm_dpo/margin": 49.88089370727539,
      "fcm_dpo/q_t": 0.44641828536987305,
      "grad_norm": 33.570716857910156,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 0.9770832061767578,
      "logits/rejected": 1.0526163578033447,
      "logps/chosen": -312.4743347167969,
      "logps/ref_chosen": -269.75634765625,
      "logps/ref_rejected": -241.08438110351562,
      "logps/rejected": -333.68328857421875,
      "loss": 4.8715,
      "margin_dpo/margin_mean": 49.88089370727539,
      "margin_dpo/margin_std": 88.0946044921875,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -33.363128662109375,
      "KL/mean": -56.44091796875,
      "KL/rejected_KL_mean": -79.51870727539062,
      "KL/std": 68.99575805664062,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.155582427978516,
      "fcm_dpo/q_t": 0.4506586790084839,
      "grad_norm": 22.8138427734375,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": 1.1148439645767212,
      "logits/rejected": 1.1456542015075684,
      "logps/chosen": -270.73492431640625,
      "logps/ref_chosen": -237.37179565429688,
      "logps/ref_rejected": -253.5463409423828,
      "logps/rejected": -333.0650329589844,
      "loss": 4.8931,
      "margin_dpo/margin_mean": 46.155582427978516,
      "margin_dpo/margin_std": 78.75967407226562,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -37.77162170410156,
      "KL/mean": -64.12367248535156,
      "KL/rejected_KL_mean": -90.4757308959961,
      "KL/std": 68.10700225830078,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.70410919189453,
      "fcm_dpo/q_t": 0.44366344809532166,
      "grad_norm": 33.96640396118164,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": 1.0519282817840576,
      "logits/rejected": 1.1010932922363281,
      "logps/chosen": -299.07928466796875,
      "logps/ref_chosen": -261.3076477050781,
      "logps/ref_rejected": -252.3170166015625,
      "logps/rejected": -342.7927551269531,
      "loss": 4.8105,
      "margin_dpo/margin_mean": 52.70410919189453,
      "margin_dpo/margin_std": 85.6024398803711,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -19.948434829711914,
      "KL/mean": -50.44887924194336,
      "KL/rejected_KL_mean": -80.94933319091797,
      "KL/std": 69.6468505859375,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.000892639160156,
      "fcm_dpo/q_t": 0.43475598096847534,
      "grad_norm": 23.980453491210938,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": 1.4357049465179443,
      "logits/rejected": 1.4255125522613525,
      "logps/chosen": -288.7201843261719,
      "logps/ref_chosen": -268.7717590332031,
      "logps/ref_rejected": -232.3352813720703,
      "logps/rejected": -313.28460693359375,
      "loss": 4.6695,
      "margin_dpo/margin_mean": 61.000892639160156,
      "margin_dpo/margin_std": 82.6962661743164,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -27.63826560974121,
      "KL/mean": -59.7108154296875,
      "KL/rejected_KL_mean": -91.78336334228516,
      "KL/std": 72.22247314453125,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.1451187133789,
      "fcm_dpo/q_t": 0.4313385784626007,
      "grad_norm": 22.01619529724121,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": 1.2977173328399658,
      "logits/rejected": 1.2851532697677612,
      "logps/chosen": -321.2514343261719,
      "logps/ref_chosen": -293.6131286621094,
      "logps/ref_rejected": -281.87738037109375,
      "logps/rejected": -373.66070556640625,
      "loss": 4.6373,
      "margin_dpo/margin_mean": 64.14511108398438,
      "margin_dpo/margin_std": 86.94947814941406,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -33.97660827636719,
      "KL/mean": -59.96274185180664,
      "KL/rejected_KL_mean": -85.94888305664062,
      "KL/std": 71.3515853881836,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.97226333618164,
      "fcm_dpo/q_t": 0.4444987177848816,
      "grad_norm": 22.005287170410156,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 0.90833979845047,
      "logits/rejected": 1.0210092067718506,
      "logps/chosen": -285.7336730957031,
      "logps/ref_chosen": -251.75704956054688,
      "logps/ref_rejected": -255.76441955566406,
      "logps/rejected": -341.7132873535156,
      "loss": 4.8212,
      "margin_dpo/margin_mean": 51.972259521484375,
      "margin_dpo/margin_std": 84.49942779541016,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -49.83913040161133,
      "KL/mean": -70.80889892578125,
      "KL/rejected_KL_mean": -91.77867126464844,
      "KL/std": 74.88032531738281,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.93954849243164,
      "fcm_dpo/q_t": 0.45526060461997986,
      "grad_norm": 34.56527328491211,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 1.377077579498291,
      "logits/rejected": 1.4851458072662354,
      "logps/chosen": -356.4867858886719,
      "logps/ref_chosen": -306.64764404296875,
      "logps/ref_rejected": -301.4248352050781,
      "logps/rejected": -393.2035217285156,
      "loss": 4.9988,
      "margin_dpo/margin_mean": 41.93954849243164,
      "margin_dpo/margin_std": 89.04151916503906,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -31.35328483581543,
      "KL/mean": -58.25117874145508,
      "KL/rejected_KL_mean": -85.1490707397461,
      "KL/std": 70.98402404785156,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.79579544067383,
      "fcm_dpo/q_t": 0.4423893392086029,
      "grad_norm": 23.44870376586914,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": 1.6065051555633545,
      "logits/rejected": 1.678582787513733,
      "logps/chosen": -294.0221862792969,
      "logps/ref_chosen": -262.6689147949219,
      "logps/ref_rejected": -260.8409118652344,
      "logps/rejected": -345.9899597167969,
      "loss": 4.773,
      "margin_dpo/margin_mean": 53.79579544067383,
      "margin_dpo/margin_std": 78.38786315917969,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -32.80494689941406,
      "KL/mean": -56.304264068603516,
      "KL/rejected_KL_mean": -79.80357360839844,
      "KL/std": 70.86469268798828,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.004443186800926924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.99864959716797,
      "fcm_dpo/q_t": 0.4497010409832001,
      "grad_norm": 24.397628784179688,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": 1.4280129671096802,
      "logits/rejected": 1.4254993200302124,
      "logps/chosen": -332.430908203125,
      "logps/ref_chosen": -299.6259765625,
      "logps/ref_rejected": -261.2701416015625,
      "logps/rejected": -341.0736999511719,
      "loss": 4.8992,
      "margin_dpo/margin_mean": 46.99864959716797,
      "margin_dpo/margin_std": 84.3766860961914,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -33.321861267089844,
      "KL/mean": -61.93470001220703,
      "KL/rejected_KL_mean": -90.54754638671875,
      "KL/std": 72.63216400146484,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.004427001811563969,
      "fcm_dpo/delta": -0.0183470007032156,
      "fcm_dpo/margin": 57.225677490234375,
      "fcm_dpo/q_t": 0.4389106035232544,
      "grad_norm": 27.705413818359375,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": 1.3028662204742432,
      "logits/rejected": 1.3481539487838745,
      "logps/chosen": -324.2723083496094,
      "logps/ref_chosen": -290.950439453125,
      "logps/ref_rejected": -255.15374755859375,
      "logps/rejected": -345.7012939453125,
      "loss": 4.7371,
      "margin_dpo/margin_mean": 57.225677490234375,
      "margin_dpo/margin_std": 81.73297882080078,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -27.60763168334961,
      "KL/mean": -50.679141998291016,
      "KL/rejected_KL_mean": -73.75064086914062,
      "KL/std": 67.98295593261719,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.14301300048828,
      "fcm_dpo/q_t": 0.45106154680252075,
      "grad_norm": 27.409439086914062,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": 1.2601933479309082,
      "logits/rejected": 1.3370109796524048,
      "logps/chosen": -300.8735046386719,
      "logps/ref_chosen": -273.2658996582031,
      "logps/ref_rejected": -263.9664306640625,
      "logps/rejected": -337.717041015625,
      "loss": 4.8869,
      "margin_dpo/margin_mean": 46.14301300048828,
      "margin_dpo/margin_std": 72.05493927001953,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -43.75952911376953,
      "KL/mean": -69.98768615722656,
      "KL/rejected_KL_mean": -96.21585845947266,
      "KL/std": 71.84439086914062,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.456329345703125,
      "fcm_dpo/q_t": 0.44457322359085083,
      "grad_norm": 37.56001281738281,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": 1.3683903217315674,
      "logits/rejected": 1.4401674270629883,
      "logps/chosen": -306.44171142578125,
      "logps/ref_chosen": -262.6821594238281,
      "logps/ref_rejected": -226.46859741210938,
      "logps/rejected": -322.6844482421875,
      "loss": 4.8203,
      "margin_dpo/margin_mean": 52.456329345703125,
      "margin_dpo/margin_std": 80.17398834228516,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -39.18180847167969,
      "KL/mean": -64.04957580566406,
      "KL/rejected_KL_mean": -88.91735076904297,
      "KL/std": 70.98311614990234,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.735538482666016,
      "fcm_dpo/q_t": 0.44718554615974426,
      "grad_norm": 25.8310489654541,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": 1.1613354682922363,
      "logits/rejected": 1.1741466522216797,
      "logps/chosen": -303.00640869140625,
      "logps/ref_chosen": -263.8246154785156,
      "logps/ref_rejected": -237.9121551513672,
      "logps/rejected": -326.8294982910156,
      "loss": 4.8467,
      "margin_dpo/margin_mean": 49.735538482666016,
      "margin_dpo/margin_std": 79.4467544555664,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -30.30510902404785,
      "KL/mean": -51.688114166259766,
      "KL/rejected_KL_mean": -73.07112121582031,
      "KL/std": 68.89720916748047,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.766021728515625,
      "fcm_dpo/q_t": 0.4547736942768097,
      "grad_norm": 25.33574867248535,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 1.2564035654067993,
      "logits/rejected": 1.2988770008087158,
      "logps/chosen": -308.4584655761719,
      "logps/ref_chosen": -278.1533203125,
      "logps/ref_rejected": -263.39691162109375,
      "logps/rejected": -336.4680480957031,
      "loss": 4.9503,
      "margin_dpo/margin_mean": 42.766021728515625,
      "margin_dpo/margin_std": 75.73143005371094,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -36.61722183227539,
      "KL/mean": -66.23300170898438,
      "KL/rejected_KL_mean": -95.84880065917969,
      "KL/std": 75.34637451171875,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.23157501220703,
      "fcm_dpo/q_t": 0.4376799762248993,
      "grad_norm": 18.26201629638672,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": 1.4236845970153809,
      "logits/rejected": 1.4627549648284912,
      "logps/chosen": -355.0722961425781,
      "logps/ref_chosen": -318.455078125,
      "logps/ref_rejected": -288.5691833496094,
      "logps/rejected": -384.41796875,
      "loss": 4.7202,
      "margin_dpo/margin_mean": 59.23157501220703,
      "margin_dpo/margin_std": 84.13404083251953,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -40.296695709228516,
      "KL/mean": -66.1799087524414,
      "KL/rejected_KL_mean": -92.06311798095703,
      "KL/std": 74.0902328491211,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.76642608642578,
      "fcm_dpo/q_t": 0.445438027381897,
      "grad_norm": 24.458274841308594,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 1.3042863607406616,
      "logits/rejected": 1.3058620691299438,
      "logps/chosen": -324.48590087890625,
      "logps/ref_chosen": -284.1891784667969,
      "logps/ref_rejected": -247.94589233398438,
      "logps/rejected": -340.00897216796875,
      "loss": 4.8481,
      "margin_dpo/margin_mean": 51.76641845703125,
      "margin_dpo/margin_std": 87.52262115478516,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -38.66436767578125,
      "KL/mean": -56.57937240600586,
      "KL/rejected_KL_mean": -74.494384765625,
      "KL/std": 71.15831756591797,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.004378447309136391,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 35.830020904541016,
      "fcm_dpo/q_t": 0.4624999463558197,
      "grad_norm": 44.567405700683594,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 1.4678698778152466,
      "logits/rejected": 1.4515212774276733,
      "logps/chosen": -341.97247314453125,
      "logps/ref_chosen": -303.30810546875,
      "logps/ref_rejected": -260.1866455078125,
      "logps/rejected": -334.6810607910156,
      "loss": 5.0949,
      "margin_dpo/margin_mean": 35.830020904541016,
      "margin_dpo/margin_std": 86.69757080078125,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -33.34721755981445,
      "KL/mean": -62.537010192871094,
      "KL/rejected_KL_mean": -91.72679901123047,
      "KL/std": 72.46097564697266,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.0043613361194729805,
      "fcm_dpo/delta": -0.009808380156755447,
      "fcm_dpo/margin": 58.37958526611328,
      "fcm_dpo/q_t": 0.43857845664024353,
      "grad_norm": 23.899044036865234,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": 1.2709671258926392,
      "logits/rejected": 1.4490203857421875,
      "logps/chosen": -280.1543273925781,
      "logps/ref_chosen": -246.80709838867188,
      "logps/ref_rejected": -253.13223266601562,
      "logps/rejected": -344.8590393066406,
      "loss": 4.7304,
      "margin_dpo/margin_mean": 58.37958526611328,
      "margin_dpo/margin_std": 81.43959045410156,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -37.511138916015625,
      "KL/mean": -67.8247299194336,
      "KL/rejected_KL_mean": -98.13832092285156,
      "KL/std": 72.19886779785156,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.6271858215332,
      "fcm_dpo/q_t": 0.43649154901504517,
      "grad_norm": 25.1179256439209,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": 1.389224886894226,
      "logits/rejected": 1.475706696510315,
      "logps/chosen": -299.11260986328125,
      "logps/ref_chosen": -261.6014709472656,
      "logps/ref_rejected": -262.74761962890625,
      "logps/rejected": -360.8859558105469,
      "loss": 4.695,
      "margin_dpo/margin_mean": 60.6271858215332,
      "margin_dpo/margin_std": 82.14624786376953,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -37.84943389892578,
      "KL/mean": -55.65714645385742,
      "KL/rejected_KL_mean": -73.46484375,
      "KL/std": 71.87853240966797,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 35.61540985107422,
      "fcm_dpo/q_t": 0.4629093408584595,
      "grad_norm": 30.090452194213867,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": 1.2626596689224243,
      "logits/rejected": 1.3778496980667114,
      "logps/chosen": -289.9978942871094,
      "logps/ref_chosen": -252.14840698242188,
      "logps/ref_rejected": -244.1017608642578,
      "logps/rejected": -317.56658935546875,
      "loss": 5.1062,
      "margin_dpo/margin_mean": 35.61540985107422,
      "margin_dpo/margin_std": 88.87792205810547,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -38.775779724121094,
      "KL/mean": -68.83124542236328,
      "KL/rejected_KL_mean": -98.8866958618164,
      "KL/std": 73.80791473388672,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.11092758178711,
      "fcm_dpo/q_t": 0.43698111176490784,
      "grad_norm": 24.467140197753906,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": 1.341346025466919,
      "logits/rejected": 1.4527244567871094,
      "logps/chosen": -324.93829345703125,
      "logps/ref_chosen": -286.16253662109375,
      "logps/ref_rejected": -272.3275146484375,
      "logps/rejected": -371.2142028808594,
      "loss": 4.7129,
      "margin_dpo/margin_mean": 60.110931396484375,
      "margin_dpo/margin_std": 84.8955307006836,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -31.9842529296875,
      "KL/mean": -55.59268569946289,
      "KL/rejected_KL_mean": -79.20112609863281,
      "KL/std": 69.07119750976562,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.21686553955078,
      "fcm_dpo/q_t": 0.4502883851528168,
      "grad_norm": 32.05209732055664,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": 1.3016653060913086,
      "logits/rejected": 1.3575738668441772,
      "logps/chosen": -230.1370391845703,
      "logps/ref_chosen": -198.1527862548828,
      "logps/ref_rejected": -209.80569458007812,
      "logps/rejected": -289.0068359375,
      "loss": 4.8829,
      "margin_dpo/margin_mean": 47.21686935424805,
      "margin_dpo/margin_std": 76.44894409179688,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -42.55330276489258,
      "KL/mean": -68.47598266601562,
      "KL/rejected_KL_mean": -94.398681640625,
      "KL/std": 70.4147720336914,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.84536361694336,
      "fcm_dpo/q_t": 0.44600585103034973,
      "grad_norm": 41.3632698059082,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": 1.4684243202209473,
      "logits/rejected": 1.54707670211792,
      "logps/chosen": -299.06048583984375,
      "logps/ref_chosen": -256.50714111328125,
      "logps/ref_rejected": -237.40769958496094,
      "logps/rejected": -331.8063659667969,
      "loss": 4.8439,
      "margin_dpo/margin_mean": 51.84536361694336,
      "margin_dpo/margin_std": 85.70298767089844,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -30.892742156982422,
      "KL/mean": -60.45337677001953,
      "KL/rejected_KL_mean": -90.01399993896484,
      "KL/std": 72.68421936035156,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.121253967285156,
      "fcm_dpo/q_t": 0.43828946352005005,
      "grad_norm": 19.033916473388672,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": 1.4608856439590454,
      "logits/rejected": 1.4099931716918945,
      "logps/chosen": -323.3399658203125,
      "logps/ref_chosen": -292.447265625,
      "logps/ref_rejected": -255.47872924804688,
      "logps/rejected": -345.49273681640625,
      "loss": 4.7454,
      "margin_dpo/margin_mean": 59.121253967285156,
      "margin_dpo/margin_std": 91.13362121582031,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -32.15397262573242,
      "KL/mean": -60.36504364013672,
      "KL/rejected_KL_mean": -88.57611846923828,
      "KL/std": 73.54418182373047,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.422149658203125,
      "fcm_dpo/q_t": 0.44069287180900574,
      "grad_norm": 23.765024185180664,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": 1.373698115348816,
      "logits/rejected": 1.3338919878005981,
      "logps/chosen": -325.9139404296875,
      "logps/ref_chosen": -293.7599792480469,
      "logps/ref_rejected": -259.4142761230469,
      "logps/rejected": -347.9903564453125,
      "loss": 4.7555,
      "margin_dpo/margin_mean": 56.422149658203125,
      "margin_dpo/margin_std": 80.6426010131836,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -42.02604293823242,
      "KL/mean": -66.42587280273438,
      "KL/rejected_KL_mean": -90.82569885253906,
      "KL/std": 72.9551773071289,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.799659729003906,
      "fcm_dpo/q_t": 0.4486715793609619,
      "grad_norm": 21.28990364074707,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 1.3276677131652832,
      "logits/rejected": 1.2721139192581177,
      "logps/chosen": -342.9397888183594,
      "logps/ref_chosen": -300.91375732421875,
      "logps/ref_rejected": -256.4080810546875,
      "logps/rejected": -347.2337646484375,
      "loss": 4.8607,
      "margin_dpo/margin_mean": 48.799659729003906,
      "margin_dpo/margin_std": 77.00198364257812,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -29.904752731323242,
      "KL/mean": -62.25552749633789,
      "KL/rejected_KL_mean": -94.60631561279297,
      "KL/std": 74.78999328613281,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.7015609741211,
      "fcm_dpo/q_t": 0.43246689438819885,
      "grad_norm": 24.19639015197754,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 1.0922892093658447,
      "logits/rejected": 1.179084062576294,
      "logps/chosen": -286.465087890625,
      "logps/ref_chosen": -256.5603332519531,
      "logps/ref_rejected": -271.043701171875,
      "logps/rejected": -365.6499938964844,
      "loss": 4.6515,
      "margin_dpo/margin_mean": 64.7015609741211,
      "margin_dpo/margin_std": 89.6834487915039,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -35.81988525390625,
      "KL/mean": -63.307823181152344,
      "KL/rejected_KL_mean": -90.79576110839844,
      "KL/std": 69.28202819824219,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.975860595703125,
      "fcm_dpo/q_t": 0.44236207008361816,
      "grad_norm": 67.99634552001953,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": 1.4027702808380127,
      "logits/rejected": 1.3210787773132324,
      "logps/chosen": -330.5203552246094,
      "logps/ref_chosen": -294.700439453125,
      "logps/ref_rejected": -239.67922973632812,
      "logps/rejected": -330.4750061035156,
      "loss": 4.7878,
      "margin_dpo/margin_mean": 54.975860595703125,
      "margin_dpo/margin_std": 84.46615600585938,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -35.68743133544922,
      "KL/mean": -68.07635498046875,
      "KL/rejected_KL_mean": -100.46527099609375,
      "KL/std": 76.26604461669922,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.77784729003906,
      "fcm_dpo/q_t": 0.43242761492729187,
      "grad_norm": 26.038848876953125,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": 1.3812525272369385,
      "logits/rejected": 1.3976426124572754,
      "logps/chosen": -329.36328125,
      "logps/ref_chosen": -293.67584228515625,
      "logps/ref_rejected": -260.85675048828125,
      "logps/rejected": -361.322021484375,
      "loss": 4.6489,
      "margin_dpo/margin_mean": 64.77784729003906,
      "margin_dpo/margin_std": 87.9860610961914,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -54.79117965698242,
      "KL/mean": -78.58402252197266,
      "KL/rejected_KL_mean": -102.37687683105469,
      "KL/std": 73.21632385253906,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.585689544677734,
      "fcm_dpo/q_t": 0.4503006935119629,
      "grad_norm": 21.15276527404785,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": 1.2017329931259155,
      "logits/rejected": 1.2650114297866821,
      "logps/chosen": -333.6865539550781,
      "logps/ref_chosen": -278.8953857421875,
      "logps/ref_rejected": -283.8898620605469,
      "logps/rejected": -386.2667236328125,
      "loss": 4.9102,
      "margin_dpo/margin_mean": 47.585689544677734,
      "margin_dpo/margin_std": 85.33905792236328,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -35.622684478759766,
      "KL/mean": -65.97052764892578,
      "KL/rejected_KL_mean": -96.318359375,
      "KL/std": 74.68295288085938,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.695674896240234,
      "fcm_dpo/q_t": 0.4365980625152588,
      "grad_norm": 33.8511962890625,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": 1.2009162902832031,
      "logits/rejected": 1.2422904968261719,
      "logps/chosen": -301.2666015625,
      "logps/ref_chosen": -265.6439208984375,
      "logps/ref_rejected": -255.40138244628906,
      "logps/rejected": -351.7197265625,
      "loss": 4.6981,
      "margin_dpo/margin_mean": 60.695674896240234,
      "margin_dpo/margin_std": 85.3858871459961,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -43.39641189575195,
      "KL/mean": -73.5914306640625,
      "KL/rejected_KL_mean": -103.78644561767578,
      "KL/std": 74.7223129272461,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.39002227783203,
      "fcm_dpo/q_t": 0.43662434816360474,
      "grad_norm": 26.632598876953125,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": 1.461929202079773,
      "logits/rejected": 1.4886159896850586,
      "logps/chosen": -323.62091064453125,
      "logps/ref_chosen": -280.2244873046875,
      "logps/ref_rejected": -248.72366333007812,
      "logps/rejected": -352.5101013183594,
      "loss": 4.7052,
      "margin_dpo/margin_mean": 60.39002227783203,
      "margin_dpo/margin_std": 84.12542724609375,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -40.50281524658203,
      "KL/mean": -70.92607116699219,
      "KL/rejected_KL_mean": -101.34932708740234,
      "KL/std": 73.6207046508789,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.004344225395470858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.84651184082031,
      "fcm_dpo/q_t": 0.43632376194000244,
      "grad_norm": 29.973752975463867,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": 1.1465647220611572,
      "logits/rejected": 1.1345621347427368,
      "logps/chosen": -333.4394836425781,
      "logps/ref_chosen": -292.9366760253906,
      "logps/ref_rejected": -261.88763427734375,
      "logps/rejected": -363.2369689941406,
      "loss": 4.6873,
      "margin_dpo/margin_mean": 60.84651184082031,
      "margin_dpo/margin_std": 81.92500305175781,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -34.453250885009766,
      "KL/mean": -69.05155181884766,
      "KL/rejected_KL_mean": -103.64984130859375,
      "KL/std": 76.39900207519531,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.004306429531425238,
      "fcm_dpo/delta": -0.017522968351840973,
      "fcm_dpo/margin": 69.19657897949219,
      "fcm_dpo/q_t": 0.4280383586883545,
      "grad_norm": 22.623035430908203,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": 1.2763627767562866,
      "logits/rejected": 1.3714910745620728,
      "logps/chosen": -319.3590087890625,
      "logps/ref_chosen": -284.90576171875,
      "logps/ref_rejected": -256.59197998046875,
      "logps/rejected": -360.2417907714844,
      "loss": 4.5804,
      "margin_dpo/margin_mean": 69.19657897949219,
      "margin_dpo/margin_std": 83.70861053466797,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -42.80347442626953,
      "KL/mean": -67.69097137451172,
      "KL/rejected_KL_mean": -92.57846069335938,
      "KL/std": 72.96141052246094,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.004283751826733351,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.775001525878906,
      "fcm_dpo/q_t": 0.44857364892959595,
      "grad_norm": 33.28280258178711,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": 1.323946475982666,
      "logits/rejected": 1.3370944261550903,
      "logps/chosen": -318.7193908691406,
      "logps/ref_chosen": -275.91595458984375,
      "logps/ref_rejected": -264.79736328125,
      "logps/rejected": -357.3758239746094,
      "loss": 4.8876,
      "margin_dpo/margin_mean": 49.77499771118164,
      "margin_dpo/margin_std": 87.79545593261719,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -40.7685546875,
      "KL/mean": -71.69973754882812,
      "KL/rejected_KL_mean": -102.63092803955078,
      "KL/std": 75.4233169555664,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.004283751826733351,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.86238098144531,
      "fcm_dpo/q_t": 0.4365013837814331,
      "grad_norm": 17.93372917175293,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 1.2045530080795288,
      "logits/rejected": 1.2724043130874634,
      "logps/chosen": -285.7105712890625,
      "logps/ref_chosen": -244.9420166015625,
      "logps/ref_rejected": -235.61978149414062,
      "logps/rejected": -338.2507019042969,
      "loss": 4.7171,
      "margin_dpo/margin_mean": 61.86238098144531,
      "margin_dpo/margin_std": 92.94063568115234,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -45.643157958984375,
      "KL/mean": -72.06182861328125,
      "KL/rejected_KL_mean": -98.48049926757812,
      "KL/std": 74.05956268310547,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.004283751826733351,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.83734130859375,
      "fcm_dpo/q_t": 0.44513291120529175,
      "grad_norm": 28.41175079345703,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 1.2597984075546265,
      "logits/rejected": 1.259942650794983,
      "logps/chosen": -334.9930114746094,
      "logps/ref_chosen": -289.3498229980469,
      "logps/ref_rejected": -243.70205688476562,
      "logps/rejected": -342.18255615234375,
      "loss": 4.8258,
      "margin_dpo/margin_mean": 52.83734130859375,
      "margin_dpo/margin_std": 83.4427261352539,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -42.08303451538086,
      "KL/mean": -76.49684143066406,
      "KL/rejected_KL_mean": -110.91064453125,
      "KL/std": 76.95083618164062,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.00425085611641407,
      "fcm_dpo/delta": -0.012864350341260433,
      "fcm_dpo/margin": 68.82762145996094,
      "fcm_dpo/q_t": 0.4293413758277893,
      "grad_norm": 23.20381736755371,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": 1.5388174057006836,
      "logits/rejected": 1.5402798652648926,
      "logps/chosen": -363.7365417480469,
      "logps/ref_chosen": -321.65350341796875,
      "logps/ref_rejected": -282.73773193359375,
      "logps/rejected": -393.6483459472656,
      "loss": 4.602,
      "margin_dpo/margin_mean": 68.8276138305664,
      "margin_dpo/margin_std": 85.5063705444336,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -42.6411247253418,
      "KL/mean": -75.324951171875,
      "KL/rejected_KL_mean": -108.00878143310547,
      "KL/std": 78.76000213623047,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.004202315583825111,
      "fcm_dpo/delta": -0.012725009582936764,
      "fcm_dpo/margin": 65.36766052246094,
      "fcm_dpo/q_t": 0.4337764084339142,
      "grad_norm": 28.29071044921875,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": 1.3283770084381104,
      "logits/rejected": 1.303499698638916,
      "logps/chosen": -352.3916015625,
      "logps/ref_chosen": -309.75048828125,
      "logps/ref_rejected": -267.41925048828125,
      "logps/rejected": -375.4280700683594,
      "loss": 4.669,
      "margin_dpo/margin_mean": 65.36766052246094,
      "margin_dpo/margin_std": 89.57645416259766,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -49.0091667175293,
      "KL/mean": -70.31108093261719,
      "KL/rejected_KL_mean": -91.61299896240234,
      "KL/std": 76.17677307128906,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.60383605957031,
      "fcm_dpo/q_t": 0.4570022225379944,
      "grad_norm": 22.250080108642578,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 1.4579730033874512,
      "logits/rejected": 1.4387532472610474,
      "logps/chosen": -360.00323486328125,
      "logps/ref_chosen": -310.9941101074219,
      "logps/ref_rejected": -279.94915771484375,
      "logps/rejected": -371.5621337890625,
      "loss": 5.0084,
      "margin_dpo/margin_mean": 42.60383605957031,
      "margin_dpo/margin_std": 88.8580093383789,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -47.24053955078125,
      "KL/mean": -74.8983154296875,
      "KL/rejected_KL_mean": -102.55609130859375,
      "KL/std": 73.35511016845703,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.31555938720703,
      "fcm_dpo/q_t": 0.44412699341773987,
      "grad_norm": 26.758668899536133,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": 1.380109190940857,
      "logits/rejected": 1.2962323427200317,
      "logps/chosen": -356.4239807128906,
      "logps/ref_chosen": -309.18341064453125,
      "logps/ref_rejected": -256.9963073730469,
      "logps/rejected": -359.5523681640625,
      "loss": 4.8137,
      "margin_dpo/margin_mean": 55.31555938720703,
      "margin_dpo/margin_std": 88.64044952392578,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -52.16647720336914,
      "KL/mean": -78.1490478515625,
      "KL/rejected_KL_mean": -104.13160705566406,
      "KL/std": 76.1008529663086,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.965126037597656,
      "fcm_dpo/q_t": 0.4475591480731964,
      "grad_norm": 25.963022232055664,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": 1.4466854333877563,
      "logits/rejected": 1.5242968797683716,
      "logps/chosen": -334.441650390625,
      "logps/ref_chosen": -282.275146484375,
      "logps/ref_rejected": -255.17941284179688,
      "logps/rejected": -359.3110046386719,
      "loss": 4.8738,
      "margin_dpo/margin_mean": 51.965126037597656,
      "margin_dpo/margin_std": 88.66751861572266,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -47.04339599609375,
      "KL/mean": -71.66465759277344,
      "KL/rejected_KL_mean": -96.2859115600586,
      "KL/std": 71.92180633544922,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.242523193359375,
      "fcm_dpo/q_t": 0.4502522349357605,
      "grad_norm": 19.284128189086914,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": 1.2401548624038696,
      "logits/rejected": 1.2654993534088135,
      "logps/chosen": -339.92059326171875,
      "logps/ref_chosen": -292.8772277832031,
      "logps/ref_rejected": -266.78509521484375,
      "logps/rejected": -363.0710144042969,
      "loss": 4.892,
      "margin_dpo/margin_mean": 49.242523193359375,
      "margin_dpo/margin_std": 85.78243255615234,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -47.17013931274414,
      "KL/mean": -71.80248260498047,
      "KL/rejected_KL_mean": -96.43482208251953,
      "KL/std": 75.65074920654297,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.264678955078125,
      "fcm_dpo/q_t": 0.4502137303352356,
      "grad_norm": 27.884639739990234,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": 1.2664755582809448,
      "logits/rejected": 1.3311673402786255,
      "logps/chosen": -320.0638427734375,
      "logps/ref_chosen": -272.8937072753906,
      "logps/ref_rejected": -293.058349609375,
      "logps/rejected": -389.4931640625,
      "loss": 4.8972,
      "margin_dpo/margin_mean": 49.264678955078125,
      "margin_dpo/margin_std": 85.5869140625,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -49.09370422363281,
      "KL/mean": -74.88701629638672,
      "KL/rejected_KL_mean": -100.68031311035156,
      "KL/std": 72.75779724121094,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.58661651611328,
      "fcm_dpo/q_t": 0.44789084792137146,
      "grad_norm": 26.835983276367188,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": 1.4252798557281494,
      "logits/rejected": 1.435435175895691,
      "logps/chosen": -308.23529052734375,
      "logps/ref_chosen": -259.1416015625,
      "logps/ref_rejected": -229.64996337890625,
      "logps/rejected": -330.33026123046875,
      "loss": 4.8869,
      "margin_dpo/margin_mean": 51.586612701416016,
      "margin_dpo/margin_std": 94.26516723632812,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -55.7342643737793,
      "KL/mean": -81.11094665527344,
      "KL/rejected_KL_mean": -106.48762512207031,
      "KL/std": 77.06127166748047,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.753353118896484,
      "fcm_dpo/q_t": 0.44902563095092773,
      "grad_norm": 44.50007247924805,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 1.2614928483963013,
      "logits/rejected": 1.1863659620285034,
      "logps/chosen": -352.07220458984375,
      "logps/ref_chosen": -296.33795166015625,
      "logps/ref_rejected": -283.7359924316406,
      "logps/rejected": -390.2236022949219,
      "loss": 4.8825,
      "margin_dpo/margin_mean": 50.75334930419922,
      "margin_dpo/margin_std": 87.9572982788086,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -51.394065856933594,
      "KL/mean": -81.62066650390625,
      "KL/rejected_KL_mean": -111.84725952148438,
      "KL/std": 75.08112335205078,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.004196947906166315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.45319747924805,
      "fcm_dpo/q_t": 0.43877285718917847,
      "grad_norm": 28.91515350341797,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 1.1803022623062134,
      "logits/rejected": 1.2145636081695557,
      "logps/chosen": -300.36724853515625,
      "logps/ref_chosen": -248.97320556640625,
      "logps/ref_rejected": -259.5140075683594,
      "logps/rejected": -371.36126708984375,
      "loss": 4.717,
      "margin_dpo/margin_mean": 60.45319747924805,
      "margin_dpo/margin_std": 83.74251556396484,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -38.31960678100586,
      "KL/mean": -75.00596618652344,
      "KL/rejected_KL_mean": -111.69232177734375,
      "KL/std": 75.5895004272461,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.00417304877191782,
      "fcm_dpo/delta": -0.011441088281571865,
      "fcm_dpo/margin": 73.37271118164062,
      "fcm_dpo/q_t": 0.4261191487312317,
      "grad_norm": 28.688562393188477,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": 1.339105486869812,
      "logits/rejected": 1.5015157461166382,
      "logps/chosen": -318.99188232421875,
      "logps/ref_chosen": -280.67230224609375,
      "logps/ref_rejected": -272.4359130859375,
      "logps/rejected": -384.1282043457031,
      "loss": 4.5563,
      "margin_dpo/margin_mean": 73.37271118164062,
      "margin_dpo/margin_std": 86.8028793334961,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -47.911827087402344,
      "KL/mean": -74.36686706542969,
      "KL/rejected_KL_mean": -100.8219223022461,
      "KL/std": 73.97142791748047,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.0041587091982364655,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.91010284423828,
      "fcm_dpo/q_t": 0.4468986392021179,
      "grad_norm": 35.59858703613281,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": 0.7876460552215576,
      "logits/rejected": 0.8263806104660034,
      "logps/chosen": -315.5702819824219,
      "logps/ref_chosen": -267.658447265625,
      "logps/ref_rejected": -262.9810791015625,
      "logps/rejected": -363.802978515625,
      "loss": 4.8493,
      "margin_dpo/margin_mean": 52.91009521484375,
      "margin_dpo/margin_std": 85.84049987792969,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -53.088321685791016,
      "KL/mean": -72.4102783203125,
      "KL/rejected_KL_mean": -91.73225402832031,
      "KL/std": 72.9914779663086,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.0041587091982364655,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.643924713134766,
      "fcm_dpo/q_t": 0.4612976908683777,
      "grad_norm": 32.9399528503418,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": 1.3110202550888062,
      "logits/rejected": 1.4908863306045532,
      "logps/chosen": -319.3348693847656,
      "logps/ref_chosen": -266.2465515136719,
      "logps/ref_rejected": -281.1450500488281,
      "logps/rejected": -372.8773193359375,
      "loss": 5.081,
      "margin_dpo/margin_mean": 38.643924713134766,
      "margin_dpo/margin_std": 92.71197509765625,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -46.63015365600586,
      "KL/mean": -73.59488677978516,
      "KL/rejected_KL_mean": -100.55960845947266,
      "KL/std": 73.98436737060547,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.0041587091982364655,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.92945861816406,
      "fcm_dpo/q_t": 0.4459315538406372,
      "grad_norm": 21.79015350341797,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": 1.4297982454299927,
      "logits/rejected": 1.4271399974822998,
      "logps/chosen": -320.93603515625,
      "logps/ref_chosen": -274.30584716796875,
      "logps/ref_rejected": -242.94247436523438,
      "logps/rejected": -343.5020751953125,
      "loss": 4.8332,
      "margin_dpo/margin_mean": 53.92945861816406,
      "margin_dpo/margin_std": 87.0567398071289,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -39.62451934814453,
      "KL/mean": -72.44171905517578,
      "KL/rejected_KL_mean": -105.25891876220703,
      "KL/std": 79.21454620361328,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.004096523858606815,
      "fcm_dpo/delta": -0.025173485279083252,
      "fcm_dpo/margin": 65.63438415527344,
      "fcm_dpo/q_t": 0.43483832478523254,
      "grad_norm": 23.75948143005371,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": 1.381344199180603,
      "logits/rejected": 1.4613856077194214,
      "logps/chosen": -348.8909912109375,
      "logps/ref_chosen": -309.2664489746094,
      "logps/ref_rejected": -298.4182434082031,
      "logps/rejected": -403.6771545410156,
      "loss": 4.6869,
      "margin_dpo/margin_mean": 65.63438415527344,
      "margin_dpo/margin_std": 87.47856140136719,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -42.249027252197266,
      "KL/mean": -73.21882629394531,
      "KL/rejected_KL_mean": -104.18862915039062,
      "KL/std": 75.32534790039062,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.004075795412063599,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.939598083496094,
      "fcm_dpo/q_t": 0.4392468333244324,
      "grad_norm": 26.593839645385742,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 1.1888291835784912,
      "logits/rejected": 1.2278916835784912,
      "logps/chosen": -306.2699279785156,
      "logps/ref_chosen": -264.0209045410156,
      "logps/ref_rejected": -264.3447570800781,
      "logps/rejected": -368.5333557128906,
      "loss": 4.7409,
      "margin_dpo/margin_mean": 61.939598083496094,
      "margin_dpo/margin_std": 89.33555603027344,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -54.78974914550781,
      "KL/mean": -79.25777435302734,
      "KL/rejected_KL_mean": -103.72582244873047,
      "KL/std": 73.52405548095703,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.004046550020575523,
      "fcm_dpo/delta": -0.010292918421328068,
      "fcm_dpo/margin": 48.93608093261719,
      "fcm_dpo/q_t": 0.45221608877182007,
      "grad_norm": 46.7589225769043,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": 1.3027372360229492,
      "logits/rejected": 1.354425311088562,
      "logps/chosen": -318.0085754394531,
      "logps/ref_chosen": -263.2188415527344,
      "logps/ref_rejected": -236.08546447753906,
      "logps/rejected": -339.811279296875,
      "loss": 4.9381,
      "margin_dpo/margin_mean": 48.93608093261719,
      "margin_dpo/margin_std": 90.86285400390625,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -44.51448059082031,
      "KL/mean": -75.58818054199219,
      "KL/rejected_KL_mean": -106.6618881225586,
      "KL/std": 72.63819122314453,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.004042371641844511,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.14739990234375,
      "fcm_dpo/q_t": 0.4390908479690552,
      "grad_norm": 28.664615631103516,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": 1.4003539085388184,
      "logits/rejected": 1.3849118947982788,
      "logps/chosen": -318.3123779296875,
      "logps/ref_chosen": -273.79791259765625,
      "logps/ref_rejected": -242.4090118408203,
      "logps/rejected": -349.07086181640625,
      "loss": 4.7394,
      "margin_dpo/margin_mean": 62.14739990234375,
      "margin_dpo/margin_std": 88.57301330566406,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -45.614280700683594,
      "KL/mean": -75.74589538574219,
      "KL/rejected_KL_mean": -105.87753295898438,
      "KL/std": 73.70301818847656,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.004042371641844511,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.26325988769531,
      "fcm_dpo/q_t": 0.4411590099334717,
      "grad_norm": 27.760183334350586,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 1.3068150281906128,
      "logits/rejected": 1.3720842599868774,
      "logps/chosen": -320.9294738769531,
      "logps/ref_chosen": -275.3151550292969,
      "logps/ref_rejected": -278.1514587402344,
      "logps/rejected": -384.02899169921875,
      "loss": 4.7879,
      "margin_dpo/margin_mean": 60.26325988769531,
      "margin_dpo/margin_std": 96.08689880371094,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -51.229759216308594,
      "KL/mean": -80.50494384765625,
      "KL/rejected_KL_mean": -109.78011322021484,
      "KL/std": 77.16510009765625,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.004042371641844511,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.55035400390625,
      "fcm_dpo/q_t": 0.44291508197784424,
      "grad_norm": 23.643173217773438,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.4086296558380127,
      "logits/rejected": 1.401594877243042,
      "logps/chosen": -326.4291076660156,
      "logps/ref_chosen": -275.1993408203125,
      "logps/ref_rejected": -261.32122802734375,
      "logps/rejected": -371.1013488769531,
      "loss": 4.7677,
      "margin_dpo/margin_mean": 58.55035400390625,
      "margin_dpo/margin_std": 79.92131042480469,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -45.685096740722656,
      "KL/mean": -82.42791748046875,
      "KL/rejected_KL_mean": -119.17076110839844,
      "KL/std": 76.87667846679688,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.004016402177512646,
      "fcm_dpo/delta": -0.010753107257187366,
      "fcm_dpo/margin": 73.48565673828125,
      "fcm_dpo/q_t": 0.42874449491500854,
      "grad_norm": 23.66162109375,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": 1.2755582332611084,
      "logits/rejected": 1.2794712781906128,
      "logps/chosen": -345.5128479003906,
      "logps/ref_chosen": -299.8277587890625,
      "logps/ref_rejected": -251.24798583984375,
      "logps/rejected": -370.4187316894531,
      "loss": 4.5675,
      "margin_dpo/margin_mean": 73.48566436767578,
      "margin_dpo/margin_std": 81.89991760253906,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -43.3099479675293,
      "KL/mean": -70.6261978149414,
      "KL/rejected_KL_mean": -97.94245147705078,
      "KL/std": 74.45367431640625,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.63249588012695,
      "fcm_dpo/q_t": 0.4471120834350586,
      "grad_norm": 31.34970474243164,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": 1.3928419351577759,
      "logits/rejected": 1.3931180238723755,
      "logps/chosen": -323.77105712890625,
      "logps/ref_chosen": -280.4610900878906,
      "logps/ref_rejected": -228.34823608398438,
      "logps/rejected": -326.2906494140625,
      "loss": 4.853,
      "margin_dpo/margin_mean": 54.63249588012695,
      "margin_dpo/margin_std": 90.97674560546875,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -38.92180633544922,
      "KL/mean": -65.41664123535156,
      "KL/rejected_KL_mean": -91.91146850585938,
      "KL/std": 73.55326843261719,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.98966979980469,
      "fcm_dpo/q_t": 0.4487919807434082,
      "grad_norm": 33.82585525512695,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": 1.3202788829803467,
      "logits/rejected": 1.425377368927002,
      "logps/chosen": -358.52191162109375,
      "logps/ref_chosen": -319.60009765625,
      "logps/ref_rejected": -316.42822265625,
      "logps/rejected": -408.3397216796875,
      "loss": 4.8636,
      "margin_dpo/margin_mean": 52.98966979980469,
      "margin_dpo/margin_std": 85.14823913574219,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -49.14684295654297,
      "KL/mean": -76.84310150146484,
      "KL/rejected_KL_mean": -104.53936004638672,
      "KL/std": 72.67108154296875,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.392494201660156,
      "fcm_dpo/q_t": 0.4463779330253601,
      "grad_norm": 21.39147186279297,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": 1.159369707107544,
      "logits/rejected": 1.2854678630828857,
      "logps/chosen": -309.564697265625,
      "logps/ref_chosen": -260.4178466796875,
      "logps/ref_rejected": -270.04498291015625,
      "logps/rejected": -374.5843505859375,
      "loss": 4.8256,
      "margin_dpo/margin_mean": 55.39249801635742,
      "margin_dpo/margin_std": 83.18388366699219,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -49.277122497558594,
      "KL/mean": -77.20388793945312,
      "KL/rejected_KL_mean": -105.13066101074219,
      "KL/std": 75.20745849609375,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.853538513183594,
      "fcm_dpo/q_t": 0.445861279964447,
      "grad_norm": 27.351747512817383,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": 1.3981199264526367,
      "logits/rejected": 1.3963196277618408,
      "logps/chosen": -329.98016357421875,
      "logps/ref_chosen": -280.7030029296875,
      "logps/ref_rejected": -250.76385498046875,
      "logps/rejected": -355.8945007324219,
      "loss": 4.8381,
      "margin_dpo/margin_mean": 55.853538513183594,
      "margin_dpo/margin_std": 91.09385681152344,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -43.688934326171875,
      "KL/mean": -70.56498718261719,
      "KL/rejected_KL_mean": -97.44105529785156,
      "KL/std": 76.60582733154297,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.752140045166016,
      "fcm_dpo/q_t": 0.4479012191295624,
      "grad_norm": 20.401777267456055,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": 1.2429544925689697,
      "logits/rejected": 1.357947826385498,
      "logps/chosen": -344.57537841796875,
      "logps/ref_chosen": -300.8864440917969,
      "logps/ref_rejected": -288.5682067871094,
      "logps/rejected": -386.00927734375,
      "loss": 4.8674,
      "margin_dpo/margin_mean": 53.75213623046875,
      "margin_dpo/margin_std": 86.61689758300781,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -58.81760025024414,
      "KL/mean": -79.3856430053711,
      "KL/rejected_KL_mean": -99.95368957519531,
      "KL/std": 76.75594329833984,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.136085510253906,
      "fcm_dpo/q_t": 0.46036970615386963,
      "grad_norm": 33.80079650878906,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": 1.2450840473175049,
      "logits/rejected": 1.304470419883728,
      "logps/chosen": -352.11993408203125,
      "logps/ref_chosen": -293.3023681640625,
      "logps/ref_rejected": -270.0457763671875,
      "logps/rejected": -369.99945068359375,
      "loss": 5.0674,
      "margin_dpo/margin_mean": 41.136085510253906,
      "margin_dpo/margin_std": 97.15029907226562,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -52.571048736572266,
      "KL/mean": -78.07589721679688,
      "KL/rejected_KL_mean": -103.58074951171875,
      "KL/std": 76.99481964111328,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.00969314575195,
      "fcm_dpo/q_t": 0.450481116771698,
      "grad_norm": 42.809165954589844,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": 1.4673502445220947,
      "logits/rejected": 1.543252944946289,
      "logps/chosen": -335.97027587890625,
      "logps/ref_chosen": -283.39923095703125,
      "logps/ref_rejected": -264.5092468261719,
      "logps/rejected": -368.0899963378906,
      "loss": 4.8918,
      "margin_dpo/margin_mean": 51.00968933105469,
      "margin_dpo/margin_std": 85.04138946533203,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -45.74210739135742,
      "KL/mean": -75.20967102050781,
      "KL/rejected_KL_mean": -104.67723083496094,
      "KL/std": 72.44214630126953,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.93511962890625,
      "fcm_dpo/q_t": 0.4428755044937134,
      "grad_norm": 25.948617935180664,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 1.3020224571228027,
      "logits/rejected": 1.3712279796600342,
      "logps/chosen": -300.2099304199219,
      "logps/ref_chosen": -254.4678192138672,
      "logps/ref_rejected": -252.00326538085938,
      "logps/rejected": -356.68048095703125,
      "loss": 4.7852,
      "margin_dpo/margin_mean": 58.93512725830078,
      "margin_dpo/margin_std": 85.72103881835938,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_KL/chosen_KL_mean": -51.116302490234375,
      "eval_KL/mean": -78.3270263671875,
      "eval_KL/rejected_KL_mean": -105.53775024414062,
      "eval_KL/std": 74.47852325439453,
      "eval_fcm_dpo/beta": 0.0040077464655041695,
      "eval_logits/chosen": 1.4423696994781494,
      "eval_logits/rejected": 1.5059665441513062,
      "eval_logps/chosen": -331.532958984375,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -370.18328857421875,
      "eval_loss": 0.6075664162635803,
      "eval_margin_dpo/margin_mean": 54.42143630981445,
      "eval_margin_dpo/margin_std": 86.70926666259766,
      "eval_runtime": 87.9903,
      "eval_samples_per_second": 22.73,
      "eval_steps_per_second": 2.841,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -43.00956344604492,
      "KL/mean": -78.3116455078125,
      "KL/rejected_KL_mean": -113.61372375488281,
      "KL/std": 78.0262680053711,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.60415649414062,
      "fcm_dpo/q_t": 0.43152254819869995,
      "grad_norm": 24.106670379638672,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 1.174905776977539,
      "logits/rejected": 1.2036446332931519,
      "logps/chosen": -353.609130859375,
      "logps/ref_chosen": -310.5995788574219,
      "logps/ref_rejected": -292.7754821777344,
      "logps/rejected": -406.38922119140625,
      "loss": 4.6144,
      "margin_dpo/margin_mean": 70.60415649414062,
      "margin_dpo/margin_std": 85.71282196044922,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -45.92796325683594,
      "KL/mean": -73.61090087890625,
      "KL/rejected_KL_mean": -101.29386138916016,
      "KL/std": 76.05755615234375,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.004007745999842882,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.36587905883789,
      "fcm_dpo/q_t": 0.44630947709083557,
      "grad_norm": 33.433135986328125,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": 1.1780697107315063,
      "logits/rejected": 1.0984666347503662,
      "logps/chosen": -345.7793273925781,
      "logps/ref_chosen": -299.8513488769531,
      "logps/ref_rejected": -278.5664367675781,
      "logps/rejected": -379.86029052734375,
      "loss": 4.8356,
      "margin_dpo/margin_mean": 55.36587905883789,
      "margin_dpo/margin_std": 87.63803100585938,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -40.932220458984375,
      "KL/mean": -79.71703338623047,
      "KL/rejected_KL_mean": -118.50186157226562,
      "KL/std": 78.32225799560547,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.004000160843133926,
      "fcm_dpo/delta": -0.01907159388065338,
      "fcm_dpo/margin": 77.56963348388672,
      "fcm_dpo/q_t": 0.4247473478317261,
      "grad_norm": 23.552175521850586,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": 1.252455711364746,
      "logits/rejected": 1.2887294292449951,
      "logps/chosen": -343.36383056640625,
      "logps/ref_chosen": -302.4316101074219,
      "logps/ref_rejected": -270.951171875,
      "logps/rejected": -389.4530029296875,
      "loss": 4.501,
      "margin_dpo/margin_mean": 77.56963348388672,
      "margin_dpo/margin_std": 76.52996063232422,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -54.24034118652344,
      "KL/mean": -81.57609558105469,
      "KL/rejected_KL_mean": -108.91184997558594,
      "KL/std": 78.87586212158203,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.6715087890625,
      "fcm_dpo/q_t": 0.44787949323654175,
      "grad_norm": 19.460147857666016,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": 1.2298345565795898,
      "logits/rejected": 1.2617648839950562,
      "logps/chosen": -344.1434326171875,
      "logps/ref_chosen": -289.9031066894531,
      "logps/ref_rejected": -252.84918212890625,
      "logps/rejected": -361.7610168457031,
      "loss": 4.867,
      "margin_dpo/margin_mean": 54.6715087890625,
      "margin_dpo/margin_std": 91.18682861328125,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -44.08171081542969,
      "KL/mean": -75.23601531982422,
      "KL/rejected_KL_mean": -106.39031982421875,
      "KL/std": 77.09130096435547,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.30860900878906,
      "fcm_dpo/q_t": 0.4406077265739441,
      "grad_norm": 24.172969818115234,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": 1.3045892715454102,
      "logits/rejected": 1.488412618637085,
      "logps/chosen": -292.2557678222656,
      "logps/ref_chosen": -248.17404174804688,
      "logps/ref_rejected": -252.23764038085938,
      "logps/rejected": -358.6279296875,
      "loss": 4.7345,
      "margin_dpo/margin_mean": 62.308616638183594,
      "margin_dpo/margin_std": 82.78563690185547,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -51.28913879394531,
      "KL/mean": -77.81452941894531,
      "KL/rejected_KL_mean": -104.33992767333984,
      "KL/std": 76.74649047851562,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.05080032348633,
      "fcm_dpo/q_t": 0.4493902027606964,
      "grad_norm": 31.147188186645508,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": 1.2977628707885742,
      "logits/rejected": 1.370347499847412,
      "logps/chosen": -327.70526123046875,
      "logps/ref_chosen": -276.4161376953125,
      "logps/ref_rejected": -264.0082702636719,
      "logps/rejected": -368.34820556640625,
      "loss": 4.8883,
      "margin_dpo/margin_mean": 53.05079650878906,
      "margin_dpo/margin_std": 90.48548126220703,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -60.00053405761719,
      "KL/mean": -89.13666534423828,
      "KL/rejected_KL_mean": -118.27279663085938,
      "KL/std": 79.67776489257812,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.27226638793945,
      "fcm_dpo/q_t": 0.44449150562286377,
      "grad_norm": 32.04490661621094,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": 1.2821760177612305,
      "logits/rejected": 1.2893635034561157,
      "logps/chosen": -317.4378662109375,
      "logps/ref_chosen": -257.4373474121094,
      "logps/ref_rejected": -219.66390991210938,
      "logps/rejected": -337.9366760253906,
      "loss": 4.8047,
      "margin_dpo/margin_mean": 58.27226257324219,
      "margin_dpo/margin_std": 89.71147918701172,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -44.801414489746094,
      "KL/mean": -75.60383605957031,
      "KL/rejected_KL_mean": -106.40626525878906,
      "KL/std": 75.45152282714844,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.60484313964844,
      "fcm_dpo/q_t": 0.4410284459590912,
      "grad_norm": 24.989063262939453,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": 1.2221871614456177,
      "logits/rejected": 1.336523413658142,
      "logps/chosen": -333.4548034667969,
      "logps/ref_chosen": -288.6533508300781,
      "logps/ref_rejected": -291.54107666015625,
      "logps/rejected": -397.94732666015625,
      "loss": 4.7515,
      "margin_dpo/margin_mean": 61.60485076904297,
      "margin_dpo/margin_std": 83.58112335205078,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -50.01149368286133,
      "KL/mean": -81.52467346191406,
      "KL/rejected_KL_mean": -113.03783416748047,
      "KL/std": 78.5564956665039,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.026336669921875,
      "fcm_dpo/q_t": 0.4400978088378906,
      "grad_norm": 20.662443161010742,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": 1.4748401641845703,
      "logits/rejected": 1.5942414999008179,
      "logps/chosen": -327.7488708496094,
      "logps/ref_chosen": -277.7373962402344,
      "logps/ref_rejected": -272.988525390625,
      "logps/rejected": -386.0263366699219,
      "loss": 4.7583,
      "margin_dpo/margin_mean": 63.026344299316406,
      "margin_dpo/margin_std": 94.7269058227539,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -50.85723114013672,
      "KL/mean": -77.39799499511719,
      "KL/rejected_KL_mean": -103.93877410888672,
      "KL/std": 78.29151153564453,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.003947063814848661,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.08154296875,
      "fcm_dpo/q_t": 0.44943591952323914,
      "grad_norm": 29.98523712158203,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 1.5315519571304321,
      "logits/rejected": 1.5120667219161987,
      "logps/chosen": -325.0783386230469,
      "logps/ref_chosen": -274.2210998535156,
      "logps/ref_rejected": -264.33465576171875,
      "logps/rejected": -368.2734069824219,
      "loss": 4.8938,
      "margin_dpo/margin_mean": 53.08154296875,
      "margin_dpo/margin_std": 93.83206176757812,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -42.975345611572266,
      "KL/mean": -75.03821563720703,
      "KL/rejected_KL_mean": -107.10107421875,
      "KL/std": 77.97648620605469,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.003905675606802106,
      "fcm_dpo/delta": -0.020864099264144897,
      "fcm_dpo/margin": 64.12574005126953,
      "fcm_dpo/q_t": 0.4392542839050293,
      "grad_norm": 25.9801025390625,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 1.3389227390289307,
      "logits/rejected": 1.402255892753601,
      "logps/chosen": -315.84332275390625,
      "logps/ref_chosen": -272.86798095703125,
      "logps/ref_rejected": -250.35772705078125,
      "logps/rejected": -357.45880126953125,
      "loss": 4.7239,
      "margin_dpo/margin_mean": 64.12574005126953,
      "margin_dpo/margin_std": 82.15746307373047,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -58.03956604003906,
      "KL/mean": -84.20643615722656,
      "KL/rejected_KL_mean": -110.37328338623047,
      "KL/std": 78.65727233886719,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.003881728509441018,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.3337287902832,
      "fcm_dpo/q_t": 0.4509528875350952,
      "grad_norm": 26.932531356811523,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": 1.2221918106079102,
      "logits/rejected": 1.3277685642242432,
      "logps/chosen": -325.9587707519531,
      "logps/ref_chosen": -267.919189453125,
      "logps/ref_rejected": -271.73162841796875,
      "logps/rejected": -382.10491943359375,
      "loss": 4.9092,
      "margin_dpo/margin_mean": 52.3337287902832,
      "margin_dpo/margin_std": 92.53997802734375,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -45.1187744140625,
      "KL/mean": -69.5746841430664,
      "KL/rejected_KL_mean": -94.03060150146484,
      "KL/std": 73.81221008300781,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.003881728509441018,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.911827087402344,
      "fcm_dpo/q_t": 0.45407813787460327,
      "grad_norm": 26.03863525390625,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 1.2914186716079712,
      "logits/rejected": 1.3522371053695679,
      "logps/chosen": -310.07684326171875,
      "logps/ref_chosen": -264.95806884765625,
      "logps/ref_rejected": -267.40093994140625,
      "logps/rejected": -361.43157958984375,
      "loss": 4.9311,
      "margin_dpo/margin_mean": 48.911827087402344,
      "margin_dpo/margin_std": 84.29749298095703,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -54.72465133666992,
      "KL/mean": -83.73922729492188,
      "KL/rejected_KL_mean": -112.75379180908203,
      "KL/std": 79.20938110351562,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.003881728509441018,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.029144287109375,
      "fcm_dpo/q_t": 0.4457838237285614,
      "grad_norm": 23.541650772094727,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": 1.357330560684204,
      "logits/rejected": 1.4239108562469482,
      "logps/chosen": -344.73040771484375,
      "logps/ref_chosen": -290.00579833984375,
      "logps/ref_rejected": -266.8810119628906,
      "logps/rejected": -379.63482666015625,
      "loss": 4.8454,
      "margin_dpo/margin_mean": 58.029144287109375,
      "margin_dpo/margin_std": 96.93766021728516,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -41.4872932434082,
      "KL/mean": -76.44320678710938,
      "KL/rejected_KL_mean": -111.39913940429688,
      "KL/std": 77.25920104980469,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.003881728509441018,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 69.9118423461914,
      "fcm_dpo/q_t": 0.4344584047794342,
      "grad_norm": 21.437795639038086,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": 1.271965503692627,
      "logits/rejected": 1.3613249063491821,
      "logps/chosen": -331.2983093261719,
      "logps/ref_chosen": -289.81103515625,
      "logps/ref_rejected": -278.2758483886719,
      "logps/rejected": -389.67498779296875,
      "loss": 4.6617,
      "margin_dpo/margin_mean": 69.9118423461914,
      "margin_dpo/margin_std": 90.83802795410156,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -42.20181655883789,
      "KL/mean": -75.60932922363281,
      "KL/rejected_KL_mean": -109.0168228149414,
      "KL/std": 76.6444320678711,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.0038561278488487005,
      "fcm_dpo/delta": -0.008271346800029278,
      "fcm_dpo/margin": 66.81501007080078,
      "fcm_dpo/q_t": 0.4376019239425659,
      "grad_norm": 28.76243019104004,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": 1.45143723487854,
      "logits/rejected": 1.4353190660476685,
      "logps/chosen": -346.8505859375,
      "logps/ref_chosen": -304.6488037109375,
      "logps/ref_rejected": -264.2703857421875,
      "logps/rejected": -373.2872314453125,
      "loss": 4.7128,
      "margin_dpo/margin_mean": 66.81501007080078,
      "margin_dpo/margin_std": 90.8250503540039,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -47.026371002197266,
      "KL/mean": -75.56942749023438,
      "KL/rejected_KL_mean": -104.11246490478516,
      "KL/std": 75.6543197631836,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.0038561278488487005,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.086097717285156,
      "fcm_dpo/q_t": 0.44674524664878845,
      "grad_norm": 32.3392219543457,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": 1.4738965034484863,
      "logits/rejected": 1.5749702453613281,
      "logps/chosen": -348.9152526855469,
      "logps/ref_chosen": -301.8889465332031,
      "logps/ref_rejected": -275.0347595214844,
      "logps/rejected": -379.147216796875,
      "loss": 4.8472,
      "margin_dpo/margin_mean": 57.086097717285156,
      "margin_dpo/margin_std": 91.12181091308594,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -57.207855224609375,
      "KL/mean": -82.05229187011719,
      "KL/rejected_KL_mean": -106.896728515625,
      "KL/std": 77.17752075195312,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.0038561278488487005,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.688873291015625,
      "fcm_dpo/q_t": 0.45354554057121277,
      "grad_norm": 25.08513641357422,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": 0.9400485754013062,
      "logits/rejected": 0.9536550641059875,
      "logps/chosen": -326.4551696777344,
      "logps/ref_chosen": -269.2473449707031,
      "logps/ref_rejected": -275.5192565917969,
      "logps/rejected": -382.4159851074219,
      "loss": 4.9351,
      "margin_dpo/margin_mean": 49.688873291015625,
      "margin_dpo/margin_std": 83.61482238769531,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -55.11341094970703,
      "KL/mean": -89.05146026611328,
      "KL/rejected_KL_mean": -122.98951721191406,
      "KL/std": 82.61481475830078,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.003842619014903903,
      "fcm_dpo/delta": -0.00878848135471344,
      "fcm_dpo/margin": 67.87610626220703,
      "fcm_dpo/q_t": 0.4370267391204834,
      "grad_norm": 35.49599838256836,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": 1.2981890439987183,
      "logits/rejected": 1.2569706439971924,
      "logps/chosen": -346.27386474609375,
      "logps/ref_chosen": -291.1604309082031,
      "logps/ref_rejected": -245.18325805664062,
      "logps/rejected": -368.1727600097656,
      "loss": 4.7229,
      "margin_dpo/margin_mean": 67.87610626220703,
      "margin_dpo/margin_std": 98.76705169677734,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -50.44483184814453,
      "KL/mean": -83.03559112548828,
      "KL/rejected_KL_mean": -115.6263656616211,
      "KL/std": 79.61160278320312,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 65.18153381347656,
      "fcm_dpo/q_t": 0.43950098752975464,
      "grad_norm": 34.68558883666992,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 1.0663297176361084,
      "logits/rejected": 1.0091660022735596,
      "logps/chosen": -308.9146423339844,
      "logps/ref_chosen": -258.4698181152344,
      "logps/ref_rejected": -242.916015625,
      "logps/rejected": -358.5423889160156,
      "loss": 4.7348,
      "margin_dpo/margin_mean": 65.18153381347656,
      "margin_dpo/margin_std": 91.51910400390625,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -38.10966110229492,
      "KL/mean": -73.8084716796875,
      "KL/rejected_KL_mean": -109.50727844238281,
      "KL/std": 75.95816040039062,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.39763641357422,
      "fcm_dpo/q_t": 0.4337023198604584,
      "grad_norm": 23.60723876953125,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.1269769668579102,
      "logits/rejected": 1.1058261394500732,
      "logps/chosen": -328.25048828125,
      "logps/ref_chosen": -290.1407775878906,
      "logps/ref_rejected": -239.65676879882812,
      "logps/rejected": -349.1640625,
      "loss": 4.6406,
      "margin_dpo/margin_mean": 71.39763641357422,
      "margin_dpo/margin_std": 86.68579864501953,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -53.40057373046875,
      "KL/mean": -77.70088958740234,
      "KL/rejected_KL_mean": -102.00119018554688,
      "KL/std": 77.84686279296875,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.600624084472656,
      "fcm_dpo/q_t": 0.4548417627811432,
      "grad_norm": 23.31188201904297,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": 1.3099086284637451,
      "logits/rejected": 1.3932809829711914,
      "logps/chosen": -330.33660888671875,
      "logps/ref_chosen": -276.93603515625,
      "logps/ref_rejected": -250.398681640625,
      "logps/rejected": -352.3998718261719,
      "loss": 4.9748,
      "margin_dpo/margin_mean": 48.600624084472656,
      "margin_dpo/margin_std": 95.55345153808594,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -44.77511215209961,
      "KL/mean": -79.75292205810547,
      "KL/rejected_KL_mean": -114.7307357788086,
      "KL/std": 79.27979278564453,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 69.95562744140625,
      "fcm_dpo/q_t": 0.4353001117706299,
      "grad_norm": 38.75774383544922,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": 1.2972794771194458,
      "logits/rejected": 1.387000322341919,
      "logps/chosen": -330.2052001953125,
      "logps/ref_chosen": -285.43011474609375,
      "logps/ref_rejected": -258.0743408203125,
      "logps/rejected": -372.80511474609375,
      "loss": 4.6815,
      "margin_dpo/margin_mean": 69.95561981201172,
      "margin_dpo/margin_std": 94.88880920410156,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -40.46819305419922,
      "KL/mean": -73.5742416381836,
      "KL/rejected_KL_mean": -106.6802978515625,
      "KL/std": 75.37496185302734,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.21212005615234,
      "fcm_dpo/q_t": 0.43845605850219727,
      "grad_norm": 32.634830474853516,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": 1.2157546281814575,
      "logits/rejected": 1.1996701955795288,
      "logps/chosen": -339.81317138671875,
      "logps/ref_chosen": -299.3450012207031,
      "logps/ref_rejected": -244.60169982910156,
      "logps/rejected": -351.2820129394531,
      "loss": 4.7036,
      "margin_dpo/margin_mean": 66.21212005615234,
      "margin_dpo/margin_std": 83.20294952392578,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -47.30335235595703,
      "KL/mean": -76.32611846923828,
      "KL/rejected_KL_mean": -105.34891510009766,
      "KL/std": 79.61099243164062,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.04555892944336,
      "fcm_dpo/q_t": 0.44630688428878784,
      "grad_norm": 34.17145919799805,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": 1.326695442199707,
      "logits/rejected": 1.4215983152389526,
      "logps/chosen": -320.2530822753906,
      "logps/ref_chosen": -272.9497375488281,
      "logps/ref_rejected": -274.6335754394531,
      "logps/rejected": -379.9825134277344,
      "loss": 4.8277,
      "margin_dpo/margin_mean": 58.04555892944336,
      "margin_dpo/margin_std": 85.69223022460938,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -41.234642028808594,
      "KL/mean": -72.0417251586914,
      "KL/rejected_KL_mean": -102.84880065917969,
      "KL/std": 76.95781707763672,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.0038291108794510365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.614166259765625,
      "fcm_dpo/q_t": 0.4427450895309448,
      "grad_norm": 21.973712921142578,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 1.2717468738555908,
      "logits/rejected": 1.3161085844039917,
      "logps/chosen": -323.1762390136719,
      "logps/ref_chosen": -281.9415588378906,
      "logps/ref_rejected": -253.20486450195312,
      "logps/rejected": -356.05364990234375,
      "loss": 4.7612,
      "margin_dpo/margin_mean": 61.61417007446289,
      "margin_dpo/margin_std": 82.16539001464844,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -38.76588439941406,
      "KL/mean": -79.86698913574219,
      "KL/rejected_KL_mean": -120.96809387207031,
      "KL/std": 79.4990005493164,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.003823521314188838,
      "fcm_dpo/delta": -0.014683729968965054,
      "fcm_dpo/margin": 82.20220184326172,
      "fcm_dpo/q_t": 0.4239819347858429,
      "grad_norm": 23.181861877441406,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": 1.357930302619934,
      "logits/rejected": 1.3600398302078247,
      "logps/chosen": -319.3824768066406,
      "logps/ref_chosen": -280.6166076660156,
      "logps/ref_rejected": -263.7532958984375,
      "logps/rejected": -384.7214050292969,
      "loss": 4.5024,
      "margin_dpo/margin_mean": 82.20220184326172,
      "margin_dpo/margin_std": 85.21337890625,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -48.4130744934082,
      "KL/mean": -87.22145080566406,
      "KL/rejected_KL_mean": -126.02981567382812,
      "KL/std": 80.02565002441406,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.0037714915815740824,
      "fcm_dpo/delta": -0.017163855955004692,
      "fcm_dpo/margin": 77.61674499511719,
      "fcm_dpo/q_t": 0.429019570350647,
      "grad_norm": 39.33031463623047,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": 1.3960022926330566,
      "logits/rejected": 1.522701382637024,
      "logps/chosen": -334.5753173828125,
      "logps/ref_chosen": -286.1622314453125,
      "logps/ref_rejected": -273.3803405761719,
      "logps/rejected": -399.41015625,
      "loss": 4.5704,
      "margin_dpo/margin_mean": 77.61674499511719,
      "margin_dpo/margin_std": 84.4375,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -48.402774810791016,
      "KL/mean": -78.86277770996094,
      "KL/rejected_KL_mean": -109.32276916503906,
      "KL/std": 76.78802490234375,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.003722705878317356,
      "fcm_dpo/delta": -0.009032982401549816,
      "fcm_dpo/margin": 60.91999435424805,
      "fcm_dpo/q_t": 0.44521915912628174,
      "grad_norm": 24.682924270629883,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": 1.4734047651290894,
      "logits/rejected": 1.5048017501831055,
      "logps/chosen": -314.7393493652344,
      "logps/ref_chosen": -266.3365783691406,
      "logps/ref_rejected": -255.01956176757812,
      "logps/rejected": -364.34234619140625,
      "loss": 4.8388,
      "margin_dpo/margin_mean": 60.91999435424805,
      "margin_dpo/margin_std": 100.00653839111328,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -49.12052917480469,
      "KL/mean": -75.51439666748047,
      "KL/rejected_KL_mean": -101.90824890136719,
      "KL/std": 76.6629867553711,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.787715911865234,
      "fcm_dpo/q_t": 0.45259132981300354,
      "grad_norm": 47.35271072387695,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 1.339878797531128,
      "logits/rejected": 1.3757476806640625,
      "logps/chosen": -331.27191162109375,
      "logps/ref_chosen": -282.1513977050781,
      "logps/ref_rejected": -263.4792175292969,
      "logps/rejected": -365.38751220703125,
      "loss": 4.9308,
      "margin_dpo/margin_mean": 52.7877197265625,
      "margin_dpo/margin_std": 92.8802490234375,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -51.75355529785156,
      "KL/mean": -77.07035827636719,
      "KL/rejected_KL_mean": -102.38716125488281,
      "KL/std": 77.80268859863281,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.63361358642578,
      "fcm_dpo/q_t": 0.4546409845352173,
      "grad_norm": 33.96522903442383,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 1.3775380849838257,
      "logits/rejected": 1.5444860458374023,
      "logps/chosen": -320.4666748046875,
      "logps/ref_chosen": -268.713134765625,
      "logps/ref_rejected": -266.13446044921875,
      "logps/rejected": -368.5216064453125,
      "loss": 4.9423,
      "margin_dpo/margin_mean": 50.63361358642578,
      "margin_dpo/margin_std": 88.77510070800781,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -40.016353607177734,
      "KL/mean": -73.08309936523438,
      "KL/rejected_KL_mean": -106.14985656738281,
      "KL/std": 77.39932250976562,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.13349151611328,
      "fcm_dpo/q_t": 0.44067487120628357,
      "grad_norm": 28.002859115600586,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": 1.4933598041534424,
      "logits/rejected": 1.544476866722107,
      "logps/chosen": -324.00054931640625,
      "logps/ref_chosen": -283.9842224121094,
      "logps/ref_rejected": -253.9568634033203,
      "logps/rejected": -360.1067199707031,
      "loss": 4.7393,
      "margin_dpo/margin_mean": 66.13349151611328,
      "margin_dpo/margin_std": 89.84209442138672,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -58.424339294433594,
      "KL/mean": -84.715087890625,
      "KL/rejected_KL_mean": -111.00582122802734,
      "KL/std": 77.23698425292969,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.581478118896484,
      "fcm_dpo/q_t": 0.45246681571006775,
      "grad_norm": 29.829626083374023,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": 1.2390364408493042,
      "logits/rejected": 1.2642860412597656,
      "logps/chosen": -300.8675231933594,
      "logps/ref_chosen": -242.4431915283203,
      "logps/ref_rejected": -252.87979125976562,
      "logps/rejected": -363.8855895996094,
      "loss": 4.9218,
      "margin_dpo/margin_mean": 52.581478118896484,
      "margin_dpo/margin_std": 90.83574676513672,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -49.839813232421875,
      "KL/mean": -82.16334533691406,
      "KL/rejected_KL_mean": -114.48689270019531,
      "KL/std": 78.09957122802734,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.6470718383789,
      "fcm_dpo/q_t": 0.44202497601509094,
      "grad_norm": 23.587186813354492,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": 1.1821305751800537,
      "logits/rejected": 1.0966591835021973,
      "logps/chosen": -338.1239929199219,
      "logps/ref_chosen": -288.2841796875,
      "logps/ref_rejected": -252.9123992919922,
      "logps/rejected": -367.3992919921875,
      "loss": 4.7595,
      "margin_dpo/margin_mean": 64.64707946777344,
      "margin_dpo/margin_std": 89.26724243164062,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -43.74205780029297,
      "KL/mean": -71.73321533203125,
      "KL/rejected_KL_mean": -99.7243881225586,
      "KL/std": 76.86331939697266,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.982330322265625,
      "fcm_dpo/q_t": 0.44957149028778076,
      "grad_norm": 32.830631256103516,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": 1.2241815328598022,
      "logits/rejected": 1.3168776035308838,
      "logps/chosen": -292.7568664550781,
      "logps/ref_chosen": -249.01483154296875,
      "logps/ref_rejected": -243.81192016601562,
      "logps/rejected": -343.53631591796875,
      "loss": 4.8678,
      "margin_dpo/margin_mean": 55.982330322265625,
      "margin_dpo/margin_std": 86.84974670410156,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -50.30791091918945,
      "KL/mean": -70.29667663574219,
      "KL/rejected_KL_mean": -90.28543853759766,
      "KL/std": 78.44680786132812,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.97752380371094,
      "fcm_dpo/q_t": 0.46402880549430847,
      "grad_norm": 42.163116455078125,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": 1.2004470825195312,
      "logits/rejected": 1.3001919984817505,
      "logps/chosen": -363.23797607421875,
      "logps/ref_chosen": -312.9300537109375,
      "logps/ref_rejected": -296.2721862792969,
      "logps/rejected": -386.5576171875,
      "loss": 5.0788,
      "margin_dpo/margin_mean": 39.97752380371094,
      "margin_dpo/margin_std": 85.77838897705078,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -42.20537185668945,
      "KL/mean": -75.98112487792969,
      "KL/rejected_KL_mean": -109.75688171386719,
      "KL/std": 80.81878662109375,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.5514907836914,
      "fcm_dpo/q_t": 0.4393227994441986,
      "grad_norm": 21.864269256591797,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": 1.4119874238967896,
      "logits/rejected": 1.4590537548065186,
      "logps/chosen": -310.5193176269531,
      "logps/ref_chosen": -268.31390380859375,
      "logps/ref_rejected": -268.7603759765625,
      "logps/rejected": -378.5173034667969,
      "loss": 4.7324,
      "margin_dpo/margin_mean": 67.55149841308594,
      "margin_dpo/margin_std": 92.91728210449219,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -61.66046142578125,
      "KL/mean": -76.74884796142578,
      "KL/rejected_KL_mean": -91.83723449707031,
      "KL/std": 76.654052734375,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 30.17676544189453,
      "fcm_dpo/q_t": 0.4728739261627197,
      "grad_norm": 50.63412857055664,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": 1.1973774433135986,
      "logits/rejected": 1.2726470232009888,
      "logps/chosen": -316.8923645019531,
      "logps/ref_chosen": -255.23190307617188,
      "logps/ref_rejected": -241.1186981201172,
      "logps/rejected": -332.9559326171875,
      "loss": 5.2319,
      "margin_dpo/margin_mean": 30.17676544189453,
      "margin_dpo/margin_std": 92.39370727539062,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -51.51026153564453,
      "KL/mean": -78.6074447631836,
      "KL/rejected_KL_mean": -105.70462036132812,
      "KL/std": 74.79816436767578,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.19435501098633,
      "fcm_dpo/q_t": 0.4512617588043213,
      "grad_norm": 26.494850158691406,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 1.2207159996032715,
      "logits/rejected": 1.2392292022705078,
      "logps/chosen": -302.4906005859375,
      "logps/ref_chosen": -250.9803466796875,
      "logps/ref_rejected": -239.9314422607422,
      "logps/rejected": -345.6360778808594,
      "loss": 4.8976,
      "margin_dpo/margin_mean": 54.19435501098633,
      "margin_dpo/margin_std": 90.15323638916016,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -59.38569259643555,
      "KL/mean": -81.62614440917969,
      "KL/rejected_KL_mean": -103.86658477783203,
      "KL/std": 75.10807800292969,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.480892181396484,
      "fcm_dpo/q_t": 0.4599992036819458,
      "grad_norm": 27.895341873168945,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 1.2230082750320435,
      "logits/rejected": 1.1524131298065186,
      "logps/chosen": -360.3884582519531,
      "logps/ref_chosen": -301.0027770996094,
      "logps/ref_rejected": -263.628662109375,
      "logps/rejected": -367.4952697753906,
      "loss": 5.0351,
      "margin_dpo/margin_mean": 44.480892181396484,
      "margin_dpo/margin_std": 87.8351058959961,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -45.665706634521484,
      "KL/mean": -78.7325439453125,
      "KL/rejected_KL_mean": -111.79939270019531,
      "KL/std": 77.54917907714844,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.1336669921875,
      "fcm_dpo/q_t": 0.44042590260505676,
      "grad_norm": 25.10434341430664,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 1.3836005926132202,
      "logits/rejected": 1.4826031923294067,
      "logps/chosen": -346.5144958496094,
      "logps/ref_chosen": -300.8487854003906,
      "logps/ref_rejected": -296.7118835449219,
      "logps/rejected": -408.51129150390625,
      "loss": 4.7249,
      "margin_dpo/margin_mean": 66.13367462158203,
      "margin_dpo/margin_std": 80.62336730957031,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -61.062076568603516,
      "KL/mean": -86.04753875732422,
      "KL/rejected_KL_mean": -111.03301239013672,
      "KL/std": 75.87094116210938,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.0037059076130390167,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.97093200683594,
      "fcm_dpo/q_t": 0.4551021456718445,
      "grad_norm": 21.035776138305664,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": 1.3698890209197998,
      "logits/rejected": 1.426091194152832,
      "logps/chosen": -332.7104797363281,
      "logps/ref_chosen": -271.6484069824219,
      "logps/ref_rejected": -238.21197509765625,
      "logps/rejected": -349.2449951171875,
      "loss": 4.9789,
      "margin_dpo/margin_mean": 49.97093200683594,
      "margin_dpo/margin_std": 99.68978881835938,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -51.86201477050781,
      "KL/mean": -78.92516326904297,
      "KL/rejected_KL_mean": -105.98829650878906,
      "KL/std": 75.4305648803711,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.0036939773708581924,
      "fcm_dpo/delta": -0.016200680285692215,
      "fcm_dpo/margin": 54.126277923583984,
      "fcm_dpo/q_t": 0.45131391286849976,
      "grad_norm": 24.907398223876953,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": 1.2850100994110107,
      "logits/rejected": 1.3771823644638062,
      "logps/chosen": -318.1585693359375,
      "logps/ref_chosen": -266.2965087890625,
      "logps/ref_rejected": -263.7326965332031,
      "logps/rejected": -369.72100830078125,
      "loss": 4.9007,
      "margin_dpo/margin_mean": 54.126277923583984,
      "margin_dpo/margin_std": 86.3017349243164,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -49.07245635986328,
      "KL/mean": -77.46115112304688,
      "KL/rejected_KL_mean": -105.84982299804688,
      "KL/std": 78.95455169677734,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.777381896972656,
      "fcm_dpo/q_t": 0.449790358543396,
      "grad_norm": 21.016345977783203,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": 1.187804937362671,
      "logits/rejected": 1.197088360786438,
      "logps/chosen": -341.4332275390625,
      "logps/ref_chosen": -292.36077880859375,
      "logps/ref_rejected": -264.35662841796875,
      "logps/rejected": -370.2064514160156,
      "loss": 4.8777,
      "margin_dpo/margin_mean": 56.777381896972656,
      "margin_dpo/margin_std": 94.15119934082031,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -50.22474670410156,
      "KL/mean": -72.84261322021484,
      "KL/rejected_KL_mean": -95.46047973632812,
      "KL/std": 77.91987609863281,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.23573303222656,
      "fcm_dpo/q_t": 0.45987898111343384,
      "grad_norm": 37.857574462890625,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": 1.2107594013214111,
      "logits/rejected": 1.3167543411254883,
      "logps/chosen": -296.9445495605469,
      "logps/ref_chosen": -246.71978759765625,
      "logps/ref_rejected": -261.339599609375,
      "logps/rejected": -356.800048828125,
      "loss": 5.0273,
      "margin_dpo/margin_mean": 45.23573303222656,
      "margin_dpo/margin_std": 87.82057189941406,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -42.3785285949707,
      "KL/mean": -74.67904663085938,
      "KL/rejected_KL_mean": -106.97957611083984,
      "KL/std": 76.47624969482422,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.60104370117188,
      "fcm_dpo/q_t": 0.44266989827156067,
      "grad_norm": 30.77305030822754,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": 1.1651244163513184,
      "logits/rejected": 1.163695216178894,
      "logps/chosen": -340.937255859375,
      "logps/ref_chosen": -298.5587158203125,
      "logps/ref_rejected": -264.5877685546875,
      "logps/rejected": -371.5673522949219,
      "loss": 4.7757,
      "margin_dpo/margin_mean": 64.60104370117188,
      "margin_dpo/margin_std": 91.7476577758789,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -35.432586669921875,
      "KL/mean": -69.55509185791016,
      "KL/rejected_KL_mean": -103.67759704589844,
      "KL/std": 75.20773315429688,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.24501037597656,
      "fcm_dpo/q_t": 0.43926534056663513,
      "grad_norm": 21.11140251159668,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": 1.2083473205566406,
      "logits/rejected": 1.2133511304855347,
      "logps/chosen": -285.28582763671875,
      "logps/ref_chosen": -249.8532257080078,
      "logps/ref_rejected": -233.2117156982422,
      "logps/rejected": -336.88934326171875,
      "loss": 4.7072,
      "margin_dpo/margin_mean": 68.2450180053711,
      "margin_dpo/margin_std": 83.74517822265625,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -45.2320556640625,
      "KL/mean": -74.77685546875,
      "KL/rejected_KL_mean": -104.32167053222656,
      "KL/std": 77.47744750976562,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.08959197998047,
      "fcm_dpo/q_t": 0.4474576413631439,
      "grad_norm": 23.45153045654297,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": 1.2245844602584839,
      "logits/rejected": 1.3806188106536865,
      "logps/chosen": -304.0536804199219,
      "logps/ref_chosen": -258.8216247558594,
      "logps/ref_rejected": -260.4953308105469,
      "logps/rejected": -364.81695556640625,
      "loss": 4.8371,
      "margin_dpo/margin_mean": 59.08959197998047,
      "margin_dpo/margin_std": 87.85456848144531,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -42.272430419921875,
      "KL/mean": -76.05693054199219,
      "KL/rejected_KL_mean": -109.8414077758789,
      "KL/std": 75.5113296508789,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.56897735595703,
      "fcm_dpo/q_t": 0.4399130344390869,
      "grad_norm": 26.06153106689453,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": 1.539032220840454,
      "logits/rejected": 1.444218397140503,
      "logps/chosen": -366.48895263671875,
      "logps/ref_chosen": -324.2165222167969,
      "logps/ref_rejected": -237.05453491210938,
      "logps/rejected": -346.89599609375,
      "loss": 4.7284,
      "margin_dpo/margin_mean": 67.56897735595703,
      "margin_dpo/margin_std": 89.10943603515625,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -57.95246887207031,
      "KL/mean": -82.31636047363281,
      "KL/rejected_KL_mean": -106.68025207519531,
      "KL/std": 79.28754425048828,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.72777557373047,
      "fcm_dpo/q_t": 0.45666706562042236,
      "grad_norm": 26.85666847229004,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 1.2300139665603638,
      "logits/rejected": 1.2062965631484985,
      "logps/chosen": -336.3537292480469,
      "logps/ref_chosen": -278.4012451171875,
      "logps/ref_rejected": -247.72671508789062,
      "logps/rejected": -354.406982421875,
      "loss": 4.9731,
      "margin_dpo/margin_mean": 48.72777557373047,
      "margin_dpo/margin_std": 88.19842529296875,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -45.424713134765625,
      "KL/mean": -74.48233032226562,
      "KL/rejected_KL_mean": -103.53993225097656,
      "KL/std": 74.4049301147461,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.11521911621094,
      "fcm_dpo/q_t": 0.44833046197891235,
      "grad_norm": 30.05942153930664,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.2040914297103882,
      "logits/rejected": 1.2198976278305054,
      "logps/chosen": -337.51934814453125,
      "logps/ref_chosen": -292.0946350097656,
      "logps/ref_rejected": -252.24237060546875,
      "logps/rejected": -355.7823181152344,
      "loss": 4.8529,
      "margin_dpo/margin_mean": 58.11521911621094,
      "margin_dpo/margin_std": 86.816162109375,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -38.94245147705078,
      "KL/mean": -66.85282135009766,
      "KL/rejected_KL_mean": -94.76319885253906,
      "KL/std": 78.1966552734375,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.82073974609375,
      "fcm_dpo/q_t": 0.45043957233428955,
      "grad_norm": 21.585350036621094,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.3405896425247192,
      "logits/rejected": 1.4040348529815674,
      "logps/chosen": -329.55322265625,
      "logps/ref_chosen": -290.61077880859375,
      "logps/ref_rejected": -279.8963928222656,
      "logps/rejected": -374.6595764160156,
      "loss": 4.8717,
      "margin_dpo/margin_mean": 55.820743560791016,
      "margin_dpo/margin_std": 84.73693084716797,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -47.57266616821289,
      "KL/mean": -79.33779907226562,
      "KL/rejected_KL_mean": -111.10292053222656,
      "KL/std": 77.59327697753906,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.530242919921875,
      "fcm_dpo/q_t": 0.443398118019104,
      "grad_norm": 20.25806999206543,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": 1.3769147396087646,
      "logits/rejected": 1.344478726387024,
      "logps/chosen": -363.887939453125,
      "logps/ref_chosen": -316.3152770996094,
      "logps/ref_rejected": -300.5086975097656,
      "logps/rejected": -411.61163330078125,
      "loss": 4.7691,
      "margin_dpo/margin_mean": 63.530242919921875,
      "margin_dpo/margin_std": 85.9817123413086,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -47.60651779174805,
      "KL/mean": -76.80510711669922,
      "KL/rejected_KL_mean": -106.00369262695312,
      "KL/std": 80.44010162353516,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.397159576416016,
      "fcm_dpo/q_t": 0.4481452703475952,
      "grad_norm": 17.79778289794922,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": 1.4645875692367554,
      "logits/rejected": 1.5705629587173462,
      "logps/chosen": -338.15179443359375,
      "logps/ref_chosen": -290.5452575683594,
      "logps/ref_rejected": -297.2102355957031,
      "logps/rejected": -403.2138977050781,
      "loss": 4.8422,
      "margin_dpo/margin_mean": 58.397159576416016,
      "margin_dpo/margin_std": 86.67648315429688,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -43.121334075927734,
      "KL/mean": -67.08934783935547,
      "KL/rejected_KL_mean": -91.057373046875,
      "KL/std": 74.71138000488281,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.936031341552734,
      "fcm_dpo/q_t": 0.4572862982749939,
      "grad_norm": 28.470338821411133,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": 1.3686734437942505,
      "logits/rejected": 1.4768179655075073,
      "logps/chosen": -306.5416259765625,
      "logps/ref_chosen": -263.4203186035156,
      "logps/ref_rejected": -270.30291748046875,
      "logps/rejected": -361.36029052734375,
      "loss": 4.978,
      "margin_dpo/margin_mean": 47.936031341552734,
      "margin_dpo/margin_std": 86.41947937011719,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -48.116947174072266,
      "KL/mean": -83.46867370605469,
      "KL/rejected_KL_mean": -118.82038879394531,
      "KL/std": 82.51374816894531,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.70342254638672,
      "fcm_dpo/q_t": 0.4376566410064697,
      "grad_norm": 27.9421329498291,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": 1.2762987613677979,
      "logits/rejected": 1.4544219970703125,
      "logps/chosen": -360.154296875,
      "logps/ref_chosen": -312.0373840332031,
      "logps/ref_rejected": -290.50946044921875,
      "logps/rejected": -409.329833984375,
      "loss": 4.7104,
      "margin_dpo/margin_mean": 70.70342254638672,
      "margin_dpo/margin_std": 97.98992156982422,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -52.338478088378906,
      "KL/mean": -84.19143676757812,
      "KL/rejected_KL_mean": -116.04435729980469,
      "KL/std": 80.43722534179688,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.70588684082031,
      "fcm_dpo/q_t": 0.4434521496295929,
      "grad_norm": 29.51431655883789,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": 1.2222213745117188,
      "logits/rejected": 1.2590748071670532,
      "logps/chosen": -349.5104675292969,
      "logps/ref_chosen": -297.1719970703125,
      "logps/ref_rejected": -269.15557861328125,
      "logps/rejected": -385.199951171875,
      "loss": 4.7831,
      "margin_dpo/margin_mean": 63.70589065551758,
      "margin_dpo/margin_std": 92.642578125,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -48.10091781616211,
      "KL/mean": -76.14604949951172,
      "KL/rejected_KL_mean": -104.19119262695312,
      "KL/std": 80.67742919921875,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.090274810791016,
      "fcm_dpo/q_t": 0.45033296942710876,
      "grad_norm": 23.2523136138916,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": 1.0998769998550415,
      "logits/rejected": 1.0086082220077515,
      "logps/chosen": -335.4658508300781,
      "logps/ref_chosen": -287.3648986816406,
      "logps/ref_rejected": -271.1472473144531,
      "logps/rejected": -375.33843994140625,
      "loss": 4.8799,
      "margin_dpo/margin_mean": 56.09028244018555,
      "margin_dpo/margin_std": 88.31593322753906,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -55.21846389770508,
      "KL/mean": -76.23003387451172,
      "KL/rejected_KL_mean": -97.24160766601562,
      "KL/std": 77.12643432617188,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.02315139770508,
      "fcm_dpo/q_t": 0.46281763911247253,
      "grad_norm": 27.277854919433594,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": 1.2196555137634277,
      "logits/rejected": 1.3054720163345337,
      "logps/chosen": -343.8245849609375,
      "logps/ref_chosen": -288.6061096191406,
      "logps/ref_rejected": -285.6366271972656,
      "logps/rejected": -382.8782043457031,
      "loss": 5.0795,
      "margin_dpo/margin_mean": 42.02314758300781,
      "margin_dpo/margin_std": 93.57447814941406,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -52.710323333740234,
      "KL/mean": -74.15288543701172,
      "KL/rejected_KL_mean": -95.59544372558594,
      "KL/std": 76.18132781982422,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.88510513305664,
      "fcm_dpo/q_t": 0.46200722455978394,
      "grad_norm": 29.442012786865234,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 1.2408792972564697,
      "logits/rejected": 1.1949920654296875,
      "logps/chosen": -362.9622497558594,
      "logps/ref_chosen": -310.251953125,
      "logps/ref_rejected": -252.962158203125,
      "logps/rejected": -348.55755615234375,
      "loss": 5.0775,
      "margin_dpo/margin_mean": 42.88510513305664,
      "margin_dpo/margin_std": 97.67604064941406,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -49.34532928466797,
      "KL/mean": -76.92855072021484,
      "KL/rejected_KL_mean": -104.51174926757812,
      "KL/std": 79.2838363647461,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.16642761230469,
      "fcm_dpo/q_t": 0.4510268568992615,
      "grad_norm": 47.549560546875,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 1.2061549425125122,
      "logits/rejected": 1.3406221866607666,
      "logps/chosen": -286.1257629394531,
      "logps/ref_chosen": -236.7804412841797,
      "logps/ref_rejected": -229.67550659179688,
      "logps/rejected": -334.187255859375,
      "loss": 4.9106,
      "margin_dpo/margin_mean": 55.16642761230469,
      "margin_dpo/margin_std": 97.54270935058594,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -41.77191162109375,
      "KL/mean": -77.37147521972656,
      "KL/rejected_KL_mean": -112.9710464477539,
      "KL/std": 79.20802307128906,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.19914245605469,
      "fcm_dpo/q_t": 0.4369131326675415,
      "grad_norm": 26.775148391723633,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": 1.3392250537872314,
      "logits/rejected": 1.2520273923873901,
      "logps/chosen": -377.075439453125,
      "logps/ref_chosen": -335.30352783203125,
      "logps/ref_rejected": -250.6796875,
      "logps/rejected": -363.6507263183594,
      "loss": 4.7022,
      "margin_dpo/margin_mean": 71.19914245605469,
      "margin_dpo/margin_std": 98.14674377441406,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -51.13401794433594,
      "KL/mean": -76.95256805419922,
      "KL/rejected_KL_mean": -102.77111053466797,
      "KL/std": 77.6183090209961,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.63710403442383,
      "fcm_dpo/q_t": 0.45403826236724854,
      "grad_norm": 34.29604721069336,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": 1.421089768409729,
      "logits/rejected": 1.2648961544036865,
      "logps/chosen": -339.3542785644531,
      "logps/ref_chosen": -288.22021484375,
      "logps/ref_rejected": -227.96673583984375,
      "logps/rejected": -330.7378234863281,
      "loss": 4.9291,
      "margin_dpo/margin_mean": 51.63710021972656,
      "margin_dpo/margin_std": 86.66275024414062,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -53.33587646484375,
      "KL/mean": -80.51286315917969,
      "KL/rejected_KL_mean": -107.68983459472656,
      "KL/std": 76.8876953125,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.0036581868771463633,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.35396194458008,
      "fcm_dpo/q_t": 0.45153653621673584,
      "grad_norm": 24.03611183166504,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": 1.4485692977905273,
      "logits/rejected": 1.48708176612854,
      "logps/chosen": -313.6728210449219,
      "logps/ref_chosen": -260.3369140625,
      "logps/ref_rejected": -254.2083740234375,
      "logps/rejected": -361.8982238769531,
      "loss": 4.8983,
      "margin_dpo/margin_mean": 54.35396194458008,
      "margin_dpo/margin_std": 87.22010803222656,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -35.262542724609375,
      "KL/mean": -70.5417709350586,
      "KL/rejected_KL_mean": -105.82099914550781,
      "KL/std": 77.85182189941406,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.0036183190532028675,
      "fcm_dpo/delta": -0.02050764113664627,
      "fcm_dpo/margin": 70.5584487915039,
      "fcm_dpo/q_t": 0.4380248188972473,
      "grad_norm": 20.150287628173828,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.3886370658874512,
      "logits/rejected": 1.4363064765930176,
      "logps/chosen": -318.3320617675781,
      "logps/ref_chosen": -283.0694885253906,
      "logps/ref_rejected": -289.1370544433594,
      "logps/rejected": -394.95806884765625,
      "loss": 4.7076,
      "margin_dpo/margin_mean": 70.5584487915039,
      "margin_dpo/margin_std": 87.4588851928711,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -37.81130599975586,
      "KL/mean": -74.4854736328125,
      "KL/rejected_KL_mean": -111.15962982177734,
      "KL/std": 78.57888793945312,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.003598659997805953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.34832000732422,
      "fcm_dpo/q_t": 0.43576788902282715,
      "grad_norm": 18.355371475219727,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": 1.069178819656372,
      "logits/rejected": 1.1379525661468506,
      "logps/chosen": -294.69635009765625,
      "logps/ref_chosen": -256.8850402832031,
      "logps/ref_rejected": -247.0802459716797,
      "logps/rejected": -358.2398681640625,
      "loss": 4.6583,
      "margin_dpo/margin_mean": 73.34831237792969,
      "margin_dpo/margin_std": 86.43624877929688,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -50.90987777709961,
      "KL/mean": -77.59112548828125,
      "KL/rejected_KL_mean": -104.27238464355469,
      "KL/std": 78.2651596069336,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.003598659997805953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.36250305175781,
      "fcm_dpo/q_t": 0.4534546136856079,
      "grad_norm": 22.07503890991211,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": 0.8889118432998657,
      "logits/rejected": 1.0434510707855225,
      "logps/chosen": -308.4339599609375,
      "logps/ref_chosen": -257.5240783691406,
      "logps/ref_rejected": -274.82122802734375,
      "logps/rejected": -379.0935974121094,
      "loss": 4.9366,
      "margin_dpo/margin_mean": 53.36250305175781,
      "margin_dpo/margin_std": 93.30384063720703,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -42.303672790527344,
      "KL/mean": -66.17556762695312,
      "KL/rejected_KL_mean": -90.04745483398438,
      "KL/std": 75.9640884399414,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.003598659997805953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.74378204345703,
      "fcm_dpo/q_t": 0.4582906663417816,
      "grad_norm": 20.87451171875,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": 1.4102617502212524,
      "logits/rejected": 1.3042927980422974,
      "logps/chosen": -354.5633850097656,
      "logps/ref_chosen": -312.2597351074219,
      "logps/ref_rejected": -259.74383544921875,
      "logps/rejected": -349.79132080078125,
      "loss": 4.9819,
      "margin_dpo/margin_mean": 47.7437858581543,
      "margin_dpo/margin_std": 81.59993743896484,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -37.20415496826172,
      "KL/mean": -68.33615112304688,
      "KL/rejected_KL_mean": -99.46813201904297,
      "KL/std": 79.89981079101562,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.003598659997805953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.26397705078125,
      "fcm_dpo/q_t": 0.4454854130744934,
      "grad_norm": 22.428630828857422,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": 1.1693406105041504,
      "logits/rejected": 1.0840368270874023,
      "logps/chosen": -354.2438659667969,
      "logps/ref_chosen": -317.03973388671875,
      "logps/ref_rejected": -260.12628173828125,
      "logps/rejected": -359.5943908691406,
      "loss": 4.8225,
      "margin_dpo/margin_mean": 62.26397705078125,
      "margin_dpo/margin_std": 95.2547607421875,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -36.61582565307617,
      "KL/mean": -72.30239868164062,
      "KL/rejected_KL_mean": -107.98896789550781,
      "KL/std": 79.65392303466797,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.003595301415771246,
      "fcm_dpo/delta": -0.009367619641125202,
      "fcm_dpo/margin": 71.37315368652344,
      "fcm_dpo/q_t": 0.43775808811187744,
      "grad_norm": 18.602933883666992,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 1.2498624324798584,
      "logits/rejected": 1.1835665702819824,
      "logps/chosen": -341.0097961425781,
      "logps/ref_chosen": -304.3939514160156,
      "logps/ref_rejected": -257.6587219238281,
      "logps/rejected": -365.6476745605469,
      "loss": 4.7058,
      "margin_dpo/margin_mean": 71.37315368652344,
      "margin_dpo/margin_std": 91.8282241821289,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -49.831260681152344,
      "KL/mean": -75.12224578857422,
      "KL/rejected_KL_mean": -100.41322326660156,
      "KL/std": 79.18484497070312,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.0035717918071895838,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.581974029541016,
      "fcm_dpo/q_t": 0.45614272356033325,
      "grad_norm": 28.72824478149414,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 1.2926006317138672,
      "logits/rejected": 1.3691781759262085,
      "logps/chosen": -334.77508544921875,
      "logps/ref_chosen": -284.94384765625,
      "logps/ref_rejected": -297.4357604980469,
      "logps/rejected": -397.8489685058594,
      "loss": 4.9582,
      "margin_dpo/margin_mean": 50.581974029541016,
      "margin_dpo/margin_std": 87.95747375488281,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -38.774532318115234,
      "KL/mean": -72.40868377685547,
      "KL/rejected_KL_mean": -106.04283142089844,
      "KL/std": 77.18767547607422,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.0035717918071895838,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.26830291748047,
      "fcm_dpo/q_t": 0.4418713450431824,
      "grad_norm": 30.57582664489746,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": 1.133286952972412,
      "logits/rejected": 1.2634952068328857,
      "logps/chosen": -271.1287841796875,
      "logps/ref_chosen": -232.35427856445312,
      "logps/ref_rejected": -245.6057891845703,
      "logps/rejected": -351.64862060546875,
      "loss": 4.7571,
      "margin_dpo/margin_mean": 67.26830291748047,
      "margin_dpo/margin_std": 93.08900451660156,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -45.16969299316406,
      "KL/mean": -78.71510314941406,
      "KL/rejected_KL_mean": -112.26051330566406,
      "KL/std": 77.87870788574219,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.0035717918071895838,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.09082794189453,
      "fcm_dpo/q_t": 0.44179800152778625,
      "grad_norm": 29.063079833984375,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": 1.1688774824142456,
      "logits/rejected": 1.2187834978103638,
      "logps/chosen": -323.17657470703125,
      "logps/ref_chosen": -278.0068664550781,
      "logps/ref_rejected": -257.19256591796875,
      "logps/rejected": -369.4530944824219,
      "loss": 4.7546,
      "margin_dpo/margin_mean": 67.09082794189453,
      "margin_dpo/margin_std": 91.21961212158203,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -56.69369125366211,
      "KL/mean": -81.50211334228516,
      "KL/rejected_KL_mean": -106.31055450439453,
      "KL/std": 76.85712432861328,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.0035717918071895838,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.61687088012695,
      "fcm_dpo/q_t": 0.45681384205818176,
      "grad_norm": 29.684049606323242,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": 1.1812150478363037,
      "logits/rejected": 1.2376434803009033,
      "logps/chosen": -337.4512023925781,
      "logps/ref_chosen": -280.75750732421875,
      "logps/ref_rejected": -264.34295654296875,
      "logps/rejected": -370.6534729003906,
      "loss": 4.9709,
      "margin_dpo/margin_mean": 49.61687469482422,
      "margin_dpo/margin_std": 83.8313217163086,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -42.402099609375,
      "KL/mean": -68.49125671386719,
      "KL/rejected_KL_mean": -94.5804214477539,
      "KL/std": 73.75789642333984,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.0035717918071895838,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.17831802368164,
      "fcm_dpo/q_t": 0.454693078994751,
      "grad_norm": 25.453914642333984,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": 1.1497206687927246,
      "logits/rejected": 1.279689908027649,
      "logps/chosen": -283.87347412109375,
      "logps/ref_chosen": -241.4713592529297,
      "logps/ref_rejected": -244.0782928466797,
      "logps/rejected": -338.6587219238281,
      "loss": 4.9369,
      "margin_dpo/margin_mean": 52.17831802368164,
      "margin_dpo/margin_std": 85.9317855834961,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -32.786128997802734,
      "KL/mean": -63.359291076660156,
      "KL/rejected_KL_mean": -93.93244934082031,
      "KL/std": 74.93460083007812,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.0035561479162424803,
      "fcm_dpo/delta": -0.008790440857410431,
      "fcm_dpo/margin": 61.146324157714844,
      "fcm_dpo/q_t": 0.44710004329681396,
      "grad_norm": 18.6362361907959,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": 1.3854923248291016,
      "logits/rejected": 1.3980940580368042,
      "logps/chosen": -351.5604248046875,
      "logps/ref_chosen": -318.7742919921875,
      "logps/ref_rejected": -293.66595458984375,
      "logps/rejected": -387.5984191894531,
      "loss": 4.8331,
      "margin_dpo/margin_mean": 61.146324157714844,
      "margin_dpo/margin_std": 90.1566162109375,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -43.47909164428711,
      "KL/mean": -76.57449340820312,
      "KL/rejected_KL_mean": -109.66988372802734,
      "KL/std": 79.80995178222656,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.003546761581674218,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.19078063964844,
      "fcm_dpo/q_t": 0.4431159198284149,
      "grad_norm": 19.38698387145996,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": 1.2483373880386353,
      "logits/rejected": 1.2916128635406494,
      "logps/chosen": -309.414794921875,
      "logps/ref_chosen": -265.9356994628906,
      "logps/ref_rejected": -243.6986083984375,
      "logps/rejected": -353.3684997558594,
      "loss": 4.7827,
      "margin_dpo/margin_mean": 66.19078063964844,
      "margin_dpo/margin_std": 96.6772232055664,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 4.986182768639779,
      "train_runtime": 7647.9571,
      "train_samples_per_second": 7.994,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}