{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.04536247253417969,
      "KL/mean": -0.05389120429754257,
      "KL/rejected_KL_mean": -0.06241607666015625,
      "KL/std": 0.5513200759887695,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01704825460910797,
      "fcm_dpo/q_t": 0.499957412481308,
      "grad_norm": 14.618134498596191,
      "learning_rate": 0.0,
      "logits/chosen": 1.9854813814163208,
      "logits/rejected": 1.7737188339233398,
      "logps/chosen": -267.0936584472656,
      "logps/ref_chosen": -267.04827880859375,
      "logps/ref_rejected": -219.76657104492188,
      "logps/rejected": -219.82899475097656,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.01704716682434082,
      "margin_dpo/margin_std": 0.7467707991600037,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.07550811767578125,
      "KL/mean": 0.03060000389814377,
      "KL/rejected_KL_mean": -0.014301300048828125,
      "KL/std": 0.6403332352638245,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08979703485965729,
      "fcm_dpo/q_t": 0.4997755289077759,
      "grad_norm": 15.140932083129883,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 2.1993117332458496,
      "logits/rejected": 2.1368322372436523,
      "logps/chosen": -256.5101623535156,
      "logps/ref_chosen": -256.585693359375,
      "logps/ref_rejected": -239.92742919921875,
      "logps/rejected": -239.94171142578125,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08979731798171997,
      "margin_dpo/margin_std": 0.9745746850967407,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": -0.03672599792480469,
      "KL/mean": -0.04589029401540756,
      "KL/rejected_KL_mean": -0.055049896240234375,
      "KL/std": 0.6621268391609192,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.018326863646507263,
      "fcm_dpo/q_t": 0.4999541640281677,
      "grad_norm": 14.624048233032227,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": 2.256566286087036,
      "logits/rejected": 2.2959213256835938,
      "logps/chosen": -265.7055969238281,
      "logps/ref_chosen": -265.66888427734375,
      "logps/ref_rejected": -284.3778991699219,
      "logps/rejected": -284.43292236328125,
      "loss": 5.5445,
      "margin_dpo/margin_mean": 0.01832614839076996,
      "margin_dpo/margin_std": 0.7637921571731567,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.08063888549804688,
      "KL/mean": 0.007859684526920319,
      "KL/rejected_KL_mean": -0.06490516662597656,
      "KL/std": 0.733795166015625,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.14553968608379364,
      "fcm_dpo/q_t": 0.49963611364364624,
      "grad_norm": 15.752913475036621,
      "learning_rate": 3.125e-08,
      "logits/chosen": 1.8713157176971436,
      "logits/rejected": 1.8103328943252563,
      "logps/chosen": -319.4268493652344,
      "logps/ref_chosen": -319.50750732421875,
      "logps/ref_rejected": -296.22314453125,
      "logps/rejected": -296.2880859375,
      "loss": 5.5395,
      "margin_dpo/margin_mean": 0.14553971588611603,
      "margin_dpo/margin_std": 0.9613664150238037,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.0189056396484375,
      "KL/mean": 0.05425246059894562,
      "KL/rejected_KL_mean": 0.08960533142089844,
      "KL/std": 0.624768853187561,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0706944465637207,
      "fcm_dpo/q_t": 0.500176727771759,
      "grad_norm": 15.613986015319824,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": 1.947765827178955,
      "logits/rejected": 1.9637106657028198,
      "logps/chosen": -292.85406494140625,
      "logps/ref_chosen": -292.87298583984375,
      "logps/ref_rejected": -270.3802795410156,
      "logps/rejected": -270.2906799316406,
      "loss": 5.5481,
      "margin_dpo/margin_mean": -0.0706949383020401,
      "margin_dpo/margin_std": 0.8553478121757507,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": 0.02173614501953125,
      "KL/mean": 0.06292004138231277,
      "KL/rejected_KL_mean": 0.10411453247070312,
      "KL/std": 0.5562564730644226,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.08236463367938995,
      "fcm_dpo/q_t": 0.5002058744430542,
      "grad_norm": 15.668643951416016,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 2.200660228729248,
      "logits/rejected": 2.1413097381591797,
      "logps/chosen": -280.26373291015625,
      "logps/ref_chosen": -280.28546142578125,
      "logps/ref_rejected": -242.7611541748047,
      "logps/rejected": -242.6570281982422,
      "loss": 5.5485,
      "margin_dpo/margin_mean": -0.08236503601074219,
      "margin_dpo/margin_std": 0.8366300463676453,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.08570480346679688,
      "KL/mean": 0.09450866281986237,
      "KL/rejected_KL_mean": 0.10331153869628906,
      "KL/std": 0.649753987789154,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.01760881394147873,
      "fcm_dpo/q_t": 0.5000439882278442,
      "grad_norm": 15.554636001586914,
      "learning_rate": 6.25e-08,
      "logits/chosen": 2.000718832015991,
      "logits/rejected": 1.912665605545044,
      "logps/chosen": -255.02001953125,
      "logps/ref_chosen": -255.10572814941406,
      "logps/ref_rejected": -251.15402221679688,
      "logps/rejected": -251.05072021484375,
      "loss": 5.546,
      "margin_dpo/margin_mean": -0.01760926842689514,
      "margin_dpo/margin_std": 0.9152372479438782,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.04639434814453125,
      "KL/mean": -0.034158289432525635,
      "KL/rejected_KL_mean": -0.021917343139648438,
      "KL/std": 0.7364514470100403,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.024482697248458862,
      "fcm_dpo/q_t": 0.5000612139701843,
      "grad_norm": 15.968576431274414,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": 2.182833194732666,
      "logits/rejected": 1.9727380275726318,
      "logps/chosen": -345.2279357910156,
      "logps/ref_chosen": -345.18157958984375,
      "logps/ref_rejected": -277.3522644042969,
      "logps/rejected": -277.3741760253906,
      "loss": 5.5463,
      "margin_dpo/margin_mean": -0.024483129382133484,
      "margin_dpo/margin_std": 1.0113160610198975,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.06563186645507812,
      "KL/mean": -0.07423791289329529,
      "KL/rejected_KL_mean": -0.08284759521484375,
      "KL/std": 0.668553352355957,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.017217963933944702,
      "fcm_dpo/q_t": 0.49995696544647217,
      "grad_norm": 14.115337371826172,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": 1.9696584939956665,
      "logits/rejected": 1.9407271146774292,
      "logps/chosen": -276.79595947265625,
      "logps/ref_chosen": -276.7303161621094,
      "logps/ref_rejected": -266.19952392578125,
      "logps/rejected": -266.2823486328125,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.017218172550201416,
      "margin_dpo/margin_std": 0.9198574423789978,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": -0.052978515625,
      "KL/mean": -0.040165700018405914,
      "KL/rejected_KL_mean": -0.027347564697265625,
      "KL/std": 0.5696783661842346,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0256320983171463,
      "fcm_dpo/q_t": 0.5000641345977783,
      "grad_norm": 13.795990943908691,
      "learning_rate": 9.375e-08,
      "logits/chosen": 1.7593462467193604,
      "logits/rejected": 1.7935895919799805,
      "logps/chosen": -246.615234375,
      "logps/ref_chosen": -246.562255859375,
      "logps/ref_rejected": -224.70401000976562,
      "logps/rejected": -224.73135375976562,
      "loss": 5.5463,
      "margin_dpo/margin_mean": -0.0256330668926239,
      "margin_dpo/margin_std": 0.751089334487915,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": -0.07063484191894531,
      "KL/mean": -0.07445236295461655,
      "KL/rejected_KL_mean": -0.0782623291015625,
      "KL/std": 0.588879406452179,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.007626146078109741,
      "fcm_dpo/q_t": 0.49998095631599426,
      "grad_norm": 15.41923713684082,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 2.0225682258605957,
      "logits/rejected": 1.9631130695343018,
      "logps/chosen": -290.57171630859375,
      "logps/ref_chosen": -290.5010986328125,
      "logps/ref_rejected": -253.9559326171875,
      "logps/rejected": -254.03414916992188,
      "loss": 5.545,
      "margin_dpo/margin_mean": 0.0076267048716545105,
      "margin_dpo/margin_std": 0.8931459188461304,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": 0.03703117370605469,
      "KL/mean": 0.07369125634431839,
      "KL/rejected_KL_mean": 0.11035537719726562,
      "KL/std": 0.620603084564209,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0733271986246109,
      "fcm_dpo/q_t": 0.5001833438873291,
      "grad_norm": 14.815467834472656,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": 1.6586673259735107,
      "logits/rejected": 1.6451406478881836,
      "logps/chosen": -278.4814758300781,
      "logps/ref_chosen": -278.5185241699219,
      "logps/ref_rejected": -273.56207275390625,
      "logps/rejected": -273.4517517089844,
      "loss": 5.5482,
      "margin_dpo/margin_mean": -0.07332716882228851,
      "margin_dpo/margin_std": 0.8553886413574219,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.112884521484375,
      "KL/mean": -0.08476006239652634,
      "KL/rejected_KL_mean": -0.056640625,
      "KL/std": 0.6445580124855042,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.056239113211631775,
      "fcm_dpo/q_t": 0.5001406073570251,
      "grad_norm": 15.993651390075684,
      "learning_rate": 1.25e-07,
      "logits/chosen": 1.9179619550704956,
      "logits/rejected": 2.052126407623291,
      "logps/chosen": -264.77001953125,
      "logps/ref_chosen": -264.6571350097656,
      "logps/ref_rejected": -274.3492736816406,
      "logps/rejected": -274.40594482421875,
      "loss": 5.5475,
      "margin_dpo/margin_mean": -0.056239962577819824,
      "margin_dpo/margin_std": 0.8782503604888916,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.03987693786621094,
      "KL/mean": 0.013990066945552826,
      "KL/rejected_KL_mean": -0.011905670166015625,
      "KL/std": 0.5607637166976929,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05178523063659668,
      "fcm_dpo/q_t": 0.49987053871154785,
      "grad_norm": 15.211677551269531,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": 1.9118162393569946,
      "logits/rejected": 1.930105209350586,
      "logps/chosen": -274.0747375488281,
      "logps/ref_chosen": -274.1145935058594,
      "logps/ref_rejected": -261.0951843261719,
      "logps/rejected": -261.10711669921875,
      "loss": 5.5432,
      "margin_dpo/margin_mean": 0.0517846941947937,
      "margin_dpo/margin_std": 0.7899793982505798,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.016139984130859375,
      "KL/mean": 0.017642252147197723,
      "KL/rejected_KL_mean": 0.051418304443359375,
      "KL/std": 0.6150820255279541,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0675523430109024,
      "fcm_dpo/q_t": 0.5001688599586487,
      "grad_norm": 16.07355499267578,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": 2.008852481842041,
      "logits/rejected": 1.868119716644287,
      "logps/chosen": -288.6008605957031,
      "logps/ref_chosen": -288.584716796875,
      "logps/ref_rejected": -224.8094482421875,
      "logps/rejected": -224.75804138183594,
      "loss": 5.548,
      "margin_dpo/margin_mean": -0.06755267083644867,
      "margin_dpo/margin_std": 0.9083594679832458,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": 0.038761138916015625,
      "KL/mean": 0.03501124680042267,
      "KL/rejected_KL_mean": 0.03125762939453125,
      "KL/std": 0.6130394339561462,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.007508665323257446,
      "fcm_dpo/q_t": 0.49998122453689575,
      "grad_norm": 16.738100051879883,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": 1.9427584409713745,
      "logits/rejected": 1.7969815731048584,
      "logps/chosen": -283.80438232421875,
      "logps/ref_chosen": -283.8431091308594,
      "logps/ref_rejected": -248.63645935058594,
      "logps/rejected": -248.605224609375,
      "loss": 5.545,
      "margin_dpo/margin_mean": 0.0075082927942276,
      "margin_dpo/margin_std": 0.8513134121894836,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.03639030456542969,
      "KL/mean": 0.00199677050113678,
      "KL/rejected_KL_mean": -0.032398223876953125,
      "KL/std": 0.5554746985435486,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06878021359443665,
      "fcm_dpo/q_t": 0.4998280107975006,
      "grad_norm": 15.201050758361816,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 1.8058106899261475,
      "logits/rejected": 1.7565910816192627,
      "logps/chosen": -254.59716796875,
      "logps/ref_chosen": -254.6335906982422,
      "logps/ref_rejected": -245.3981170654297,
      "logps/rejected": -245.43052673339844,
      "loss": 5.5425,
      "margin_dpo/margin_mean": 0.0687805712223053,
      "margin_dpo/margin_std": 0.7906660437583923,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.022411346435546875,
      "KL/mean": 0.007853120565414429,
      "KL/rejected_KL_mean": 0.038112640380859375,
      "KL/std": 0.6385173201560974,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.06053623557090759,
      "fcm_dpo/q_t": 0.5001513361930847,
      "grad_norm": 15.104546546936035,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": 1.8749041557312012,
      "logits/rejected": 1.7903879880905151,
      "logps/chosen": -287.7719421386719,
      "logps/ref_chosen": -287.7495422363281,
      "logps/ref_rejected": -273.03363037109375,
      "logps/rejected": -272.9955139160156,
      "loss": 5.5477,
      "margin_dpo/margin_mean": -0.06053653359413147,
      "margin_dpo/margin_std": 0.9036540389060974,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.1364421844482422,
      "KL/mean": -0.03129519522190094,
      "KL/rejected_KL_mean": 0.0738525390625,
      "KL/std": 0.5760006308555603,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.21028992533683777,
      "fcm_dpo/q_t": 0.500525712966919,
      "grad_norm": 14.887845039367676,
      "learning_rate": 1.875e-07,
      "logits/chosen": 1.9813300371170044,
      "logits/rejected": 1.9398515224456787,
      "logps/chosen": -273.54034423828125,
      "logps/ref_chosen": -273.4039001464844,
      "logps/ref_rejected": -260.09722900390625,
      "logps/rejected": -260.0233459472656,
      "loss": 5.5537,
      "margin_dpo/margin_mean": -0.21028956770896912,
      "margin_dpo/margin_std": 0.7919566035270691,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.042339324951171875,
      "KL/mean": 0.03954256325960159,
      "KL/rejected_KL_mean": 0.0367584228515625,
      "KL/std": 0.6084397435188293,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0055839866399765015,
      "fcm_dpo/q_t": 0.49998605251312256,
      "grad_norm": 15.5026216506958,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.753187894821167,
      "logits/rejected": 1.6854172945022583,
      "logps/chosen": -292.9197692871094,
      "logps/ref_chosen": -292.96209716796875,
      "logps/ref_rejected": -256.602783203125,
      "logps/rejected": -256.5660095214844,
      "loss": 5.545,
      "margin_dpo/margin_mean": 0.005584493279457092,
      "margin_dpo/margin_std": 0.7811954021453857,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": -0.0003681182861328125,
      "KL/mean": -0.015139669179916382,
      "KL/rejected_KL_mean": -0.029916763305664062,
      "KL/std": 0.6366888880729675,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02955716848373413,
      "fcm_dpo/q_t": 0.4999260902404785,
      "grad_norm": 15.356853485107422,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 1.9073911905288696,
      "logits/rejected": 1.8031268119812012,
      "logps/chosen": -292.3275451660156,
      "logps/ref_chosen": -292.3271789550781,
      "logps/ref_rejected": -274.20635986328125,
      "logps/rejected": -274.2362976074219,
      "loss": 5.5441,
      "margin_dpo/margin_mean": 0.02955658733844757,
      "margin_dpo/margin_std": 0.8595911264419556,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.07914924621582031,
      "KL/mean": 0.01912657544016838,
      "KL/rejected_KL_mean": -0.04089164733886719,
      "KL/std": 0.561476469039917,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12004531174898148,
      "fcm_dpo/q_t": 0.4996998906135559,
      "grad_norm": 13.75589656829834,
      "learning_rate": 2.1875e-07,
      "logits/chosen": 2.0094804763793945,
      "logits/rejected": 2.015531301498413,
      "logps/chosen": -253.18707275390625,
      "logps/ref_chosen": -253.2662353515625,
      "logps/ref_rejected": -241.4681396484375,
      "logps/rejected": -241.50906372070312,
      "loss": 5.5404,
      "margin_dpo/margin_mean": 0.1200462281703949,
      "margin_dpo/margin_std": 0.7070102691650391,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.029508590698242188,
      "KL/mean": 0.03904109448194504,
      "KL/rejected_KL_mean": 0.0485687255859375,
      "KL/std": 0.589747428894043,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.019068017601966858,
      "fcm_dpo/q_t": 0.5000476837158203,
      "grad_norm": 15.193405151367188,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": 2.2477259635925293,
      "logits/rejected": 2.2513201236724854,
      "logps/chosen": -275.39208984375,
      "logps/ref_chosen": -275.4216003417969,
      "logps/ref_rejected": -264.44024658203125,
      "logps/rejected": -264.3916931152344,
      "loss": 5.546,
      "margin_dpo/margin_mean": -0.019068002700805664,
      "margin_dpo/margin_std": 0.8205732107162476,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": 0.025625228881835938,
      "KL/mean": 0.0019395425915718079,
      "KL/rejected_KL_mean": -0.0217437744140625,
      "KL/std": 0.636381983757019,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04736296832561493,
      "fcm_dpo/q_t": 0.4998816251754761,
      "grad_norm": 16.530879974365234,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": 1.9548557996749878,
      "logits/rejected": 1.9809250831604004,
      "logps/chosen": -293.3519592285156,
      "logps/ref_chosen": -293.3775634765625,
      "logps/ref_rejected": -301.59014892578125,
      "logps/rejected": -301.6119079589844,
      "loss": 5.5434,
      "margin_dpo/margin_mean": 0.04736223816871643,
      "margin_dpo/margin_std": 0.9364877343177795,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.0345916748046875,
      "KL/mean": 0.002740629017353058,
      "KL/rejected_KL_mean": -0.029109954833984375,
      "KL/std": 0.5930765271186829,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06370113790035248,
      "fcm_dpo/q_t": 0.49984073638916016,
      "grad_norm": 14.483416557312012,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.8845875263214111,
      "logits/rejected": 1.9289454221725464,
      "logps/chosen": -240.89907836914062,
      "logps/ref_chosen": -240.93368530273438,
      "logps/ref_rejected": -235.3230743408203,
      "logps/rejected": -235.3521728515625,
      "loss": 5.5427,
      "margin_dpo/margin_mean": 0.06370164453983307,
      "margin_dpo/margin_std": 0.7359025478363037,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": 0.04900932312011719,
      "KL/mean": 0.051973551511764526,
      "KL/rejected_KL_mean": 0.054935455322265625,
      "KL/std": 0.6064915060997009,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.005920231342315674,
      "fcm_dpo/q_t": 0.5000148415565491,
      "grad_norm": 15.5724515914917,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": 1.708189606666565,
      "logits/rejected": 1.6875150203704834,
      "logps/chosen": -273.54632568359375,
      "logps/ref_chosen": -273.5953369140625,
      "logps/ref_rejected": -234.49148559570312,
      "logps/rejected": -234.43655395507812,
      "loss": 5.5455,
      "margin_dpo/margin_mean": -0.005920037627220154,
      "margin_dpo/margin_std": 0.7354068160057068,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": -0.09484100341796875,
      "KL/mean": -0.07518193125724792,
      "KL/rejected_KL_mean": -0.05550956726074219,
      "KL/std": 0.6533788442611694,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03933638334274292,
      "fcm_dpo/q_t": 0.5000983476638794,
      "grad_norm": 15.164326667785645,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": 2.0936453342437744,
      "logits/rejected": 1.8376694917678833,
      "logps/chosen": -314.3587341308594,
      "logps/ref_chosen": -314.26385498046875,
      "logps/ref_rejected": -242.06735229492188,
      "logps/rejected": -242.12283325195312,
      "loss": 5.5468,
      "margin_dpo/margin_mean": -0.03933677077293396,
      "margin_dpo/margin_std": 0.9094981551170349,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.1735382080078125,
      "KL/mean": 0.07980683445930481,
      "KL/rejected_KL_mean": -0.013927459716796875,
      "KL/std": 0.633030116558075,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.18746662139892578,
      "fcm_dpo/q_t": 0.4995313286781311,
      "grad_norm": 15.444108963012695,
      "learning_rate": 2.8125e-07,
      "logits/chosen": 2.056436538696289,
      "logits/rejected": 2.0653440952301025,
      "logps/chosen": -240.56527709960938,
      "logps/ref_chosen": -240.7388153076172,
      "logps/ref_rejected": -237.8331298828125,
      "logps/rejected": -237.84707641601562,
      "loss": 5.5378,
      "margin_dpo/margin_mean": 0.1874668449163437,
      "margin_dpo/margin_std": 0.8341378569602966,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.13158607482910156,
      "KL/mean": 0.07876824587583542,
      "KL/rejected_KL_mean": 0.0259552001953125,
      "KL/std": 0.6310297250747681,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10563193261623383,
      "fcm_dpo/q_t": 0.49973589181900024,
      "grad_norm": 14.743512153625488,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": 2.1906561851501465,
      "logits/rejected": 2.114691734313965,
      "logps/chosen": -307.6978454589844,
      "logps/ref_chosen": -307.82940673828125,
      "logps/ref_rejected": -281.8109130859375,
      "logps/rejected": -281.7849426269531,
      "loss": 5.541,
      "margin_dpo/margin_mean": 0.10563047230243683,
      "margin_dpo/margin_std": 0.9293301701545715,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.11613273620605469,
      "KL/mean": 0.0564073845744133,
      "KL/rejected_KL_mean": -0.003322601318359375,
      "KL/std": 0.6247383952140808,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11947070062160492,
      "fcm_dpo/q_t": 0.4997013211250305,
      "grad_norm": 14.406644821166992,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.8984812498092651,
      "logits/rejected": 1.8988927602767944,
      "logps/chosen": -247.89764404296875,
      "logps/ref_chosen": -248.01376342773438,
      "logps/ref_rejected": -259.0014953613281,
      "logps/rejected": -259.00482177734375,
      "loss": 5.5405,
      "margin_dpo/margin_mean": 0.119469553232193,
      "margin_dpo/margin_std": 0.897596001625061,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.041584014892578125,
      "KL/mean": 0.011088438332080841,
      "KL/rejected_KL_mean": -0.019412994384765625,
      "KL/std": 0.597226083278656,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.060997024178504944,
      "fcm_dpo/q_t": 0.49984750151634216,
      "grad_norm": 13.694740295410156,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 1.9279181957244873,
      "logits/rejected": 1.8626850843429565,
      "logps/chosen": -240.021240234375,
      "logps/ref_chosen": -240.0628204345703,
      "logps/ref_rejected": -201.91021728515625,
      "logps/rejected": -201.92962646484375,
      "loss": 5.5428,
      "margin_dpo/margin_mean": 0.060995444655418396,
      "margin_dpo/margin_std": 0.8025551438331604,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.15796470642089844,
      "KL/mean": 0.11249299347400665,
      "KL/rejected_KL_mean": 0.0670166015625,
      "KL/std": 0.6189375519752502,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09094783663749695,
      "fcm_dpo/q_t": 0.49977266788482666,
      "grad_norm": 15.560298919677734,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": 1.8913456201553345,
      "logits/rejected": 1.8109095096588135,
      "logps/chosen": -293.9742736816406,
      "logps/ref_chosen": -294.1322326660156,
      "logps/ref_rejected": -265.8150634765625,
      "logps/rejected": -265.748046875,
      "loss": 5.5416,
      "margin_dpo/margin_mean": 0.09094801545143127,
      "margin_dpo/margin_std": 0.8829372525215149,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.18601036071777344,
      "KL/mean": 0.15075883269309998,
      "KL/rejected_KL_mean": 0.11549568176269531,
      "KL/std": 0.691373348236084,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07050499320030212,
      "fcm_dpo/q_t": 0.49982374906539917,
      "grad_norm": 14.9357271194458,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 2.0923514366149902,
      "logits/rejected": 2.0680880546569824,
      "logps/chosen": -275.4691162109375,
      "logps/ref_chosen": -275.6551208496094,
      "logps/ref_rejected": -253.5972137451172,
      "logps/rejected": -253.48171997070312,
      "loss": 5.5425,
      "margin_dpo/margin_mean": 0.070505291223526,
      "margin_dpo/margin_std": 0.9538919925689697,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.2265644073486328,
      "KL/mean": 0.10528434813022614,
      "KL/rejected_KL_mean": -0.015995025634765625,
      "KL/std": 0.6891208291053772,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.24255713820457458,
      "fcm_dpo/q_t": 0.49939364194869995,
      "grad_norm": 17.676149368286133,
      "learning_rate": 3.4375e-07,
      "logits/chosen": 1.967774510383606,
      "logits/rejected": 1.8476389646530151,
      "logps/chosen": -332.7435607910156,
      "logps/ref_chosen": -332.9701232910156,
      "logps/ref_rejected": -278.2213439941406,
      "logps/rejected": -278.2373352050781,
      "loss": 5.5356,
      "margin_dpo/margin_mean": 0.2425571084022522,
      "margin_dpo/margin_std": 0.974843442440033,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.20142555236816406,
      "KL/mean": 0.1311315894126892,
      "KL/rejected_KL_mean": 0.060848236083984375,
      "KL/std": 0.6592605113983154,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.14058779180049896,
      "fcm_dpo/q_t": 0.4996485412120819,
      "grad_norm": 13.623767852783203,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": 1.8241209983825684,
      "logits/rejected": 1.8836913108825684,
      "logps/chosen": -258.2274475097656,
      "logps/ref_chosen": -258.42889404296875,
      "logps/ref_rejected": -259.5119934082031,
      "logps/rejected": -259.45111083984375,
      "loss": 5.5396,
      "margin_dpo/margin_mean": 0.14058685302734375,
      "margin_dpo/margin_std": 0.834118664264679,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.39520835876464844,
      "KL/mean": 0.20667989552021027,
      "KL/rejected_KL_mean": 0.018157958984375,
      "KL/std": 0.7614282965660095,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3770589828491211,
      "fcm_dpo/q_t": 0.4990573823451996,
      "grad_norm": 17.739418029785156,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": 1.9250999689102173,
      "logits/rejected": 1.8744609355926514,
      "logps/chosen": -290.2241516113281,
      "logps/ref_chosen": -290.6193542480469,
      "logps/ref_rejected": -280.2503967285156,
      "logps/rejected": -280.23223876953125,
      "loss": 5.5302,
      "margin_dpo/margin_mean": 0.37705913186073303,
      "margin_dpo/margin_std": 1.0507339239120483,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.2626533508300781,
      "KL/mean": 0.18545621633529663,
      "KL/rejected_KL_mean": 0.10825729370117188,
      "KL/std": 0.8073261380195618,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1543990671634674,
      "fcm_dpo/q_t": 0.49961403012275696,
      "grad_norm": 16.74368667602539,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.8011395931243896,
      "logits/rejected": 1.7847681045532227,
      "logps/chosen": -303.3101501464844,
      "logps/ref_chosen": -303.57281494140625,
      "logps/ref_rejected": -294.56878662109375,
      "logps/rejected": -294.46051025390625,
      "loss": 5.5391,
      "margin_dpo/margin_mean": 0.15439794957637787,
      "margin_dpo/margin_std": 1.0543599128723145,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.1928234100341797,
      "KL/mean": 0.1806654930114746,
      "KL/rejected_KL_mean": 0.16851234436035156,
      "KL/std": 0.6902084350585938,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.024316295981407166,
      "fcm_dpo/q_t": 0.4999392032623291,
      "grad_norm": 15.073845863342285,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": 1.8739616870880127,
      "logits/rejected": 1.9309576749801636,
      "logps/chosen": -272.36065673828125,
      "logps/ref_chosen": -272.5534973144531,
      "logps/ref_rejected": -266.9136047363281,
      "logps/rejected": -266.7451171875,
      "loss": 5.5443,
      "margin_dpo/margin_mean": 0.024315327405929565,
      "margin_dpo/margin_std": 0.8675382137298584,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 0.48826026916503906,
      "KL/mean": 0.2726801931858063,
      "KL/rejected_KL_mean": 0.05710792541503906,
      "KL/std": 0.771026074886322,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.431158185005188,
      "fcm_dpo/q_t": 0.4989221394062042,
      "grad_norm": 15.735392570495605,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": 1.806620717048645,
      "logits/rejected": 1.7423847913742065,
      "logps/chosen": -264.89825439453125,
      "logps/ref_chosen": -265.38653564453125,
      "logps/ref_rejected": -244.34774780273438,
      "logps/rejected": -244.2906494140625,
      "loss": 5.5281,
      "margin_dpo/margin_mean": 0.43115881085395813,
      "margin_dpo/margin_std": 0.9863160252571106,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 0.4456062316894531,
      "KL/mean": 0.27801263332366943,
      "KL/rejected_KL_mean": 0.11041259765625,
      "KL/std": 0.7505092024803162,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3351830840110779,
      "fcm_dpo/q_t": 0.49916204810142517,
      "grad_norm": 14.963577270507812,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.8067268133163452,
      "logits/rejected": 1.79926335811615,
      "logps/chosen": -274.44818115234375,
      "logps/ref_chosen": -274.893798828125,
      "logps/ref_rejected": -244.3120880126953,
      "logps/rejected": -244.20169067382812,
      "loss": 5.5319,
      "margin_dpo/margin_mean": 0.33518385887145996,
      "margin_dpo/margin_std": 1.1011130809783936,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 0.37471961975097656,
      "KL/mean": 0.2612507939338684,
      "KL/rejected_KL_mean": 0.1477794647216797,
      "KL/std": 0.8031635284423828,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.22692570090293884,
      "fcm_dpo/q_t": 0.49943268299102783,
      "grad_norm": 17.122434616088867,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 2.00527286529541,
      "logits/rejected": 1.9334419965744019,
      "logps/chosen": -284.4244384765625,
      "logps/ref_chosen": -284.7991027832031,
      "logps/ref_rejected": -248.8882598876953,
      "logps/rejected": -248.74046325683594,
      "loss": 5.5362,
      "margin_dpo/margin_mean": 0.22692617774009705,
      "margin_dpo/margin_std": 1.0667978525161743,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.49989891052246094,
      "KL/mean": 0.32119229435920715,
      "KL/rejected_KL_mean": 0.14249229431152344,
      "KL/std": 0.8011913895606995,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3574041724205017,
      "fcm_dpo/q_t": 0.4991064965724945,
      "grad_norm": 14.545714378356934,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": 1.8545764684677124,
      "logits/rejected": 1.9096955060958862,
      "logps/chosen": -245.3397674560547,
      "logps/ref_chosen": -245.83966064453125,
      "logps/ref_rejected": -259.2870788574219,
      "logps/rejected": -259.14459228515625,
      "loss": 5.531,
      "margin_dpo/margin_mean": 0.35740482807159424,
      "margin_dpo/margin_std": 1.0676418542861938,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.5051307678222656,
      "KL/mean": 0.3226029872894287,
      "KL/rejected_KL_mean": 0.14006996154785156,
      "KL/std": 0.879235029220581,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.36506056785583496,
      "fcm_dpo/q_t": 0.499087393283844,
      "grad_norm": 15.643047332763672,
      "learning_rate": 4.375e-07,
      "logits/chosen": 1.8542848825454712,
      "logits/rejected": 1.836254358291626,
      "logps/chosen": -307.8924255371094,
      "logps/ref_chosen": -308.39752197265625,
      "logps/ref_rejected": -270.3797912597656,
      "logps/rejected": -270.23974609375,
      "loss": 5.5307,
      "margin_dpo/margin_mean": 0.365061491727829,
      "margin_dpo/margin_std": 1.1090121269226074,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 0.6265964508056641,
      "KL/mean": 0.38098621368408203,
      "KL/rejected_KL_mean": 0.1353778839111328,
      "KL/std": 0.8458372354507446,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4912188649177551,
      "fcm_dpo/q_t": 0.498771995306015,
      "grad_norm": 15.776525497436523,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": 1.7811824083328247,
      "logits/rejected": 1.7740356922149658,
      "logps/chosen": -274.7351379394531,
      "logps/ref_chosen": -275.36175537109375,
      "logps/ref_rejected": -279.7540588378906,
      "logps/rejected": -279.61871337890625,
      "loss": 5.5257,
      "margin_dpo/margin_mean": 0.4912201464176178,
      "margin_dpo/margin_std": 1.0937857627868652,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 0.750152587890625,
      "KL/mean": 0.47431397438049316,
      "KL/rejected_KL_mean": 0.19846153259277344,
      "KL/std": 0.9970504641532898,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5517086982727051,
      "fcm_dpo/q_t": 0.4986208975315094,
      "grad_norm": 16.889419555664062,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": 1.9225780963897705,
      "logits/rejected": 1.9059786796569824,
      "logps/chosen": -314.37005615234375,
      "logps/ref_chosen": -315.1202087402344,
      "logps/ref_rejected": -290.34210205078125,
      "logps/rejected": -290.1436462402344,
      "loss": 5.5233,
      "margin_dpo/margin_mean": 0.5517078042030334,
      "margin_dpo/margin_std": 1.3647775650024414,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 0.7611846923828125,
      "KL/mean": 0.42571714520454407,
      "KL/rejected_KL_mean": 0.09025192260742188,
      "KL/std": 1.0334023237228394,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6709355711936951,
      "fcm_dpo/q_t": 0.4983227849006653,
      "grad_norm": 16.471134185791016,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": 1.8256908655166626,
      "logits/rejected": 1.8389817476272583,
      "logps/chosen": -292.1794128417969,
      "logps/ref_chosen": -292.94061279296875,
      "logps/ref_rejected": -264.0979309082031,
      "logps/rejected": -264.0076904296875,
      "loss": 5.5186,
      "margin_dpo/margin_mean": 0.67093425989151,
      "margin_dpo/margin_std": 1.3013173341751099,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 0.6107082366943359,
      "KL/mean": 0.4547903537750244,
      "KL/rejected_KL_mean": 0.2988719940185547,
      "KL/std": 0.9335572719573975,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.31183767318725586,
      "fcm_dpo/q_t": 0.49922043085098267,
      "grad_norm": 13.607484817504883,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": 1.884634256362915,
      "logits/rejected": 1.992400884628296,
      "logps/chosen": -254.2419891357422,
      "logps/ref_chosen": -254.85269165039062,
      "logps/ref_rejected": -255.88380432128906,
      "logps/rejected": -255.58493041992188,
      "loss": 5.5329,
      "margin_dpo/margin_mean": 0.31183677911758423,
      "margin_dpo/margin_std": 1.395988941192627,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 0.8479785919189453,
      "KL/mean": 0.5087786912918091,
      "KL/rejected_KL_mean": 0.1695709228515625,
      "KL/std": 1.156220555305481,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6784172654151917,
      "fcm_dpo/q_t": 0.4983041286468506,
      "grad_norm": 14.795721054077148,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": 2.0796058177948,
      "logits/rejected": 2.07013201713562,
      "logps/chosen": -261.6597595214844,
      "logps/ref_chosen": -262.5077209472656,
      "logps/ref_rejected": -239.25320434570312,
      "logps/rejected": -239.08363342285156,
      "loss": 5.5183,
      "margin_dpo/margin_mean": 0.6784173250198364,
      "margin_dpo/margin_std": 1.3573176860809326,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.9582386016845703,
      "KL/mean": 0.6537343263626099,
      "KL/rejected_KL_mean": 0.3492393493652344,
      "KL/std": 1.161227822303772,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6090017557144165,
      "fcm_dpo/q_t": 0.49847760796546936,
      "grad_norm": 15.251431465148926,
      "learning_rate": 5e-07,
      "logits/chosen": 1.868577241897583,
      "logits/rejected": 1.984484076499939,
      "logps/chosen": -248.56964111328125,
      "logps/ref_chosen": -249.5278778076172,
      "logps/ref_rejected": -254.6465301513672,
      "logps/rejected": -254.29730224609375,
      "loss": 5.5211,
      "margin_dpo/margin_mean": 0.6090023517608643,
      "margin_dpo/margin_std": 1.5797652006149292,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 0.9332027435302734,
      "KL/mean": 0.7218273878097534,
      "KL/rejected_KL_mean": 0.5104503631591797,
      "KL/std": 1.2131540775299072,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.42275744676589966,
      "fcm_dpo/q_t": 0.49894317984580994,
      "grad_norm": 14.623034477233887,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 1.9431989192962646,
      "logits/rejected": 1.9520351886749268,
      "logps/chosen": -291.5685119628906,
      "logps/ref_chosen": -292.501708984375,
      "logps/ref_rejected": -294.6496887207031,
      "logps/rejected": -294.1392517089844,
      "loss": 5.5286,
      "margin_dpo/margin_mean": 0.4227587580680847,
      "margin_dpo/margin_std": 1.566399097442627,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 0.9206581115722656,
      "KL/mean": 0.6049964427947998,
      "KL/rejected_KL_mean": 0.2893257141113281,
      "KL/std": 1.3041614294052124,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6313223838806152,
      "fcm_dpo/q_t": 0.4984217882156372,
      "grad_norm": 14.75484561920166,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 1.5082911252975464,
      "logits/rejected": 1.6356755495071411,
      "logps/chosen": -258.94122314453125,
      "logps/ref_chosen": -259.8619079589844,
      "logps/ref_rejected": -293.1994934082031,
      "logps/rejected": -292.91015625,
      "loss": 5.5203,
      "margin_dpo/margin_mean": 0.6313219666481018,
      "margin_dpo/margin_std": 1.8078160285949707,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 1.229990005493164,
      "KL/mean": 0.7647634744644165,
      "KL/rejected_KL_mean": 0.2995433807373047,
      "KL/std": 1.4671592712402344,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9304394721984863,
      "fcm_dpo/q_t": 0.49767428636550903,
      "grad_norm": 17.076921463012695,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": 2.139455795288086,
      "logits/rejected": 2.0715012550354004,
      "logps/chosen": -297.19537353515625,
      "logps/ref_chosen": -298.42529296875,
      "logps/ref_rejected": -278.5128479003906,
      "logps/rejected": -278.2132873535156,
      "loss": 5.5085,
      "margin_dpo/margin_mean": 0.9304391741752625,
      "margin_dpo/margin_std": 1.9810266494750977,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 1.1322364807128906,
      "KL/mean": 0.7810690402984619,
      "KL/rejected_KL_mean": 0.4298973083496094,
      "KL/std": 1.4851223230361938,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7023428678512573,
      "fcm_dpo/q_t": 0.49824434518814087,
      "grad_norm": 15.991063117980957,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": 1.860839605331421,
      "logits/rejected": 1.9301422834396362,
      "logps/chosen": -274.17626953125,
      "logps/ref_chosen": -275.3085021972656,
      "logps/ref_rejected": -267.97381591796875,
      "logps/rejected": -267.5439453125,
      "loss": 5.5176,
      "margin_dpo/margin_mean": 0.7023434638977051,
      "margin_dpo/margin_std": 1.8617404699325562,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 1.2302780151367188,
      "KL/mean": 0.8439311385154724,
      "KL/rejected_KL_mean": 0.4575843811035156,
      "KL/std": 1.4644657373428345,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7727017402648926,
      "fcm_dpo/q_t": 0.49806851148605347,
      "grad_norm": 14.581894874572754,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": 1.6643800735473633,
      "logits/rejected": 1.6880788803100586,
      "logps/chosen": -296.9170837402344,
      "logps/ref_chosen": -298.1473388671875,
      "logps/ref_rejected": -269.7539367675781,
      "logps/rejected": -269.29632568359375,
      "loss": 5.5148,
      "margin_dpo/margin_mean": 0.7727015614509583,
      "margin_dpo/margin_std": 2.022977352142334,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.1460151672363281,
      "KL/mean": 0.8334861993789673,
      "KL/rejected_KL_mean": 0.5209541320800781,
      "KL/std": 1.348872184753418,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6250677108764648,
      "fcm_dpo/q_t": 0.4984375238418579,
      "grad_norm": 12.835525512695312,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": 1.690880298614502,
      "logits/rejected": 1.743366003036499,
      "logps/chosen": -215.60394287109375,
      "logps/ref_chosen": -216.74998474121094,
      "logps/ref_rejected": -202.7234649658203,
      "logps/rejected": -202.20249938964844,
      "loss": 5.5206,
      "margin_dpo/margin_mean": 0.625067949295044,
      "margin_dpo/margin_std": 1.8053765296936035,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 1.233602523803711,
      "KL/mean": 0.779560387134552,
      "KL/rejected_KL_mean": 0.32552337646484375,
      "KL/std": 1.7314212322235107,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9080754518508911,
      "fcm_dpo/q_t": 0.49773019552230835,
      "grad_norm": 14.39735221862793,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": 1.6636072397232056,
      "logits/rejected": 1.7644498348236084,
      "logps/chosen": -242.4578094482422,
      "logps/ref_chosen": -243.69140625,
      "logps/ref_rejected": -234.970947265625,
      "logps/rejected": -234.64541625976562,
      "loss": 5.5095,
      "margin_dpo/margin_mean": 0.9080753922462463,
      "margin_dpo/margin_std": 2.1765475273132324,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 1.7009468078613281,
      "KL/mean": 1.0212335586547852,
      "KL/rejected_KL_mean": 0.341522216796875,
      "KL/std": 1.7641031742095947,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3594133853912354,
      "fcm_dpo/q_t": 0.496602326631546,
      "grad_norm": 17.691123962402344,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": 1.918069839477539,
      "logits/rejected": 1.809198021888733,
      "logps/chosen": -309.0453796386719,
      "logps/ref_chosen": -310.746337890625,
      "logps/ref_rejected": -270.6407165527344,
      "logps/rejected": -270.2991943359375,
      "loss": 5.4915,
      "margin_dpo/margin_mean": 1.3594138622283936,
      "margin_dpo/margin_std": 2.1983845233917236,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 1.2956829071044922,
      "KL/mean": 0.8063839673995972,
      "KL/rejected_KL_mean": 0.3170909881591797,
      "KL/std": 1.5938609838485718,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9785892963409424,
      "fcm_dpo/q_t": 0.49755361676216125,
      "grad_norm": 14.826539039611816,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": 1.715901494026184,
      "logits/rejected": 1.7014119625091553,
      "logps/chosen": -257.517333984375,
      "logps/ref_chosen": -258.8130187988281,
      "logps/ref_rejected": -268.2101745605469,
      "logps/rejected": -267.8930969238281,
      "loss": 5.5066,
      "margin_dpo/margin_mean": 0.9785890579223633,
      "margin_dpo/margin_std": 1.8862122297286987,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 1.4026012420654297,
      "KL/mean": 0.9664520621299744,
      "KL/rejected_KL_mean": 0.5303077697753906,
      "KL/std": 1.8877261877059937,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8722949624061584,
      "fcm_dpo/q_t": 0.497819721698761,
      "grad_norm": 15.261961936950684,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": 1.8645663261413574,
      "logits/rejected": 1.812644124031067,
      "logps/chosen": -275.8388366699219,
      "logps/ref_chosen": -277.2414245605469,
      "logps/ref_rejected": -259.13519287109375,
      "logps/rejected": -258.6048583984375,
      "loss": 5.511,
      "margin_dpo/margin_mean": 0.8722929954528809,
      "margin_dpo/margin_std": 2.4151854515075684,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 1.5031375885009766,
      "KL/mean": 0.9414680600166321,
      "KL/rejected_KL_mean": 0.37980079650878906,
      "KL/std": 1.694183349609375,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1233311891555786,
      "fcm_dpo/q_t": 0.4971920847892761,
      "grad_norm": 14.749018669128418,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.7953989505767822,
      "logits/rejected": 1.8343262672424316,
      "logps/chosen": -249.84271240234375,
      "logps/ref_chosen": -251.34585571289062,
      "logps/ref_rejected": -251.4120635986328,
      "logps/rejected": -251.03224182128906,
      "loss": 5.5009,
      "margin_dpo/margin_mean": 1.123329997062683,
      "margin_dpo/margin_std": 2.237375497817993,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 1.6720962524414062,
      "KL/mean": 1.1075739860534668,
      "KL/rejected_KL_mean": 0.5430488586425781,
      "KL/std": 1.7782368659973145,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1290454864501953,
      "fcm_dpo/q_t": 0.4971778988838196,
      "grad_norm": 14.994930267333984,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 1.8543516397476196,
      "logits/rejected": 1.8152318000793457,
      "logps/chosen": -267.31048583984375,
      "logps/ref_chosen": -268.98260498046875,
      "logps/ref_rejected": -249.34474182128906,
      "logps/rejected": -248.80169677734375,
      "loss": 5.5008,
      "margin_dpo/margin_mean": 1.1290453672409058,
      "margin_dpo/margin_std": 2.3628954887390137,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 2.033855438232422,
      "KL/mean": 1.4320567846298218,
      "KL/rejected_KL_mean": 0.8302536010742188,
      "KL/std": 1.9428855180740356,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.203598976135254,
      "fcm_dpo/q_t": 0.49699172377586365,
      "grad_norm": 14.76717472076416,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 1.944303274154663,
      "logits/rejected": 1.920440673828125,
      "logps/chosen": -262.1696472167969,
      "logps/ref_chosen": -264.2034912109375,
      "logps/ref_rejected": -260.2911376953125,
      "logps/rejected": -259.4609069824219,
      "loss": 5.4978,
      "margin_dpo/margin_mean": 1.2035984992980957,
      "margin_dpo/margin_std": 2.516101360321045,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 1.8930244445800781,
      "KL/mean": 1.2375445365905762,
      "KL/rejected_KL_mean": 0.582061767578125,
      "KL/std": 2.1726200580596924,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3109641075134277,
      "fcm_dpo/q_t": 0.49672356247901917,
      "grad_norm": 14.794206619262695,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": 1.7451893091201782,
      "logits/rejected": 1.767672061920166,
      "logps/chosen": -269.5752868652344,
      "logps/ref_chosen": -271.4682922363281,
      "logps/ref_rejected": -273.0804748535156,
      "logps/rejected": -272.4984130859375,
      "loss": 5.4938,
      "margin_dpo/margin_mean": 1.3109649419784546,
      "margin_dpo/margin_std": 2.8862228393554688,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 2.358053207397461,
      "KL/mean": 1.5844343900680542,
      "KL/rejected_KL_mean": 0.8108100891113281,
      "KL/std": 2.413503646850586,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5472376346588135,
      "fcm_dpo/q_t": 0.49613311886787415,
      "grad_norm": 15.7456636428833,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": 1.6990879774093628,
      "logits/rejected": 1.6928479671478271,
      "logps/chosen": -271.2453308105469,
      "logps/ref_chosen": -273.6033935546875,
      "logps/ref_rejected": -259.54107666015625,
      "logps/rejected": -258.73028564453125,
      "loss": 5.4846,
      "margin_dpo/margin_mean": 1.5472368001937866,
      "margin_dpo/margin_std": 3.081818103790283,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 2.280658721923828,
      "KL/mean": 1.7423800230026245,
      "KL/rejected_KL_mean": 1.2041034698486328,
      "KL/std": 2.067718744277954,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.076554536819458,
      "fcm_dpo/q_t": 0.49730899930000305,
      "grad_norm": 14.50003433227539,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": 1.7265160083770752,
      "logits/rejected": 1.8276622295379639,
      "logps/chosen": -256.23699951171875,
      "logps/ref_chosen": -258.5176696777344,
      "logps/ref_rejected": -264.9944763183594,
      "logps/rejected": -263.7903747558594,
      "loss": 5.503,
      "margin_dpo/margin_mean": 1.076553225517273,
      "margin_dpo/margin_std": 2.6526999473571777,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 2.257251739501953,
      "KL/mean": 1.538353681564331,
      "KL/rejected_KL_mean": 0.8194503784179688,
      "KL/std": 2.7307987213134766,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4377973079681396,
      "fcm_dpo/q_t": 0.4964069128036499,
      "grad_norm": 14.774269104003906,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": 2.0766892433166504,
      "logits/rejected": 2.187027931213379,
      "logps/chosen": -265.2521667480469,
      "logps/ref_chosen": -267.5094299316406,
      "logps/ref_rejected": -277.03228759765625,
      "logps/rejected": -276.21282958984375,
      "loss": 5.4893,
      "margin_dpo/margin_mean": 1.4377973079681396,
      "margin_dpo/margin_std": 3.6756179332733154,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 2.5749645233154297,
      "KL/mean": 1.9054312705993652,
      "KL/rejected_KL_mean": 1.2359027862548828,
      "KL/std": 2.6004509925842285,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3390545845031738,
      "fcm_dpo/q_t": 0.4966532588005066,
      "grad_norm": 14.22062873840332,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": 1.7028077840805054,
      "logits/rejected": 1.7106297016143799,
      "logps/chosen": -254.88998413085938,
      "logps/ref_chosen": -257.4649353027344,
      "logps/ref_rejected": -247.29161071777344,
      "logps/rejected": -246.0557098388672,
      "loss": 5.4929,
      "margin_dpo/margin_mean": 1.3390545845031738,
      "margin_dpo/margin_std": 3.1694726943969727,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 2.4945125579833984,
      "KL/mean": 1.617600917816162,
      "KL/rejected_KL_mean": 0.7406959533691406,
      "KL/std": 2.9393086433410645,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7538151741027832,
      "fcm_dpo/q_t": 0.4956182837486267,
      "grad_norm": 14.78510856628418,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": 1.7923305034637451,
      "logits/rejected": 1.7484714984893799,
      "logps/chosen": -287.9896545410156,
      "logps/ref_chosen": -290.4841613769531,
      "logps/ref_rejected": -260.8398132324219,
      "logps/rejected": -260.09912109375,
      "loss": 5.477,
      "margin_dpo/margin_mean": 1.7538150548934937,
      "margin_dpo/margin_std": 3.903461217880249,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 2.792255401611328,
      "KL/mean": 1.7233097553253174,
      "KL/rejected_KL_mean": 0.6543674468994141,
      "KL/std": 2.9945123195648193,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.137897253036499,
      "fcm_dpo/q_t": 0.4946572482585907,
      "grad_norm": 15.024805068969727,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": 1.8298320770263672,
      "logits/rejected": 1.8647361993789673,
      "logps/chosen": -254.8453369140625,
      "logps/ref_chosen": -257.6375732421875,
      "logps/ref_rejected": -235.6666717529297,
      "logps/rejected": -235.01229858398438,
      "loss": 5.4616,
      "margin_dpo/margin_mean": 2.137897491455078,
      "margin_dpo/margin_std": 3.7497520446777344,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 3.315113067626953,
      "KL/mean": 2.292840003967285,
      "KL/rejected_KL_mean": 1.2705669403076172,
      "KL/std": 2.9956541061401367,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.044552803039551,
      "fcm_dpo/q_t": 0.4948912262916565,
      "grad_norm": 15.647452354431152,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.6968492269515991,
      "logits/rejected": 1.6497589349746704,
      "logps/chosen": -284.15008544921875,
      "logps/ref_chosen": -287.4651794433594,
      "logps/ref_rejected": -253.4770965576172,
      "logps/rejected": -252.20654296875,
      "loss": 5.4653,
      "margin_dpo/margin_mean": 2.044552803039551,
      "margin_dpo/margin_std": 3.7087039947509766,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 3.6179866790771484,
      "KL/mean": 2.5625696182250977,
      "KL/rejected_KL_mean": 1.507150650024414,
      "KL/std": 3.2792158126831055,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.1108458042144775,
      "fcm_dpo/q_t": 0.49472564458847046,
      "grad_norm": 14.928447723388672,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 1.9985840320587158,
      "logits/rejected": 2.0097146034240723,
      "logps/chosen": -277.3174133300781,
      "logps/ref_chosen": -280.9353942871094,
      "logps/ref_rejected": -275.998779296875,
      "logps/rejected": -274.49163818359375,
      "loss": 5.4633,
      "margin_dpo/margin_mean": 2.1108458042144775,
      "margin_dpo/margin_std": 4.323350429534912,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 2.9976882934570312,
      "KL/mean": 1.9823945760726929,
      "KL/rejected_KL_mean": 0.96710205078125,
      "KL/std": 3.0653140544891357,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0305914878845215,
      "fcm_dpo/q_t": 0.4949260354042053,
      "grad_norm": 14.597126960754395,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": 1.7832833528518677,
      "logits/rejected": 1.9279091358184814,
      "logps/chosen": -227.14785766601562,
      "logps/ref_chosen": -230.14553833007812,
      "logps/ref_rejected": -253.37661743164062,
      "logps/rejected": -252.40951538085938,
      "loss": 5.4661,
      "margin_dpo/margin_mean": 2.030590772628784,
      "margin_dpo/margin_std": 3.9879555702209473,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 3.759725570678711,
      "KL/mean": 2.8096165657043457,
      "KL/rejected_KL_mean": 1.8595046997070312,
      "KL/std": 3.213318109512329,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9002217054367065,
      "fcm_dpo/q_t": 0.49525153636932373,
      "grad_norm": 14.8279447555542,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": 1.6659836769104004,
      "logits/rejected": 1.6833094358444214,
      "logps/chosen": -261.1065673828125,
      "logps/ref_chosen": -264.86627197265625,
      "logps/ref_rejected": -245.21304321289062,
      "logps/rejected": -243.35353088378906,
      "loss": 5.4716,
      "margin_dpo/margin_mean": 1.9002211093902588,
      "margin_dpo/margin_std": 4.318003177642822,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.7834510803222656,
      "KL/mean": 2.7796220779418945,
      "KL/rejected_KL_mean": 1.7758007049560547,
      "KL/std": 3.5168850421905518,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0076563358306885,
      "fcm_dpo/q_t": 0.4949833154678345,
      "grad_norm": 15.704973220825195,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": 1.6338582038879395,
      "logits/rejected": 1.6818538904190063,
      "logps/chosen": -330.7700500488281,
      "logps/ref_chosen": -334.55352783203125,
      "logps/ref_rejected": -325.8403015136719,
      "logps/rejected": -324.0644836425781,
      "loss": 5.4675,
      "margin_dpo/margin_mean": 2.007658004760742,
      "margin_dpo/margin_std": 4.4814019203186035,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.611869812011719,
      "KL/mean": 3.2838966846466064,
      "KL/rejected_KL_mean": 1.9559268951416016,
      "KL/std": 3.616438627243042,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.6559457778930664,
      "fcm_dpo/q_t": 0.493363618850708,
      "grad_norm": 16.99039649963379,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 1.7592467069625854,
      "logits/rejected": 1.8385319709777832,
      "logps/chosen": -255.8100128173828,
      "logps/ref_chosen": -260.421875,
      "logps/ref_rejected": -273.0882263183594,
      "logps/rejected": -271.1322937011719,
      "loss": 5.4416,
      "margin_dpo/margin_mean": 2.65594482421875,
      "margin_dpo/margin_std": 4.165529251098633,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.472038269042969,
      "KL/mean": 2.982435703277588,
      "KL/rejected_KL_mean": 1.492837905883789,
      "KL/std": 3.8591573238372803,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9792044162750244,
      "fcm_dpo/q_t": 0.4925594925880432,
      "grad_norm": 16.7109432220459,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": 1.9359476566314697,
      "logits/rejected": 1.9304898977279663,
      "logps/chosen": -267.2633972167969,
      "logps/ref_chosen": -271.7354431152344,
      "logps/ref_rejected": -262.64111328125,
      "logps/rejected": -261.1482849121094,
      "loss": 5.4295,
      "margin_dpo/margin_mean": 2.9792048931121826,
      "margin_dpo/margin_std": 4.890514850616455,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 4.73944091796875,
      "KL/mean": 3.5847487449645996,
      "KL/rejected_KL_mean": 2.4300613403320312,
      "KL/std": 4.100776672363281,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.3093743324279785,
      "fcm_dpo/q_t": 0.4942324459552765,
      "grad_norm": 14.883354187011719,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": 1.6879582405090332,
      "logits/rejected": 1.5230216979980469,
      "logps/chosen": -289.9311828613281,
      "logps/ref_chosen": -294.6706237792969,
      "logps/ref_rejected": -247.6900177001953,
      "logps/rejected": -245.2599639892578,
      "loss": 5.4563,
      "margin_dpo/margin_mean": 2.309375047683716,
      "margin_dpo/margin_std": 5.1498637199401855,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 5.174715042114258,
      "KL/mean": 3.6691222190856934,
      "KL/rejected_KL_mean": 2.1635208129882812,
      "KL/std": 3.742504835128784,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.011192560195923,
      "fcm_dpo/q_t": 0.49247947335243225,
      "grad_norm": 14.63006591796875,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": 1.4049131870269775,
      "logits/rejected": 1.312789797782898,
      "logps/chosen": -242.97756958007812,
      "logps/ref_chosen": -248.1522979736328,
      "logps/ref_rejected": -209.12808227539062,
      "logps/rejected": -206.9645538330078,
      "loss": 5.4284,
      "margin_dpo/margin_mean": 3.0111918449401855,
      "margin_dpo/margin_std": 5.217677593231201,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 4.970842361450195,
      "KL/mean": 3.447924852371216,
      "KL/rejected_KL_mean": 1.9250030517578125,
      "KL/std": 4.875251770019531,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0458428859710693,
      "fcm_dpo/q_t": 0.49238458275794983,
      "grad_norm": 16.769733428955078,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": 1.801648497581482,
      "logits/rejected": 1.8646525144577026,
      "logps/chosen": -254.50355529785156,
      "logps/ref_chosen": -259.4743957519531,
      "logps/ref_rejected": -246.9627227783203,
      "logps/rejected": -245.0377197265625,
      "loss": 5.4292,
      "margin_dpo/margin_mean": 3.0458428859710693,
      "margin_dpo/margin_std": 6.482162952423096,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 5.315942764282227,
      "KL/mean": 4.034660339355469,
      "KL/rejected_KL_mean": 2.753376007080078,
      "KL/std": 4.733192443847656,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.5625572204589844,
      "fcm_dpo/q_t": 0.4936003088951111,
      "grad_norm": 15.577733039855957,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.5281394720077515,
      "logits/rejected": 1.5076630115509033,
      "logps/chosen": -280.610595703125,
      "logps/ref_chosen": -285.926513671875,
      "logps/ref_rejected": -253.77996826171875,
      "logps/rejected": -251.02659606933594,
      "loss": 5.4471,
      "margin_dpo/margin_mean": 2.5625574588775635,
      "margin_dpo/margin_std": 5.6602301597595215,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 5.944952011108398,
      "KL/mean": 4.446666240692139,
      "KL/rejected_KL_mean": 2.948373794555664,
      "KL/std": 4.439742565155029,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9965803623199463,
      "fcm_dpo/q_t": 0.49251455068588257,
      "grad_norm": 20.235883712768555,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 1.8477516174316406,
      "logits/rejected": 1.7993559837341309,
      "logps/chosen": -256.6492614746094,
      "logps/ref_chosen": -262.5942077636719,
      "logps/ref_rejected": -236.687744140625,
      "logps/rejected": -233.73936462402344,
      "loss": 5.4305,
      "margin_dpo/margin_mean": 2.996581554412842,
      "margin_dpo/margin_std": 6.232707977294922,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 6.362844467163086,
      "KL/mean": 4.823806285858154,
      "KL/rejected_KL_mean": 3.284761428833008,
      "KL/std": 4.32936429977417,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0780715942382812,
      "fcm_dpo/q_t": 0.492312490940094,
      "grad_norm": 15.653958320617676,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": 1.995041012763977,
      "logits/rejected": 1.9689539670944214,
      "logps/chosen": -264.6803283691406,
      "logps/ref_chosen": -271.04315185546875,
      "logps/ref_rejected": -234.03359985351562,
      "logps/rejected": -230.7488250732422,
      "loss": 5.4264,
      "margin_dpo/margin_mean": 3.0780720710754395,
      "margin_dpo/margin_std": 5.431643962860107,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 6.543645858764648,
      "KL/mean": 4.76614236831665,
      "KL/rejected_KL_mean": 2.9886417388916016,
      "KL/std": 4.88400411605835,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.555004596710205,
      "fcm_dpo/q_t": 0.4911275804042816,
      "grad_norm": 15.295555114746094,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": 1.7873146533966064,
      "logits/rejected": 1.8668912649154663,
      "logps/chosen": -259.74163818359375,
      "logps/ref_chosen": -266.2852783203125,
      "logps/ref_rejected": -274.32501220703125,
      "logps/rejected": -271.33636474609375,
      "loss": 5.4091,
      "margin_dpo/margin_mean": 3.5550055503845215,
      "margin_dpo/margin_std": 6.606444835662842,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 6.633832931518555,
      "KL/mean": 5.238231658935547,
      "KL/rejected_KL_mean": 3.8426342010498047,
      "KL/std": 5.041012287139893,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.791196823120117,
      "fcm_dpo/q_t": 0.4930274486541748,
      "grad_norm": 15.445594787597656,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": 1.7801809310913086,
      "logits/rejected": 1.7634309530258179,
      "logps/chosen": -280.14678955078125,
      "logps/ref_chosen": -286.7806091308594,
      "logps/ref_rejected": -276.1176452636719,
      "logps/rejected": -272.2750244140625,
      "loss": 5.4391,
      "margin_dpo/margin_mean": 2.79119610786438,
      "margin_dpo/margin_std": 6.559935092926025,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": 6.293603897094727,
      "KL/mean": 5.2747273445129395,
      "KL/rejected_KL_mean": 4.25584602355957,
      "KL/std": 4.392521381378174,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0377628803253174,
      "fcm_dpo/q_t": 0.4949093163013458,
      "grad_norm": 13.86800479888916,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": 1.7670834064483643,
      "logits/rejected": 1.86789071559906,
      "logps/chosen": -228.73626708984375,
      "logps/ref_chosen": -235.02987670898438,
      "logps/ref_rejected": -259.80999755859375,
      "logps/rejected": -255.55416870117188,
      "loss": 5.4674,
      "margin_dpo/margin_mean": 2.0377633571624756,
      "margin_dpo/margin_std": 5.394864082336426,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 7.72691535949707,
      "KL/mean": 5.854956150054932,
      "KL/rejected_KL_mean": 3.9830093383789062,
      "KL/std": 5.555895805358887,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.7439122200012207,
      "fcm_dpo/q_t": 0.4906545877456665,
      "grad_norm": 16.525583267211914,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": 1.8323261737823486,
      "logits/rejected": 1.8741343021392822,
      "logps/chosen": -293.5858154296875,
      "logps/ref_chosen": -301.312744140625,
      "logps/ref_rejected": -288.8368225097656,
      "logps/rejected": -284.85382080078125,
      "loss": 5.4027,
      "margin_dpo/margin_mean": 3.743913412094116,
      "margin_dpo/margin_std": 7.464703559875488,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": 7.861959457397461,
      "KL/mean": 5.8625407218933105,
      "KL/rejected_KL_mean": 3.8631134033203125,
      "KL/std": 5.401138782501221,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.9988510608673096,
      "fcm_dpo/q_t": 0.4900166690349579,
      "grad_norm": 15.381290435791016,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": 1.8424619436264038,
      "logits/rejected": 1.840616226196289,
      "logps/chosen": -257.80401611328125,
      "logps/ref_chosen": -265.6659851074219,
      "logps/ref_rejected": -242.0697479248047,
      "logps/rejected": -238.20663452148438,
      "loss": 5.3911,
      "margin_dpo/margin_mean": 3.9988512992858887,
      "margin_dpo/margin_std": 6.359364986419678,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": 7.316184997558594,
      "KL/mean": 5.700174808502197,
      "KL/rejected_KL_mean": 4.084161758422852,
      "KL/std": 5.528467655181885,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.2320237159729004,
      "fcm_dpo/q_t": 0.49193090200424194,
      "grad_norm": 14.052903175354004,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 1.7988606691360474,
      "logits/rejected": 1.816406488418579,
      "logps/chosen": -253.40525817871094,
      "logps/ref_chosen": -260.721435546875,
      "logps/ref_rejected": -262.6593322753906,
      "logps/rejected": -258.57513427734375,
      "loss": 5.4224,
      "margin_dpo/margin_mean": 3.2320241928100586,
      "margin_dpo/margin_std": 7.131723880767822,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": 8.91480827331543,
      "KL/mean": 6.977107048034668,
      "KL/rejected_KL_mean": 5.039403915405273,
      "KL/std": 5.199066638946533,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.8754138946533203,
      "fcm_dpo/q_t": 0.4903239905834198,
      "grad_norm": 15.85583782196045,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": 1.744780421257019,
      "logits/rejected": 1.7024192810058594,
      "logps/chosen": -248.08799743652344,
      "logps/ref_chosen": -257.0028076171875,
      "logps/ref_rejected": -254.66232299804688,
      "logps/rejected": -249.6229248046875,
      "loss": 5.396,
      "margin_dpo/margin_mean": 3.875415563583374,
      "margin_dpo/margin_std": 6.206583499908447,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": 8.962141036987305,
      "KL/mean": 7.253349781036377,
      "KL/rejected_KL_mean": 5.544553756713867,
      "KL/std": 5.543599605560303,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.4175894260406494,
      "fcm_dpo/q_t": 0.4914683699607849,
      "grad_norm": 15.222030639648438,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 1.9043039083480835,
      "logits/rejected": 1.8396852016448975,
      "logps/chosen": -289.4482421875,
      "logps/ref_chosen": -298.4103698730469,
      "logps/ref_rejected": -262.8761901855469,
      "logps/rejected": -257.3316345214844,
      "loss": 5.4149,
      "margin_dpo/margin_mean": 3.417588710784912,
      "margin_dpo/margin_std": 6.722899436950684,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": 8.673236846923828,
      "KL/mean": 6.997888565063477,
      "KL/rejected_KL_mean": 5.322538375854492,
      "KL/std": 5.426290035247803,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.350696086883545,
      "fcm_dpo/q_t": 0.49164003133773804,
      "grad_norm": 14.669853210449219,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 1.5951306819915771,
      "logits/rejected": 1.6337463855743408,
      "logps/chosen": -218.85064697265625,
      "logps/ref_chosen": -227.5238800048828,
      "logps/ref_rejected": -227.11428833007812,
      "logps/rejected": -221.791748046875,
      "loss": 5.418,
      "margin_dpo/margin_mean": 3.350695848464966,
      "margin_dpo/margin_std": 7.318633556365967,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": 8.928094863891602,
      "KL/mean": 7.114223957061768,
      "KL/rejected_KL_mean": 5.300344467163086,
      "KL/std": 6.519954681396484,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6277565956115723,
      "fcm_dpo/q_t": 0.49094149470329285,
      "grad_norm": 15.608501434326172,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": 1.8032417297363281,
      "logits/rejected": 1.7031846046447754,
      "logps/chosen": -266.4541320800781,
      "logps/ref_chosen": -275.3822326660156,
      "logps/ref_rejected": -269.8356018066406,
      "logps/rejected": -264.5352478027344,
      "loss": 5.4076,
      "margin_dpo/margin_mean": 3.6277568340301514,
      "margin_dpo/margin_std": 7.5850677490234375,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": 9.135509490966797,
      "KL/mean": 7.4146318435668945,
      "KL/rejected_KL_mean": 5.693750381469727,
      "KL/std": 5.743583679199219,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.441770076751709,
      "fcm_dpo/q_t": 0.49141159653663635,
      "grad_norm": 14.305583953857422,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": 1.713180661201477,
      "logits/rejected": 1.6899499893188477,
      "logps/chosen": -218.9808807373047,
      "logps/ref_chosen": -228.1163787841797,
      "logps/ref_rejected": -228.26266479492188,
      "logps/rejected": -222.5688934326172,
      "loss": 5.4143,
      "margin_dpo/margin_mean": 3.44176983833313,
      "margin_dpo/margin_std": 7.184349060058594,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": 10.082416534423828,
      "KL/mean": 8.04116153717041,
      "KL/rejected_KL_mean": 5.999895095825195,
      "KL/std": 6.637115478515625,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.082524299621582,
      "fcm_dpo/q_t": 0.48981940746307373,
      "grad_norm": 15.618288040161133,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": 1.6476315259933472,
      "logits/rejected": 1.7307677268981934,
      "logps/chosen": -273.94482421875,
      "logps/ref_chosen": -284.0272521972656,
      "logps/ref_rejected": -282.1720275878906,
      "logps/rejected": -276.172119140625,
      "loss": 5.3919,
      "margin_dpo/margin_mean": 4.082524299621582,
      "margin_dpo/margin_std": 8.705501556396484,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": 9.81295394897461,
      "KL/mean": 7.582304000854492,
      "KL/rejected_KL_mean": 5.351655960083008,
      "KL/std": 6.855978012084961,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.461294651031494,
      "fcm_dpo/q_t": 0.48887497186660767,
      "grad_norm": 18.12920379638672,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": 1.9543178081512451,
      "logits/rejected": 1.896885633468628,
      "logps/chosen": -293.23687744140625,
      "logps/ref_chosen": -303.0498352050781,
      "logps/ref_rejected": -259.8980712890625,
      "logps/rejected": -254.5464324951172,
      "loss": 5.3769,
      "margin_dpo/margin_mean": 4.461293697357178,
      "margin_dpo/margin_std": 8.783276557922363,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": 9.891937255859375,
      "KL/mean": 7.75474739074707,
      "KL/rejected_KL_mean": 5.6175537109375,
      "KL/std": 6.965407848358154,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.274378776550293,
      "fcm_dpo/q_t": 0.48933839797973633,
      "grad_norm": 15.175575256347656,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": 1.933483362197876,
      "logits/rejected": 1.8999509811401367,
      "logps/chosen": -260.69573974609375,
      "logps/ref_chosen": -270.58770751953125,
      "logps/ref_rejected": -238.90069580078125,
      "logps/rejected": -233.2831268310547,
      "loss": 5.3829,
      "margin_dpo/margin_mean": 4.274379730224609,
      "margin_dpo/margin_std": 7.79229211807251,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": 9.73392105102539,
      "KL/mean": 7.504912376403809,
      "KL/rejected_KL_mean": 5.275903701782227,
      "KL/std": 6.893660068511963,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.458014488220215,
      "fcm_dpo/q_t": 0.4888973832130432,
      "grad_norm": 15.308623313903809,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": 1.8387027978897095,
      "logits/rejected": 1.7226899862289429,
      "logps/chosen": -286.4516296386719,
      "logps/ref_chosen": -296.1855773925781,
      "logps/ref_rejected": -260.35333251953125,
      "logps/rejected": -255.07742309570312,
      "loss": 5.3787,
      "margin_dpo/margin_mean": 4.458015441894531,
      "margin_dpo/margin_std": 9.660484313964844,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": 10.027368545532227,
      "KL/mean": 7.647181510925293,
      "KL/rejected_KL_mean": 5.266988754272461,
      "KL/std": 7.323658466339111,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.76038122177124,
      "fcm_dpo/q_t": 0.4881379008293152,
      "grad_norm": 15.692994117736816,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": 1.9335343837738037,
      "logits/rejected": 1.9183515310287476,
      "logps/chosen": -267.68798828125,
      "logps/ref_chosen": -277.7153625488281,
      "logps/ref_rejected": -272.9094543457031,
      "logps/rejected": -267.6424865722656,
      "loss": 5.3676,
      "margin_dpo/margin_mean": 4.76038122177124,
      "margin_dpo/margin_std": 9.4072265625,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": 10.275821685791016,
      "KL/mean": 7.720584392547607,
      "KL/rejected_KL_mean": 5.165353775024414,
      "KL/std": 7.543099880218506,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.1104736328125,
      "fcm_dpo/q_t": 0.4872530996799469,
      "grad_norm": 15.053325653076172,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": 1.7000428438186646,
      "logits/rejected": 1.6693438291549683,
      "logps/chosen": -262.261962890625,
      "logps/ref_chosen": -272.5378112792969,
      "logps/ref_rejected": -244.6329345703125,
      "logps/rejected": -239.46759033203125,
      "loss": 5.3546,
      "margin_dpo/margin_mean": 5.110472679138184,
      "margin_dpo/margin_std": 9.97677993774414,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": 9.296737670898438,
      "KL/mean": 7.074584007263184,
      "KL/rejected_KL_mean": 4.852437973022461,
      "KL/std": 7.750631809234619,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.444304943084717,
      "fcm_dpo/q_t": 0.4889252185821533,
      "grad_norm": 15.504831314086914,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.6485273838043213,
      "logits/rejected": 1.5641793012619019,
      "logps/chosen": -280.515625,
      "logps/ref_chosen": -289.8123474121094,
      "logps/ref_rejected": -274.29071044921875,
      "logps/rejected": -269.4382629394531,
      "loss": 5.3787,
      "margin_dpo/margin_mean": 4.444304943084717,
      "margin_dpo/margin_std": 9.017261505126953,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": 10.043233871459961,
      "KL/mean": 7.734489440917969,
      "KL/rejected_KL_mean": 5.425750732421875,
      "KL/std": 8.07658863067627,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.617471694946289,
      "fcm_dpo/q_t": 0.4884904623031616,
      "grad_norm": 15.460589408874512,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 1.8521496057510376,
      "logits/rejected": 2.0272319316864014,
      "logps/chosen": -235.66989135742188,
      "logps/ref_chosen": -245.71311950683594,
      "logps/ref_rejected": -272.08935546875,
      "logps/rejected": -266.6636047363281,
      "loss": 5.3734,
      "margin_dpo/margin_mean": 4.617470741271973,
      "margin_dpo/margin_std": 9.746196746826172,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": 10.364633560180664,
      "KL/mean": 7.178472518920898,
      "KL/rejected_KL_mean": 3.9923267364501953,
      "KL/std": 7.504856109619141,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.372310161590576,
      "fcm_dpo/q_t": 0.48411256074905396,
      "grad_norm": 15.164575576782227,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": 1.373188853263855,
      "logits/rejected": 1.456892967224121,
      "logps/chosen": -199.57777404785156,
      "logps/ref_chosen": -209.94239807128906,
      "logps/ref_rejected": -230.66897583007812,
      "logps/rejected": -226.6766357421875,
      "loss": 5.3034,
      "margin_dpo/margin_mean": 6.372309684753418,
      "margin_dpo/margin_std": 8.945518493652344,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": 9.337991714477539,
      "KL/mean": 6.94564151763916,
      "KL/rejected_KL_mean": 4.553287506103516,
      "KL/std": 8.22175407409668,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.784703731536865,
      "fcm_dpo/q_t": 0.48808154463768005,
      "grad_norm": 16.39162826538086,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": 1.7419049739837646,
      "logits/rejected": 1.8380424976348877,
      "logps/chosen": -241.9616241455078,
      "logps/ref_chosen": -251.2996368408203,
      "logps/ref_rejected": -277.4847412109375,
      "logps/rejected": -272.93145751953125,
      "loss": 5.3669,
      "margin_dpo/margin_mean": 4.784704685211182,
      "margin_dpo/margin_std": 10.069666862487793,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": 8.432588577270508,
      "KL/mean": 6.196615695953369,
      "KL/rejected_KL_mean": 3.9606380462646484,
      "KL/std": 8.564009666442871,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.471951484680176,
      "fcm_dpo/q_t": 0.488866925239563,
      "grad_norm": 14.32288932800293,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": 1.8083268404006958,
      "logits/rejected": 1.8534597158432007,
      "logps/chosen": -249.34683227539062,
      "logps/ref_chosen": -257.7794189453125,
      "logps/ref_rejected": -227.91546630859375,
      "logps/rejected": -223.95480346679688,
      "loss": 5.3814,
      "margin_dpo/margin_mean": 4.471951484680176,
      "margin_dpo/margin_std": 11.081856727600098,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": 10.272048950195312,
      "KL/mean": 6.237697601318359,
      "KL/rejected_KL_mean": 2.2033615112304688,
      "KL/std": 9.601115226745605,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.068692207336426,
      "fcm_dpo/q_t": 0.47989726066589355,
      "grad_norm": 18.2125301361084,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": 1.8094466924667358,
      "logits/rejected": 1.7722769975662231,
      "logps/chosen": -301.82196044921875,
      "logps/ref_chosen": -312.09405517578125,
      "logps/ref_rejected": -277.97332763671875,
      "logps/rejected": -275.7699279785156,
      "loss": 5.2414,
      "margin_dpo/margin_mean": 8.068690299987793,
      "margin_dpo/margin_std": 10.587115287780762,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": 6.717323303222656,
      "KL/mean": 3.9726412296295166,
      "KL/rejected_KL_mean": 1.2279529571533203,
      "KL/std": 8.904565811157227,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.489373683929443,
      "fcm_dpo/q_t": 0.48633456230163574,
      "grad_norm": 16.498191833496094,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": 1.8615291118621826,
      "logits/rejected": 1.8797656297683716,
      "logps/chosen": -278.6490173339844,
      "logps/ref_chosen": -285.3663330078125,
      "logps/ref_rejected": -262.2779541015625,
      "logps/rejected": -261.04998779296875,
      "loss": 5.3421,
      "margin_dpo/margin_mean": 5.489375114440918,
      "margin_dpo/margin_std": 11.117755889892578,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": 9.393083572387695,
      "KL/mean": 5.730623722076416,
      "KL/rejected_KL_mean": 2.068164825439453,
      "KL/std": 9.502227783203125,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.324916362762451,
      "fcm_dpo/q_t": 0.48174092173576355,
      "grad_norm": 16.526874542236328,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": 1.7952461242675781,
      "logits/rejected": 1.8625985383987427,
      "logps/chosen": -286.9005126953125,
      "logps/ref_chosen": -296.2935791015625,
      "logps/ref_rejected": -291.5162353515625,
      "logps/rejected": -289.44805908203125,
      "loss": 5.2726,
      "margin_dpo/margin_mean": 7.324915409088135,
      "margin_dpo/margin_std": 11.52578067779541,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": 9.495731353759766,
      "KL/mean": 5.4516425132751465,
      "KL/rejected_KL_mean": 1.4075469970703125,
      "KL/std": 8.898553848266602,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.088191986083984,
      "fcm_dpo/q_t": 0.47985130548477173,
      "grad_norm": 16.13837432861328,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": 1.9154037237167358,
      "logits/rejected": 1.786590576171875,
      "logps/chosen": -278.62860107421875,
      "logps/ref_chosen": -288.12432861328125,
      "logps/ref_rejected": -232.35720825195312,
      "logps/rejected": -230.9496612548828,
      "loss": 5.2396,
      "margin_dpo/margin_mean": 8.088191032409668,
      "margin_dpo/margin_std": 10.321402549743652,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": 8.040447235107422,
      "KL/mean": 3.9166314601898193,
      "KL/rejected_KL_mean": -0.20717620849609375,
      "KL/std": 10.056496620178223,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.247621536254883,
      "fcm_dpo/q_t": 0.4795139729976654,
      "grad_norm": 17.64774513244629,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": 1.6587417125701904,
      "logits/rejected": 1.6023626327514648,
      "logps/chosen": -309.1611328125,
      "logps/ref_chosen": -317.2015686035156,
      "logps/ref_rejected": -278.0653381347656,
      "logps/rejected": -278.2724914550781,
      "loss": 5.2401,
      "margin_dpo/margin_mean": 8.247618675231934,
      "margin_dpo/margin_std": 12.782524108886719,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": 8.622020721435547,
      "KL/mean": 4.673813819885254,
      "KL/rejected_KL_mean": 0.7256183624267578,
      "KL/std": 10.075484275817871,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.896410942077637,
      "fcm_dpo/q_t": 0.4803505539894104,
      "grad_norm": 15.88173770904541,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.8079415559768677,
      "logits/rejected": 1.8058743476867676,
      "logps/chosen": -254.3174285888672,
      "logps/ref_chosen": -262.939453125,
      "logps/ref_rejected": -257.82275390625,
      "logps/rejected": -257.0971374511719,
      "loss": 5.2535,
      "margin_dpo/margin_mean": 7.8964104652404785,
      "margin_dpo/margin_std": 13.03802490234375,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": 9.08038330078125,
      "KL/mean": 4.754354000091553,
      "KL/rejected_KL_mean": 0.42832183837890625,
      "KL/std": 11.183330535888672,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.652055740356445,
      "fcm_dpo/q_t": 0.4785040318965912,
      "grad_norm": 18.077545166015625,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 1.879668116569519,
      "logits/rejected": 1.784064531326294,
      "logps/chosen": -310.07342529296875,
      "logps/ref_chosen": -319.15380859375,
      "logps/ref_rejected": -281.7733154296875,
      "logps/rejected": -281.3450012207031,
      "loss": 5.2251,
      "margin_dpo/margin_mean": 8.652054786682129,
      "margin_dpo/margin_std": 12.81071949005127,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": 5.550897598266602,
      "KL/mean": 3.350287675857544,
      "KL/rejected_KL_mean": 1.149688720703125,
      "KL/std": 10.498481750488281,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.401219367980957,
      "fcm_dpo/q_t": 0.48904141783714294,
      "grad_norm": 14.67580509185791,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": 1.9130809307098389,
      "logits/rejected": 2.0241482257843018,
      "logps/chosen": -244.9495391845703,
      "logps/ref_chosen": -250.50042724609375,
      "logps/ref_rejected": -268.32635498046875,
      "logps/rejected": -267.1766662597656,
      "loss": 5.3941,
      "margin_dpo/margin_mean": 4.401219367980957,
      "margin_dpo/margin_std": 14.07012939453125,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": 6.7470703125,
      "KL/mean": 1.6851611137390137,
      "KL/rejected_KL_mean": -3.3767528533935547,
      "KL/std": 11.53531551361084,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.12381362915039,
      "fcm_dpo/q_t": 0.4748704731464386,
      "grad_norm": 16.903310775756836,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": 1.7780122756958008,
      "logits/rejected": 1.8382484912872314,
      "logps/chosen": -272.6545104980469,
      "logps/ref_chosen": -279.4015808105469,
      "logps/ref_rejected": -266.6392822265625,
      "logps/rejected": -270.0160827636719,
      "loss": 5.1722,
      "margin_dpo/margin_mean": 10.12381362915039,
      "margin_dpo/margin_std": 14.13444709777832,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": 5.2010345458984375,
      "KL/mean": 2.62278413772583,
      "KL/rejected_KL_mean": 0.044536590576171875,
      "KL/std": 10.696566581726074,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.156493663787842,
      "fcm_dpo/q_t": 0.487183153629303,
      "grad_norm": 15.92874526977539,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 1.5905275344848633,
      "logits/rejected": 1.5697118043899536,
      "logps/chosen": -249.05946350097656,
      "logps/ref_chosen": -254.26048278808594,
      "logps/ref_rejected": -241.8769989013672,
      "logps/rejected": -241.83245849609375,
      "loss": 5.3606,
      "margin_dpo/margin_mean": 5.156494617462158,
      "margin_dpo/margin_std": 13.0711088180542,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": 5.559000015258789,
      "KL/mean": 2.684415102005005,
      "KL/rejected_KL_mean": -0.19017791748046875,
      "KL/std": 11.807351112365723,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.749182224273682,
      "fcm_dpo/q_t": 0.4857518970966339,
      "grad_norm": 15.834238052368164,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": 1.6776496171951294,
      "logits/rejected": 1.709686517715454,
      "logps/chosen": -283.2455139160156,
      "logps/ref_chosen": -288.80450439453125,
      "logps/ref_rejected": -266.8573913574219,
      "logps/rejected": -267.0475769042969,
      "loss": 5.3416,
      "margin_dpo/margin_mean": 5.749182224273682,
      "margin_dpo/margin_std": 14.371601104736328,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": 6.950733184814453,
      "KL/mean": 2.922211170196533,
      "KL/rejected_KL_mean": -1.1063232421875,
      "KL/std": 11.13856315612793,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.05705451965332,
      "fcm_dpo/q_t": 0.47997626662254333,
      "grad_norm": 16.346704483032227,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": 1.6449549198150635,
      "logits/rejected": 1.6187565326690674,
      "logps/chosen": -265.2392272949219,
      "logps/ref_chosen": -272.18994140625,
      "logps/ref_rejected": -253.14334106445312,
      "logps/rejected": -254.24964904785156,
      "loss": 5.2506,
      "margin_dpo/margin_mean": 8.057055473327637,
      "margin_dpo/margin_std": 14.087621688842773,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": 5.696207046508789,
      "KL/mean": 1.4905540943145752,
      "KL/rejected_KL_mean": -2.715097427368164,
      "KL/std": 11.035455703735352,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.411304473876953,
      "fcm_dpo/q_t": 0.47909414768218994,
      "grad_norm": 15.938675880432129,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": 1.4418600797653198,
      "logits/rejected": 1.409030795097351,
      "logps/chosen": -244.70643615722656,
      "logps/ref_chosen": -250.4026336669922,
      "logps/ref_rejected": -229.63853454589844,
      "logps/rejected": -232.35362243652344,
      "loss": 5.2348,
      "margin_dpo/margin_mean": 8.411303520202637,
      "margin_dpo/margin_std": 12.481833457946777,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": 5.417490005493164,
      "KL/mean": 0.9396879076957703,
      "KL/rejected_KL_mean": -3.5381240844726562,
      "KL/std": 12.709689140319824,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.955621719360352,
      "fcm_dpo/q_t": 0.47775962948799133,
      "grad_norm": 20.289796829223633,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": 1.800328016281128,
      "logits/rejected": 1.8531248569488525,
      "logps/chosen": -252.9783935546875,
      "logps/ref_chosen": -258.3959045410156,
      "logps/ref_rejected": -268.84228515625,
      "logps/rejected": -272.38043212890625,
      "loss": 5.2178,
      "margin_dpo/margin_mean": 8.955621719360352,
      "margin_dpo/margin_std": 14.531270027160645,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": 6.025569915771484,
      "KL/mean": 2.2033252716064453,
      "KL/rejected_KL_mean": -1.6189250946044922,
      "KL/std": 14.194839477539062,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.6444902420043945,
      "fcm_dpo/q_t": 0.4811151623725891,
      "grad_norm": 15.814275741577148,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": 1.7583012580871582,
      "logits/rejected": 1.7272329330444336,
      "logps/chosen": -311.2770080566406,
      "logps/ref_chosen": -317.30255126953125,
      "logps/ref_rejected": -300.47552490234375,
      "logps/rejected": -302.09442138671875,
      "loss": 5.2787,
      "margin_dpo/margin_mean": 7.644491195678711,
      "margin_dpo/margin_std": 16.89623260498047,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": 7.087528228759766,
      "KL/mean": 2.574556589126587,
      "KL/rejected_KL_mean": -1.9384117126464844,
      "KL/std": 13.3712739944458,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.025940895080566,
      "fcm_dpo/q_t": 0.4776258170604706,
      "grad_norm": 16.527212142944336,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.7176724672317505,
      "logits/rejected": 1.6553864479064941,
      "logps/chosen": -298.1669006347656,
      "logps/ref_chosen": -305.2544250488281,
      "logps/ref_rejected": -259.40716552734375,
      "logps/rejected": -261.3455505371094,
      "loss": 5.221,
      "margin_dpo/margin_mean": 9.02593994140625,
      "margin_dpo/margin_std": 16.198793411254883,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": 3.514894485473633,
      "KL/mean": 0.8094793558120728,
      "KL/rejected_KL_mean": -1.8959331512451172,
      "KL/std": 11.925463676452637,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.410823822021484,
      "fcm_dpo/q_t": 0.48657190799713135,
      "grad_norm": 15.410111427307129,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 1.6124632358551025,
      "logits/rejected": 1.632853388786316,
      "logps/chosen": -213.12583923339844,
      "logps/ref_chosen": -216.64073181152344,
      "logps/ref_rejected": -224.36422729492188,
      "logps/rejected": -226.26014709472656,
      "loss": 5.355,
      "margin_dpo/margin_mean": 5.410823822021484,
      "margin_dpo/margin_std": 14.422788619995117,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": 5.785375595092773,
      "KL/mean": 2.0622177124023438,
      "KL/rejected_KL_mean": -1.6609439849853516,
      "KL/std": 14.050432205200195,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.446321964263916,
      "fcm_dpo/q_t": 0.48156237602233887,
      "grad_norm": 15.768331527709961,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": 1.7849597930908203,
      "logits/rejected": 1.7804847955703735,
      "logps/chosen": -263.0334167480469,
      "logps/ref_chosen": -268.8188171386719,
      "logps/ref_rejected": -272.55853271484375,
      "logps/rejected": -274.219482421875,
      "loss": 5.2864,
      "margin_dpo/margin_mean": 7.446322441101074,
      "margin_dpo/margin_std": 17.560386657714844,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": 8.549701690673828,
      "KL/mean": 3.2901999950408936,
      "KL/rejected_KL_mean": -1.9692878723144531,
      "KL/std": 14.909716606140137,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.518993377685547,
      "fcm_dpo/q_t": 0.4739653170108795,
      "grad_norm": 18.204126358032227,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": 1.7290822267532349,
      "logits/rejected": 1.7112373113632202,
      "logps/chosen": -313.14208984375,
      "logps/ref_chosen": -321.6917419433594,
      "logps/ref_rejected": -279.1517333984375,
      "logps/rejected": -281.1209716796875,
      "loss": 5.1696,
      "margin_dpo/margin_mean": 10.51899242401123,
      "margin_dpo/margin_std": 18.130516052246094,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": 6.242259979248047,
      "KL/mean": 1.4488016366958618,
      "KL/rejected_KL_mean": -3.3446693420410156,
      "KL/std": 14.445108413696289,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.586933135986328,
      "fcm_dpo/q_t": 0.4762246906757355,
      "grad_norm": 16.7393798828125,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": 1.5432134866714478,
      "logits/rejected": 1.4192038774490356,
      "logps/chosen": -265.4329528808594,
      "logps/ref_chosen": -271.6752014160156,
      "logps/ref_rejected": -221.85577392578125,
      "logps/rejected": -225.20046997070312,
      "loss": 5.2013,
      "margin_dpo/margin_mean": 9.586933135986328,
      "margin_dpo/margin_std": 16.124324798583984,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": 5.612039566040039,
      "KL/mean": 0.9343008399009705,
      "KL/rejected_KL_mean": -3.743438720703125,
      "KL/std": 14.065815925598145,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.355475425720215,
      "fcm_dpo/q_t": 0.47688087821006775,
      "grad_norm": 16.27495765686035,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": 1.623723030090332,
      "logits/rejected": 1.6291615962982178,
      "logps/chosen": -264.7870178222656,
      "logps/ref_chosen": -270.3990478515625,
      "logps/ref_rejected": -264.16925048828125,
      "logps/rejected": -267.9127197265625,
      "loss": 5.2154,
      "margin_dpo/margin_mean": 9.355476379394531,
      "margin_dpo/margin_std": 18.374380111694336,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": 6.888246536254883,
      "KL/mean": 1.5430099964141846,
      "KL/rejected_KL_mean": -3.8022212982177734,
      "KL/std": 15.5655517578125,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.690484046936035,
      "fcm_dpo/q_t": 0.4735196530818939,
      "grad_norm": 16.703195571899414,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": 1.6616361141204834,
      "logits/rejected": 1.6271593570709229,
      "logps/chosen": -306.3630676269531,
      "logps/ref_chosen": -313.25128173828125,
      "logps/ref_rejected": -283.0433349609375,
      "logps/rejected": -286.8455810546875,
      "loss": 5.1654,
      "margin_dpo/margin_mean": 10.690484046936035,
      "margin_dpo/margin_std": 18.25865364074707,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": 8.188390731811523,
      "KL/mean": 2.025113344192505,
      "KL/rejected_KL_mean": -4.138158798217773,
      "KL/std": 16.11878204345703,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.326554298400879,
      "fcm_dpo/q_t": 0.4695105254650116,
      "grad_norm": 17.106164932250977,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 1.652585744857788,
      "logits/rejected": 1.5759227275848389,
      "logps/chosen": -290.3756103515625,
      "logps/ref_chosen": -298.5639953613281,
      "logps/ref_rejected": -274.021728515625,
      "logps/rejected": -278.159912109375,
      "loss": 5.1044,
      "margin_dpo/margin_mean": 12.326556205749512,
      "margin_dpo/margin_std": 18.264963150024414,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": 8.683004379272461,
      "KL/mean": 1.8398021459579468,
      "KL/rejected_KL_mean": -5.003396987915039,
      "KL/std": 16.92917251586914,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.686408996582031,
      "fcm_dpo/q_t": 0.4662187695503235,
      "grad_norm": 16.039901733398438,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": 1.7340830564498901,
      "logits/rejected": 1.7791649103164673,
      "logps/chosen": -248.4478759765625,
      "logps/ref_chosen": -257.130859375,
      "logps/ref_rejected": -247.71636962890625,
      "logps/rejected": -252.71975708007812,
      "loss": 5.0594,
      "margin_dpo/margin_mean": 13.686408042907715,
      "margin_dpo/margin_std": 20.029903411865234,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": 9.040241241455078,
      "KL/mean": 2.74595308303833,
      "KL/rejected_KL_mean": -3.5483360290527344,
      "KL/std": 17.033199310302734,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.588574409484863,
      "fcm_dpo/q_t": 0.4689618647098541,
      "grad_norm": 16.696441650390625,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": 1.5907576084136963,
      "logits/rejected": 1.6255662441253662,
      "logps/chosen": -285.9203186035156,
      "logps/ref_chosen": -294.9605712890625,
      "logps/ref_rejected": -272.6249694824219,
      "logps/rejected": -276.1733093261719,
      "loss": 5.1032,
      "margin_dpo/margin_mean": 12.588573455810547,
      "margin_dpo/margin_std": 20.232376098632812,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": 7.547658920288086,
      "KL/mean": 1.70832359790802,
      "KL/rejected_KL_mean": -4.131021499633789,
      "KL/std": 15.507174491882324,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.678691864013672,
      "fcm_dpo/q_t": 0.4710816740989685,
      "grad_norm": 20.7784366607666,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 1.4702523946762085,
      "logits/rejected": 1.5250898599624634,
      "logps/chosen": -268.8945617675781,
      "logps/ref_chosen": -276.4422302246094,
      "logps/ref_rejected": -290.241455078125,
      "logps/rejected": -294.3724670410156,
      "loss": 5.13,
      "margin_dpo/margin_mean": 11.678691864013672,
      "margin_dpo/margin_std": 19.250728607177734,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": 4.794488906860352,
      "KL/mean": -0.03219972550868988,
      "KL/rejected_KL_mean": -4.858894348144531,
      "KL/std": 16.509620666503906,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.65339183807373,
      "fcm_dpo/q_t": 0.4762312173843384,
      "grad_norm": 17.51825714111328,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 1.5192365646362305,
      "logits/rejected": 1.5191075801849365,
      "logps/chosen": -300.03021240234375,
      "logps/ref_chosen": -304.82470703125,
      "logps/ref_rejected": -289.9726867675781,
      "logps/rejected": -294.83160400390625,
      "loss": 5.2169,
      "margin_dpo/margin_mean": 9.653390884399414,
      "margin_dpo/margin_std": 21.26504135131836,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": 7.096508026123047,
      "KL/mean": 0.591003954410553,
      "KL/rejected_KL_mean": -5.914497375488281,
      "KL/std": 18.584745407104492,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.011009216308594,
      "fcm_dpo/q_t": 0.46816810965538025,
      "grad_norm": 18.00429916381836,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": 1.5481804609298706,
      "logits/rejected": 1.6589641571044922,
      "logps/chosen": -253.21823120117188,
      "logps/ref_chosen": -260.31475830078125,
      "logps/ref_rejected": -280.93756103515625,
      "logps/rejected": -286.85205078125,
      "loss": 5.1025,
      "margin_dpo/margin_mean": 13.011011123657227,
      "margin_dpo/margin_std": 23.44249153137207,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": 7.448966979980469,
      "KL/mean": 0.6639432907104492,
      "KL/rejected_KL_mean": -6.121076583862305,
      "KL/std": 17.271480560302734,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.570048332214355,
      "fcm_dpo/q_t": 0.4664422571659088,
      "grad_norm": 16.16029167175293,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": 1.8354668617248535,
      "logits/rejected": 1.8340388536453247,
      "logps/chosen": -271.671630859375,
      "logps/ref_chosen": -279.12060546875,
      "logps/ref_rejected": -247.22982788085938,
      "logps/rejected": -253.35092163085938,
      "loss": 5.0703,
      "margin_dpo/margin_mean": 13.570047378540039,
      "margin_dpo/margin_std": 21.082441329956055,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": 1.4288806915283203,
      "KL/mean": -3.2474002838134766,
      "KL/rejected_KL_mean": -7.923681259155273,
      "KL/std": 17.611831665039062,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.352568626403809,
      "fcm_dpo/q_t": 0.4769529104232788,
      "grad_norm": 20.622295379638672,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": 1.622707724571228,
      "logits/rejected": 1.6467121839523315,
      "logps/chosen": -314.1752014160156,
      "logps/ref_chosen": -315.6040954589844,
      "logps/ref_rejected": -291.45635986328125,
      "logps/rejected": -299.3800354003906,
      "loss": 5.2244,
      "margin_dpo/margin_mean": 9.352567672729492,
      "margin_dpo/margin_std": 20.2061767578125,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": 5.105659484863281,
      "KL/mean": -1.5070035457611084,
      "KL/rejected_KL_mean": -8.119661331176758,
      "KL/std": 18.098840713500977,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.225325584411621,
      "fcm_dpo/q_t": 0.4673457741737366,
      "grad_norm": 17.272735595703125,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": 1.5253934860229492,
      "logits/rejected": 1.518908143043518,
      "logps/chosen": -277.767333984375,
      "logps/ref_chosen": -282.87298583984375,
      "logps/ref_rejected": -265.0682373046875,
      "logps/rejected": -273.1878662109375,
      "loss": 5.0766,
      "margin_dpo/margin_mean": 13.225324630737305,
      "margin_dpo/margin_std": 19.304866790771484,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": 4.408140182495117,
      "KL/mean": -1.86943781375885,
      "KL/rejected_KL_mean": -8.147012710571289,
      "KL/std": 17.620100021362305,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.555158615112305,
      "fcm_dpo/q_t": 0.4690578877925873,
      "grad_norm": 20.035966873168945,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": 1.2938097715377808,
      "logits/rejected": 1.449841022491455,
      "logps/chosen": -259.03375244140625,
      "logps/ref_chosen": -263.4419250488281,
      "logps/ref_rejected": -251.43150329589844,
      "logps/rejected": -259.5784912109375,
      "loss": 5.1108,
      "margin_dpo/margin_mean": 12.555157661437988,
      "margin_dpo/margin_std": 21.81502914428711,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": 2.4925613403320312,
      "KL/mean": -3.1512343883514404,
      "KL/rejected_KL_mean": -8.795032501220703,
      "KL/std": 19.188785552978516,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.287603378295898,
      "fcm_dpo/q_t": 0.4721390902996063,
      "grad_norm": 18.578805923461914,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": 1.5265250205993652,
      "logits/rejected": 1.5208193063735962,
      "logps/chosen": -292.7781982421875,
      "logps/ref_chosen": -295.2707214355469,
      "logps/ref_rejected": -283.9851989746094,
      "logps/rejected": -292.7802734375,
      "loss": 5.154,
      "margin_dpo/margin_mean": 11.287605285644531,
      "margin_dpo/margin_std": 20.726428985595703,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": 1.1052379608154297,
      "KL/mean": -4.508172988891602,
      "KL/rejected_KL_mean": -10.121585845947266,
      "KL/std": 18.477428436279297,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.226819038391113,
      "fcm_dpo/q_t": 0.4724728465080261,
      "grad_norm": 16.500743865966797,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": 1.4822649955749512,
      "logits/rejected": 1.4741019010543823,
      "logps/chosen": -281.45263671875,
      "logps/ref_chosen": -282.5578918457031,
      "logps/ref_rejected": -262.4772644042969,
      "logps/rejected": -272.5988464355469,
      "loss": 5.1685,
      "margin_dpo/margin_mean": 11.226819038391113,
      "margin_dpo/margin_std": 23.494342803955078,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": 1.9564666748046875,
      "KL/mean": -2.091911554336548,
      "KL/rejected_KL_mean": -6.140293121337891,
      "KL/std": 19.2132568359375,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.096758842468262,
      "fcm_dpo/q_t": 0.48008954524993896,
      "grad_norm": 16.436861038208008,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": 1.6937000751495361,
      "logits/rejected": 1.770328164100647,
      "logps/chosen": -261.6943359375,
      "logps/ref_chosen": -263.65081787109375,
      "logps/ref_rejected": -266.2420959472656,
      "logps/rejected": -272.38238525390625,
      "loss": 5.2863,
      "margin_dpo/margin_mean": 8.096759796142578,
      "margin_dpo/margin_std": 23.786102294921875,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": 5.584157943725586,
      "KL/mean": -1.878004550933838,
      "KL/rejected_KL_mean": -9.340160369873047,
      "KL/std": 19.12067985534668,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.924320220947266,
      "fcm_dpo/q_t": 0.4632810354232788,
      "grad_norm": 16.136503219604492,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 1.6616230010986328,
      "logits/rejected": 1.6243776082992554,
      "logps/chosen": -266.79827880859375,
      "logps/ref_chosen": -272.3824462890625,
      "logps/ref_rejected": -248.96444702148438,
      "logps/rejected": -258.30462646484375,
      "loss": 5.0212,
      "margin_dpo/margin_mean": 14.924320220947266,
      "margin_dpo/margin_std": 21.41767692565918,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": 3.1255455017089844,
      "KL/mean": -4.009986400604248,
      "KL/rejected_KL_mean": -11.145502090454102,
      "KL/std": 20.294437408447266,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.27104377746582,
      "fcm_dpo/q_t": 0.46490606665611267,
      "grad_norm": 22.256866455078125,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 1.6400327682495117,
      "logits/rejected": 1.6940089464187622,
      "logps/chosen": -284.94549560546875,
      "logps/ref_chosen": -288.071044921875,
      "logps/ref_rejected": -274.83477783203125,
      "logps/rejected": -285.980224609375,
      "loss": 5.0585,
      "margin_dpo/margin_mean": 14.27104377746582,
      "margin_dpo/margin_std": 24.495113372802734,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": 6.04503059387207,
      "KL/mean": -1.0072931051254272,
      "KL/rejected_KL_mean": -8.059608459472656,
      "KL/std": 20.112831115722656,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.104639053344727,
      "fcm_dpo/q_t": 0.4654046893119812,
      "grad_norm": 17.448545455932617,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": 1.7004122734069824,
      "logits/rejected": 1.6050386428833008,
      "logps/chosen": -282.94683837890625,
      "logps/ref_chosen": -288.9918518066406,
      "logps/ref_rejected": -228.26071166992188,
      "logps/rejected": -236.3203125,
      "loss": 5.0651,
      "margin_dpo/margin_mean": 14.104639053344727,
      "margin_dpo/margin_std": 24.29148292541504,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": 5.657957077026367,
      "KL/mean": -2.450857162475586,
      "KL/rejected_KL_mean": -10.55966567993164,
      "KL/std": 21.273914337158203,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.217622756958008,
      "fcm_dpo/q_t": 0.46028897166252136,
      "grad_norm": 18.56569480895996,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": 1.51508367061615,
      "logits/rejected": 1.548736810684204,
      "logps/chosen": -283.8165283203125,
      "logps/ref_chosen": -289.4744873046875,
      "logps/ref_rejected": -265.130615234375,
      "logps/rejected": -275.6903381347656,
      "loss": 4.9987,
      "margin_dpo/margin_mean": 16.217622756958008,
      "margin_dpo/margin_std": 26.78700065612793,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": 6.687526702880859,
      "KL/mean": 0.01107865571975708,
      "KL/rejected_KL_mean": -6.665369033813477,
      "KL/std": 20.569087982177734,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.352893829345703,
      "fcm_dpo/q_t": 0.4672967195510864,
      "grad_norm": 17.38410186767578,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": 1.7959340810775757,
      "logits/rejected": 1.7862217426300049,
      "logps/chosen": -283.89886474609375,
      "logps/ref_chosen": -290.5863342285156,
      "logps/ref_rejected": -263.41046142578125,
      "logps/rejected": -270.0758361816406,
      "loss": 5.0926,
      "margin_dpo/margin_mean": 13.35289478302002,
      "margin_dpo/margin_std": 24.242507934570312,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": 7.184700012207031,
      "KL/mean": -1.5372321605682373,
      "KL/rejected_KL_mean": -10.259162902832031,
      "KL/std": 19.50295639038086,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.44386100769043,
      "fcm_dpo/q_t": 0.4573151469230652,
      "grad_norm": 16.982301712036133,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": 1.563586711883545,
      "logits/rejected": 1.4701849222183228,
      "logps/chosen": -257.35552978515625,
      "logps/ref_chosen": -264.5402526855469,
      "logps/ref_rejected": -214.7958221435547,
      "logps/rejected": -225.05499267578125,
      "loss": 4.9461,
      "margin_dpo/margin_mean": 17.44386100769043,
      "margin_dpo/margin_std": 24.419090270996094,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": 2.879018783569336,
      "KL/mean": -4.183847904205322,
      "KL/rejected_KL_mean": -11.246709823608398,
      "KL/std": 19.734619140625,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.125726699829102,
      "fcm_dpo/q_t": 0.46529585123062134,
      "grad_norm": 29.311634063720703,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": 1.5596039295196533,
      "logits/rejected": 1.680344820022583,
      "logps/chosen": -267.3007507324219,
      "logps/ref_chosen": -270.17974853515625,
      "logps/ref_rejected": -261.2323913574219,
      "logps/rejected": -272.4790954589844,
      "loss": 5.0632,
      "margin_dpo/margin_mean": 14.125726699829102,
      "margin_dpo/margin_std": 23.987531661987305,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": 1.5898914337158203,
      "KL/mean": -4.320026874542236,
      "KL/rejected_KL_mean": -10.229944229125977,
      "KL/std": 22.100967407226562,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.819831848144531,
      "fcm_dpo/q_t": 0.47107842564582825,
      "grad_norm": 17.442039489746094,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": 1.674518346786499,
      "logits/rejected": 1.6614737510681152,
      "logps/chosen": -281.1907043457031,
      "logps/ref_chosen": -282.7806091308594,
      "logps/ref_rejected": -249.23399353027344,
      "logps/rejected": -259.46392822265625,
      "loss": 5.1677,
      "margin_dpo/margin_mean": 11.819829940795898,
      "margin_dpo/margin_std": 26.621383666992188,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": 2.060588836669922,
      "KL/mean": -4.167018890380859,
      "KL/rejected_KL_mean": -10.394632339477539,
      "KL/std": 20.83150863647461,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 12.455221176147461,
      "fcm_dpo/q_t": 0.46953946352005005,
      "grad_norm": 16.677736282348633,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": 1.6153652667999268,
      "logits/rejected": 1.6330746412277222,
      "logps/chosen": -226.85670471191406,
      "logps/ref_chosen": -228.9173126220703,
      "logps/ref_rejected": -226.81222534179688,
      "logps/rejected": -237.20687866210938,
      "loss": 5.135,
      "margin_dpo/margin_mean": 12.455220222473145,
      "margin_dpo/margin_std": 25.950029373168945,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": 3.5226306915283203,
      "KL/mean": -4.322154998779297,
      "KL/rejected_KL_mean": -12.166948318481445,
      "KL/std": 21.527376174926758,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 15.689584732055664,
      "fcm_dpo/q_t": 0.4615984559059143,
      "grad_norm": 19.310375213623047,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": 1.6946752071380615,
      "logits/rejected": 1.6704270839691162,
      "logps/chosen": -307.56329345703125,
      "logps/ref_chosen": -311.0859375,
      "logps/ref_rejected": -303.225341796875,
      "logps/rejected": -315.3923034667969,
      "loss": 5.0107,
      "margin_dpo/margin_mean": 15.689586639404297,
      "margin_dpo/margin_std": 25.566741943359375,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": 0.9385471343994141,
      "KL/mean": -6.48879861831665,
      "KL/rejected_KL_mean": -13.916149139404297,
      "KL/std": 20.787277221679688,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 14.854694366455078,
      "fcm_dpo/q_t": 0.4635513722896576,
      "grad_norm": 17.304546356201172,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 1.6063247919082642,
      "logits/rejected": 1.618200659751892,
      "logps/chosen": -269.5448913574219,
      "logps/ref_chosen": -270.48345947265625,
      "logps/ref_rejected": -269.0491638183594,
      "logps/rejected": -282.9653015136719,
      "loss": 5.0371,
      "margin_dpo/margin_mean": 14.854696273803711,
      "margin_dpo/margin_std": 24.72213363647461,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": 4.152484893798828,
      "KL/mean": -7.05013370513916,
      "KL/rejected_KL_mean": -18.25274658203125,
      "KL/std": 22.920394897460938,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.40522003173828,
      "fcm_dpo/q_t": 0.4453064203262329,
      "grad_norm": 19.656705856323242,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.4898055791854858,
      "logits/rejected": 1.503796935081482,
      "logps/chosen": -284.01324462890625,
      "logps/ref_chosen": -288.16571044921875,
      "logps/ref_rejected": -264.7412414550781,
      "logps/rejected": -282.9939880371094,
      "loss": 4.7835,
      "margin_dpo/margin_mean": 22.40522003173828,
      "margin_dpo/margin_std": 28.56892967224121,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": 2.673847198486328,
      "KL/mean": -5.841534614562988,
      "KL/rejected_KL_mean": -14.356914520263672,
      "KL/std": 23.154693603515625,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.0307674407959,
      "fcm_dpo/q_t": 0.45838403701782227,
      "grad_norm": 18.83303451538086,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": 1.2311562299728394,
      "logits/rejected": 1.2978957891464233,
      "logps/chosen": -257.2646484375,
      "logps/ref_chosen": -259.9385070800781,
      "logps/ref_rejected": -266.2453308105469,
      "logps/rejected": -280.6022644042969,
      "loss": 4.9771,
      "margin_dpo/margin_mean": 17.0307674407959,
      "margin_dpo/margin_std": 27.742591857910156,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": 1.5581169128417969,
      "KL/mean": -10.75131893157959,
      "KL/rejected_KL_mean": -23.060762405395508,
      "KL/std": 24.474565505981445,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.009995264001190662,
      "fcm_dpo/delta": -0.0007565049454569817,
      "fcm_dpo/margin": 24.618871688842773,
      "fcm_dpo/q_t": 0.439913809299469,
      "grad_norm": 23.273386001586914,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 1.5982732772827148,
      "logits/rejected": 1.6644251346588135,
      "logps/chosen": -264.0189208984375,
      "logps/ref_chosen": -265.5769958496094,
      "logps/ref_rejected": -277.6246032714844,
      "logps/rejected": -300.6853332519531,
      "loss": 4.6971,
      "margin_dpo/margin_mean": 24.618873596191406,
      "margin_dpo/margin_std": 27.06637191772461,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -0.3784065246582031,
      "KL/mean": -9.293580055236816,
      "KL/rejected_KL_mean": -18.208755493164062,
      "KL/std": 25.133440017700195,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.009993948973715305,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.83034896850586,
      "fcm_dpo/q_t": 0.4565150737762451,
      "grad_norm": 19.511394500732422,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": 1.5495045185089111,
      "logits/rejected": 1.5304827690124512,
      "logps/chosen": -282.65179443359375,
      "logps/ref_chosen": -282.2734069824219,
      "logps/ref_rejected": -261.5135498046875,
      "logps/rejected": -279.7223205566406,
      "loss": 4.9692,
      "margin_dpo/margin_mean": 17.83034896850586,
      "margin_dpo/margin_std": 31.433273315429688,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -1.4722671508789062,
      "KL/mean": -8.12285327911377,
      "KL/rejected_KL_mean": -14.773447036743164,
      "KL/std": 22.899776458740234,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.009993948973715305,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.301179885864258,
      "fcm_dpo/q_t": 0.46754178404808044,
      "grad_norm": 19.214637756347656,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": 1.5865799188613892,
      "logits/rejected": 1.681269645690918,
      "logps/chosen": -280.183837890625,
      "logps/ref_chosen": -278.7115783691406,
      "logps/ref_rejected": -298.0945739746094,
      "logps/rejected": -312.8680419921875,
      "loss": 5.1183,
      "margin_dpo/margin_mean": 13.301179885864258,
      "margin_dpo/margin_std": 28.247825622558594,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -0.22804641723632812,
      "KL/mean": -11.32726764678955,
      "KL/rejected_KL_mean": -22.42649269104004,
      "KL/std": 25.214733123779297,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.009993948973715305,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.198450088500977,
      "fcm_dpo/q_t": 0.4460859000682831,
      "grad_norm": 19.43053436279297,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": 1.5009217262268066,
      "logits/rejected": 1.5161949396133423,
      "logps/chosen": -268.5791931152344,
      "logps/ref_chosen": -268.35113525390625,
      "logps/ref_rejected": -241.9660186767578,
      "logps/rejected": -264.39251708984375,
      "loss": 4.819,
      "margin_dpo/margin_mean": 22.198450088500977,
      "margin_dpo/margin_std": 33.27134323120117,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -1.3504199981689453,
      "KL/mean": -8.988754272460938,
      "KL/rejected_KL_mean": -16.627086639404297,
      "KL/std": 24.742692947387695,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.009990633465349674,
      "fcm_dpo/delta": -0.0004148362495470792,
      "fcm_dpo/margin": 15.276663780212402,
      "fcm_dpo/q_t": 0.46282583475112915,
      "grad_norm": 24.745080947875977,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": 1.3932499885559082,
      "logits/rejected": 1.452117919921875,
      "logps/chosen": -266.7540588378906,
      "logps/ref_chosen": -265.40362548828125,
      "logps/ref_rejected": -257.82440185546875,
      "logps/rejected": -274.45147705078125,
      "loss": 5.0507,
      "margin_dpo/margin_mean": 15.276664733886719,
      "margin_dpo/margin_std": 28.926172256469727,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -0.11916542053222656,
      "KL/mean": -8.577759742736816,
      "KL/rejected_KL_mean": -17.036344528198242,
      "KL/std": 24.489749908447266,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.009990633465349674,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.917184829711914,
      "fcm_dpo/q_t": 0.45870956778526306,
      "grad_norm": 20.150331497192383,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": 1.483914852142334,
      "logits/rejected": 1.5301220417022705,
      "logps/chosen": -281.281494140625,
      "logps/ref_chosen": -281.1622619628906,
      "logps/ref_rejected": -268.2899475097656,
      "logps/rejected": -285.3262634277344,
      "loss": 4.9895,
      "margin_dpo/margin_mean": 16.917186737060547,
      "margin_dpo/margin_std": 29.337505340576172,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -0.9003562927246094,
      "KL/mean": -7.60732364654541,
      "KL/rejected_KL_mean": -14.314277648925781,
      "KL/std": 25.489770889282227,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.009990633465349674,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 13.41391658782959,
      "fcm_dpo/q_t": 0.4676581025123596,
      "grad_norm": 20.158632278442383,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": 1.6424541473388672,
      "logits/rejected": 1.542797327041626,
      "logps/chosen": -312.2253723144531,
      "logps/ref_chosen": -311.32501220703125,
      "logps/ref_rejected": -248.85350036621094,
      "logps/rejected": -263.16778564453125,
      "loss": 5.1534,
      "margin_dpo/margin_mean": 13.413918495178223,
      "margin_dpo/margin_std": 33.91361999511719,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": 3.419717788696289,
      "KL/mean": -6.92333984375,
      "KL/rejected_KL_mean": -17.266403198242188,
      "KL/std": 25.268657684326172,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.00998927466571331,
      "fcm_dpo/delta": -0.0014671060489490628,
      "fcm_dpo/margin": 20.686128616333008,
      "fcm_dpo/q_t": 0.4498874843120575,
      "grad_norm": 19.27452850341797,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": 1.4546760320663452,
      "logits/rejected": 1.4584565162658691,
      "logps/chosen": -263.6892395019531,
      "logps/ref_chosen": -267.10894775390625,
      "logps/ref_rejected": -258.66162109375,
      "logps/rejected": -275.92803955078125,
      "loss": 4.8573,
      "margin_dpo/margin_mean": 20.686128616333008,
      "margin_dpo/margin_std": 29.415864944458008,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": 2.7684383392333984,
      "KL/mean": -8.059282302856445,
      "KL/rejected_KL_mean": -18.886999130249023,
      "KL/std": 26.406251907348633,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.009979812428355217,
      "fcm_dpo/delta": 0.00029973231721669436,
      "fcm_dpo/margin": 21.655437469482422,
      "fcm_dpo/q_t": 0.4476424753665924,
      "grad_norm": 29.706693649291992,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 1.6041282415390015,
      "logits/rejected": 1.485061764717102,
      "logps/chosen": -299.1239318847656,
      "logps/ref_chosen": -301.89239501953125,
      "logps/ref_rejected": -247.645263671875,
      "logps/rejected": -266.5322570800781,
      "loss": 4.8402,
      "margin_dpo/margin_mean": 21.655437469482422,
      "margin_dpo/margin_std": 33.19287872314453,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": 4.464748382568359,
      "KL/mean": -5.3116912841796875,
      "KL/rejected_KL_mean": -15.088132858276367,
      "KL/std": 25.467761993408203,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.009891926310956478,
      "fcm_dpo/delta": -0.011244099587202072,
      "fcm_dpo/margin": 19.552873611450195,
      "fcm_dpo/q_t": 0.45277583599090576,
      "grad_norm": 21.963769912719727,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": 1.5187616348266602,
      "logits/rejected": 1.4979108572006226,
      "logps/chosen": -283.768310546875,
      "logps/ref_chosen": -288.2330627441406,
      "logps/ref_rejected": -252.5148162841797,
      "logps/rejected": -267.6029357910156,
      "loss": 4.9122,
      "margin_dpo/margin_mean": 19.552875518798828,
      "margin_dpo/margin_std": 30.845937728881836,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": 3.1825485229492188,
      "KL/mean": -8.50892448425293,
      "KL/rejected_KL_mean": -20.20039176940918,
      "KL/std": 24.872844696044922,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.009860838763415813,
      "fcm_dpo/delta": -0.007233901880681515,
      "fcm_dpo/margin": 23.38294219970703,
      "fcm_dpo/q_t": 0.44364166259765625,
      "grad_norm": 21.43950653076172,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": 1.641586422920227,
      "logits/rejected": 1.6236480474472046,
      "logps/chosen": -283.5812683105469,
      "logps/ref_chosen": -286.7638244628906,
      "logps/ref_rejected": -262.8302917480469,
      "logps/rejected": -283.0306396484375,
      "loss": 4.7555,
      "margin_dpo/margin_mean": 23.38294219970703,
      "margin_dpo/margin_std": 27.76974105834961,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -4.252696990966797,
      "KL/mean": -14.268733978271484,
      "KL/rejected_KL_mean": -24.284772872924805,
      "KL/std": 26.910919189453125,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.009834406897425652,
      "fcm_dpo/delta": -0.0004478114715311676,
      "fcm_dpo/margin": 20.032072067260742,
      "fcm_dpo/q_t": 0.4520826041698456,
      "grad_norm": 19.745298385620117,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": 1.59385085105896,
      "logits/rejected": 1.5379009246826172,
      "logps/chosen": -264.53118896484375,
      "logps/ref_chosen": -260.2784729003906,
      "logps/ref_rejected": -217.10556030273438,
      "logps/rejected": -241.3903350830078,
      "loss": 4.8931,
      "margin_dpo/margin_mean": 20.032072067260742,
      "margin_dpo/margin_std": 30.785341262817383,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -0.38128662109375,
      "KL/mean": -9.025415420532227,
      "KL/rejected_KL_mean": -17.669544219970703,
      "KL/std": 24.980144500732422,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.009831324219703674,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.28824806213379,
      "fcm_dpo/q_t": 0.4585840702056885,
      "grad_norm": 19.754850387573242,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": 1.5577529668807983,
      "logits/rejected": 1.6014890670776367,
      "logps/chosen": -270.9355773925781,
      "logps/ref_chosen": -270.5543212890625,
      "logps/ref_rejected": -259.80218505859375,
      "logps/rejected": -277.4717102050781,
      "loss": 4.9814,
      "margin_dpo/margin_mean": 17.288249969482422,
      "margin_dpo/margin_std": 28.863998413085938,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -3.4009971618652344,
      "KL/mean": -11.112358093261719,
      "KL/rejected_KL_mean": -18.823715209960938,
      "KL/std": 27.504425048828125,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.009834278374910355,
      "fcm_dpo/delta": 0.0003755904908757657,
      "fcm_dpo/margin": 15.422724723815918,
      "fcm_dpo/q_t": 0.4634696841239929,
      "grad_norm": 17.79340362548828,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 1.4999161958694458,
      "logits/rejected": 1.4775928258895874,
      "logps/chosen": -273.6954345703125,
      "logps/ref_chosen": -270.29443359375,
      "logps/ref_rejected": -230.40151977539062,
      "logps/rejected": -249.2252197265625,
      "loss": 5.0892,
      "margin_dpo/margin_mean": 15.422723770141602,
      "margin_dpo/margin_std": 35.43870162963867,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -3.945913314819336,
      "KL/mean": -13.22420597076416,
      "KL/rejected_KL_mean": -22.502498626708984,
      "KL/std": 26.921138763427734,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.00983255822211504,
      "fcm_dpo/delta": -0.0017504449933767319,
      "fcm_dpo/margin": 18.556583404541016,
      "fcm_dpo/q_t": 0.45566678047180176,
      "grad_norm": 19.33642578125,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": 1.9277331829071045,
      "logits/rejected": 1.89165198802948,
      "logps/chosen": -277.406494140625,
      "logps/ref_chosen": -273.4605712890625,
      "logps/ref_rejected": -248.86997985839844,
      "logps/rejected": -271.3724670410156,
      "loss": 4.9547,
      "margin_dpo/margin_mean": 18.556583404541016,
      "margin_dpo/margin_std": 32.274539947509766,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": -0.8726158142089844,
      "KL/mean": -12.831059455871582,
      "KL/rejected_KL_mean": -24.789505004882812,
      "KL/std": 29.189624786376953,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.009743379428982735,
      "fcm_dpo/delta": -0.016302837058901787,
      "fcm_dpo/margin": 23.91689109802246,
      "fcm_dpo/q_t": 0.4432268738746643,
      "grad_norm": 21.300148010253906,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": 1.644484519958496,
      "logits/rejected": 1.6502946615219116,
      "logps/chosen": -306.5973205566406,
      "logps/ref_chosen": -305.7247009277344,
      "logps/ref_rejected": -290.16650390625,
      "logps/rejected": -314.9560241699219,
      "loss": 4.7733,
      "margin_dpo/margin_mean": 23.91689109802246,
      "margin_dpo/margin_std": 31.794357299804688,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -1.3619880676269531,
      "KL/mean": -15.22814655303955,
      "KL/rejected_KL_mean": -29.09429931640625,
      "KL/std": 29.04392433166504,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.009559480473399162,
      "fcm_dpo/delta": -0.03935708478093147,
      "fcm_dpo/margin": 27.732315063476562,
      "fcm_dpo/q_t": 0.43571561574935913,
      "grad_norm": 20.254554748535156,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": 1.4861019849777222,
      "logits/rejected": 1.5532933473587036,
      "logps/chosen": -253.5322723388672,
      "logps/ref_chosen": -252.1702880859375,
      "logps/ref_rejected": -233.34750366210938,
      "logps/rejected": -262.4418029785156,
      "loss": 4.6667,
      "margin_dpo/margin_mean": 27.73231315612793,
      "margin_dpo/margin_std": 32.52809524536133,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": -0.17887496948242188,
      "KL/mean": -12.304729461669922,
      "KL/rejected_KL_mean": -24.43058204650879,
      "KL/std": 27.611595153808594,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.009392821229994297,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.251708984375,
      "fcm_dpo/q_t": 0.4447784721851349,
      "grad_norm": 20.589649200439453,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": 1.4018384218215942,
      "logits/rejected": 1.4390873908996582,
      "logps/chosen": -302.6572570800781,
      "logps/ref_chosen": -302.4783935546875,
      "logps/ref_rejected": -259.40777587890625,
      "logps/rejected": -283.8383483886719,
      "loss": 4.7825,
      "margin_dpo/margin_mean": 24.251705169677734,
      "margin_dpo/margin_std": 33.139339447021484,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -0.4895973205566406,
      "KL/mean": -12.768810272216797,
      "KL/rejected_KL_mean": -25.048017501831055,
      "KL/std": 28.41954803466797,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.00931854359805584,
      "fcm_dpo/delta": -0.012652948498725891,
      "fcm_dpo/margin": 24.558420181274414,
      "fcm_dpo/q_t": 0.4444689154624939,
      "grad_norm": 21.295276641845703,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 1.4507535696029663,
      "logits/rejected": 1.5624240636825562,
      "logps/chosen": -251.13632202148438,
      "logps/ref_chosen": -250.646728515625,
      "logps/ref_rejected": -259.1659851074219,
      "logps/rejected": -284.2140197753906,
      "loss": 4.7822,
      "margin_dpo/margin_mean": 24.558420181274414,
      "margin_dpo/margin_std": 32.15560531616211,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": 0.6685714721679688,
      "KL/mean": -7.651555061340332,
      "KL/rejected_KL_mean": -15.971687316894531,
      "KL/std": 29.07240104675293,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.009298224002122879,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.64024543762207,
      "fcm_dpo/q_t": 0.4621499180793762,
      "grad_norm": 19.16387939453125,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": 1.580474615097046,
      "logits/rejected": 1.5812137126922607,
      "logps/chosen": -328.3563537597656,
      "logps/ref_chosen": -329.02490234375,
      "logps/ref_rejected": -293.0215759277344,
      "logps/rejected": -308.9932556152344,
      "loss": 5.0568,
      "margin_dpo/margin_mean": 16.640243530273438,
      "margin_dpo/margin_std": 33.943267822265625,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -0.3389263153076172,
      "KL/mean": -10.601463317871094,
      "KL/rejected_KL_mean": -20.86400032043457,
      "KL/std": 28.963924407958984,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.009294095449149609,
      "fcm_dpo/delta": -0.0015207678079605103,
      "fcm_dpo/margin": 20.525075912475586,
      "fcm_dpo/q_t": 0.4536864757537842,
      "grad_norm": 19.17975425720215,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": 1.4235897064208984,
      "logits/rejected": 1.3849798440933228,
      "logps/chosen": -290.47955322265625,
      "logps/ref_chosen": -290.140625,
      "logps/ref_rejected": -271.474853515625,
      "logps/rejected": -292.3388366699219,
      "loss": 4.9176,
      "margin_dpo/margin_mean": 20.525074005126953,
      "margin_dpo/margin_std": 32.40163803100586,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": 0.5062465667724609,
      "KL/mean": -12.566383361816406,
      "KL/rejected_KL_mean": -25.63901138305664,
      "KL/std": 28.489694595336914,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.009213409386575222,
      "fcm_dpo/delta": -0.015376418828964233,
      "fcm_dpo/margin": 26.1452579498291,
      "fcm_dpo/q_t": 0.4415484070777893,
      "grad_norm": 25.520977020263672,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": 1.5168626308441162,
      "logits/rejected": 1.432292103767395,
      "logps/chosen": -328.801025390625,
      "logps/ref_chosen": -329.3072509765625,
      "logps/ref_rejected": -256.6839599609375,
      "logps/rejected": -282.3229675292969,
      "loss": 4.7348,
      "margin_dpo/margin_mean": 26.1452579498291,
      "margin_dpo/margin_std": 32.158935546875,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -2.4713592529296875,
      "KL/mean": -13.017889976501465,
      "KL/rejected_KL_mean": -23.564430236816406,
      "KL/std": 29.537261962890625,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.009160052053630352,
      "fcm_dpo/delta": -0.00363737135194242,
      "fcm_dpo/margin": 21.09307289123535,
      "fcm_dpo/q_t": 0.4529308080673218,
      "grad_norm": 23.662338256835938,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": 1.4738250970840454,
      "logits/rejected": 1.4628231525421143,
      "logps/chosen": -278.3736267089844,
      "logps/ref_chosen": -275.9023132324219,
      "logps/ref_rejected": -281.4163513183594,
      "logps/rejected": -304.98077392578125,
      "loss": 4.9195,
      "margin_dpo/margin_mean": 21.093074798583984,
      "margin_dpo/margin_std": 35.23958206176758,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -4.555610656738281,
      "KL/mean": -12.401081085205078,
      "KL/rejected_KL_mean": -20.246553421020508,
      "KL/std": 29.156707763671875,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.00911801028996706,
      "fcm_dpo/delta": -0.008613883517682552,
      "fcm_dpo/margin": 15.690947532653809,
      "fcm_dpo/q_t": 0.465049684047699,
      "grad_norm": 26.095781326293945,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": 1.4742199182510376,
      "logits/rejected": 1.544607400894165,
      "logps/chosen": -248.1198272705078,
      "logps/ref_chosen": -243.56423950195312,
      "logps/ref_rejected": -239.12905883789062,
      "logps/rejected": -259.3756103515625,
      "loss": 5.1153,
      "margin_dpo/margin_mean": 15.690948486328125,
      "margin_dpo/margin_std": 35.76212692260742,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -5.992158889770508,
      "KL/mean": -13.610573768615723,
      "KL/rejected_KL_mean": -21.22899627685547,
      "KL/std": 30.305936813354492,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.009079324081540108,
      "fcm_dpo/delta": -0.0016844363417476416,
      "fcm_dpo/margin": 15.236846923828125,
      "fcm_dpo/q_t": 0.4663734436035156,
      "grad_norm": 30.18008041381836,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": 1.4929338693618774,
      "logits/rejected": 1.414791226387024,
      "logps/chosen": -299.29583740234375,
      "logps/ref_chosen": -293.30364990234375,
      "logps/ref_rejected": -292.293701171875,
      "logps/rejected": -313.522705078125,
      "loss": 5.1551,
      "margin_dpo/margin_mean": 15.236845970153809,
      "margin_dpo/margin_std": 39.812374114990234,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -1.4761123657226562,
      "KL/mean": -13.943385124206543,
      "KL/rejected_KL_mean": -26.410659790039062,
      "KL/std": 31.966270446777344,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.00902348943054676,
      "fcm_dpo/delta": -0.009051913395524025,
      "fcm_dpo/margin": 24.934547424316406,
      "fcm_dpo/q_t": 0.44572871923446655,
      "grad_norm": 18.953073501586914,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": 1.444204568862915,
      "logits/rejected": 1.4593132734298706,
      "logps/chosen": -277.6023254394531,
      "logps/ref_chosen": -276.126220703125,
      "logps/ref_rejected": -240.31179809570312,
      "logps/rejected": -266.72247314453125,
      "loss": 4.8213,
      "margin_dpo/margin_mean": 24.934545516967773,
      "margin_dpo/margin_std": 38.33395767211914,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -1.830230712890625,
      "KL/mean": -11.061882972717285,
      "KL/rejected_KL_mean": -20.293535232543945,
      "KL/std": 30.859411239624023,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.009007247164845467,
      "fcm_dpo/delta": 0.00016247756138909608,
      "fcm_dpo/margin": 18.463321685791016,
      "fcm_dpo/q_t": 0.4596945643424988,
      "grad_norm": 33.445045471191406,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 1.3365435600280762,
      "logits/rejected": 1.4510858058929443,
      "logps/chosen": -251.69265747070312,
      "logps/ref_chosen": -249.8624267578125,
      "logps/ref_rejected": -271.18603515625,
      "logps/rejected": -291.4795837402344,
      "loss": 5.0241,
      "margin_dpo/margin_mean": 18.46332359313965,
      "margin_dpo/margin_std": 36.16477966308594,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -2.092733383178711,
      "KL/mean": -14.033413887023926,
      "KL/rejected_KL_mean": -25.97409439086914,
      "KL/std": 30.43889808654785,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.009008214809000492,
      "fcm_dpo/delta": 0.0003039139846805483,
      "fcm_dpo/margin": 23.881370544433594,
      "fcm_dpo/q_t": 0.4480440616607666,
      "grad_norm": 22.631591796875,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": 1.6473186016082764,
      "logits/rejected": 1.7018965482711792,
      "logps/chosen": -262.7618408203125,
      "logps/ref_chosen": -260.66912841796875,
      "logps/ref_rejected": -229.25491333007812,
      "logps/rejected": -255.22900390625,
      "loss": 4.844,
      "margin_dpo/margin_mean": 23.881370544433594,
      "margin_dpo/margin_std": 36.19492721557617,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -0.7707386016845703,
      "KL/mean": -12.198413848876953,
      "KL/rejected_KL_mean": -23.62607765197754,
      "KL/std": 31.259761810302734,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.009004123508930206,
      "fcm_dpo/delta": -0.00015471281949430704,
      "fcm_dpo/margin": 22.855342864990234,
      "fcm_dpo/q_t": 0.4502585232257843,
      "grad_norm": 21.132755279541016,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.6749849319458008,
      "logits/rejected": 1.6622705459594727,
      "logps/chosen": -291.6593322753906,
      "logps/ref_chosen": -290.8885803222656,
      "logps/ref_rejected": -275.1922302246094,
      "logps/rejected": -298.81829833984375,
      "loss": 4.8963,
      "margin_dpo/margin_mean": 22.855342864990234,
      "margin_dpo/margin_std": 38.71193313598633,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -2.9608802795410156,
      "KL/mean": -17.346607208251953,
      "KL/rejected_KL_mean": -31.732336044311523,
      "KL/std": 31.395177841186523,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.008964559994637966,
      "fcm_dpo/delta": -0.011344683356583118,
      "fcm_dpo/margin": 28.771465301513672,
      "fcm_dpo/q_t": 0.437173068523407,
      "grad_norm": 30.354036331176758,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": 1.2978819608688354,
      "logits/rejected": 1.2993167638778687,
      "logps/chosen": -273.5277099609375,
      "logps/ref_chosen": -270.56683349609375,
      "logps/ref_rejected": -260.3321533203125,
      "logps/rejected": -292.06451416015625,
      "loss": 4.6755,
      "margin_dpo/margin_mean": 28.771465301513672,
      "margin_dpo/margin_std": 33.544090270996094,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -3.531404495239258,
      "KL/mean": -15.652979850769043,
      "KL/rejected_KL_mean": -27.774555206298828,
      "KL/std": 31.97277069091797,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.008923518471419811,
      "fcm_dpo/delta": -0.0006651444127783179,
      "fcm_dpo/margin": 24.243160247802734,
      "fcm_dpo/q_t": 0.44765886664390564,
      "grad_norm": 21.057531356811523,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": 1.472924828529358,
      "logits/rejected": 1.5117961168289185,
      "logps/chosen": -264.55938720703125,
      "logps/ref_chosen": -261.0279846191406,
      "logps/ref_rejected": -240.72361755371094,
      "logps/rejected": -268.4981994628906,
      "loss": 4.85,
      "margin_dpo/margin_mean": 24.243160247802734,
      "margin_dpo/margin_std": 38.97475814819336,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": 0.018901824951171875,
      "KL/mean": -18.113346099853516,
      "KL/rejected_KL_mean": -36.24559020996094,
      "KL/std": 36.01204299926758,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.00870735477656126,
      "fcm_dpo/delta": -0.03583183512091637,
      "fcm_dpo/margin": 36.26449966430664,
      "fcm_dpo/q_t": 0.4236949384212494,
      "grad_norm": 41.38897705078125,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": 1.5731693506240845,
      "logits/rejected": 1.5615172386169434,
      "logps/chosen": -309.06494140625,
      "logps/ref_chosen": -309.0838317871094,
      "logps/ref_rejected": -263.6435241699219,
      "logps/rejected": -299.8891296386719,
      "loss": 4.5255,
      "margin_dpo/margin_mean": 36.26449966430664,
      "margin_dpo/margin_std": 43.175025939941406,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -8.22397232055664,
      "KL/mean": -20.043304443359375,
      "KL/rejected_KL_mean": -31.86264419555664,
      "KL/std": 33.87177276611328,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.008590037934482098,
      "fcm_dpo/delta": -0.018629377707839012,
      "fcm_dpo/margin": 23.638673782348633,
      "fcm_dpo/q_t": 0.4508914053440094,
      "grad_norm": 21.754732131958008,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": 1.5534719228744507,
      "logits/rejected": 1.6342486143112183,
      "logps/chosen": -273.9256896972656,
      "logps/ref_chosen": -265.70172119140625,
      "logps/ref_rejected": -257.5074157714844,
      "logps/rejected": -289.37005615234375,
      "loss": 4.9049,
      "margin_dpo/margin_mean": 23.638671875,
      "margin_dpo/margin_std": 40.475074768066406,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -11.72828483581543,
      "KL/mean": -23.24701690673828,
      "KL/rejected_KL_mean": -34.76573944091797,
      "KL/std": 32.52372360229492,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.008514071814715862,
      "fcm_dpo/delta": -0.008183183148503304,
      "fcm_dpo/margin": 23.03746223449707,
      "fcm_dpo/q_t": 0.4524497985839844,
      "grad_norm": 28.596054077148438,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": 1.522101640701294,
      "logits/rejected": 1.5488725900650024,
      "logps/chosen": -303.4532775878906,
      "logps/ref_chosen": -291.7249755859375,
      "logps/ref_rejected": -273.54010009765625,
      "logps/rejected": -308.3058776855469,
      "loss": 4.9074,
      "margin_dpo/margin_mean": 23.03746223449707,
      "margin_dpo/margin_std": 37.29114532470703,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -11.68828010559082,
      "KL/mean": -23.277896881103516,
      "KL/rejected_KL_mean": -34.867515563964844,
      "KL/std": 34.64895248413086,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.008465162478387356,
      "fcm_dpo/delta": -0.004968404769897461,
      "fcm_dpo/margin": 23.179241180419922,
      "fcm_dpo/q_t": 0.4528391361236572,
      "grad_norm": 20.818782806396484,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": 1.581821084022522,
      "logits/rejected": 1.6136670112609863,
      "logps/chosen": -333.3901062011719,
      "logps/ref_chosen": -321.70184326171875,
      "logps/ref_rejected": -261.8386535644531,
      "logps/rejected": -296.7061767578125,
      "loss": 4.9466,
      "margin_dpo/margin_mean": 23.179241180419922,
      "margin_dpo/margin_std": 45.09851837158203,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -14.058746337890625,
      "KL/mean": -26.37923812866211,
      "KL/rejected_KL_mean": -38.699737548828125,
      "KL/std": 34.09037780761719,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.00842692144215107,
      "fcm_dpo/delta": -0.007600238546729088,
      "fcm_dpo/margin": 24.640995025634766,
      "fcm_dpo/q_t": 0.4496771991252899,
      "grad_norm": 40.07311248779297,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": 1.394249439239502,
      "logits/rejected": 1.3684768676757812,
      "logps/chosen": -269.9741516113281,
      "logps/ref_chosen": -255.91542053222656,
      "logps/ref_rejected": -245.1119842529297,
      "logps/rejected": -283.8117370605469,
      "loss": 4.8996,
      "margin_dpo/margin_mean": 24.640995025634766,
      "margin_dpo/margin_std": 43.73637390136719,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -6.180440902709961,
      "KL/mean": -20.932891845703125,
      "KL/rejected_KL_mean": -35.68534851074219,
      "KL/std": 34.318817138671875,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.00838015042245388,
      "fcm_dpo/delta": -0.009131775237619877,
      "fcm_dpo/margin": 29.504898071289062,
      "fcm_dpo/q_t": 0.43997615575790405,
      "grad_norm": 29.615734100341797,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": 1.5812218189239502,
      "logits/rejected": 1.6139171123504639,
      "logps/chosen": -272.1990051269531,
      "logps/ref_chosen": -266.0185546875,
      "logps/ref_rejected": -272.862548828125,
      "logps/rejected": -308.54791259765625,
      "loss": 4.7173,
      "margin_dpo/margin_mean": 29.504899978637695,
      "margin_dpo/margin_std": 36.251216888427734,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -8.521604537963867,
      "KL/mean": -22.739160537719727,
      "KL/rejected_KL_mean": -36.95671463012695,
      "KL/std": 34.48044967651367,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.00831204280257225,
      "fcm_dpo/delta": -0.007393582724034786,
      "fcm_dpo/margin": 28.43511962890625,
      "fcm_dpo/q_t": 0.4426947832107544,
      "grad_norm": 26.062965393066406,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 1.4773905277252197,
      "logits/rejected": 1.4219218492507935,
      "logps/chosen": -258.6725769042969,
      "logps/ref_chosen": -250.15098571777344,
      "logps/ref_rejected": -223.34393310546875,
      "logps/rejected": -260.3006286621094,
      "loss": 4.7857,
      "margin_dpo/margin_mean": 28.43511962890625,
      "margin_dpo/margin_std": 42.52570724487305,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -5.975837707519531,
      "KL/mean": -19.097375869750977,
      "KL/rejected_KL_mean": -32.21891784667969,
      "KL/std": 35.73162841796875,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.00825824961066246,
      "fcm_dpo/delta": -0.0045311786234378815,
      "fcm_dpo/margin": 26.243078231811523,
      "fcm_dpo/q_t": 0.4477168917655945,
      "grad_norm": 23.03513526916504,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 1.288292407989502,
      "logits/rejected": 1.3513566255569458,
      "logps/chosen": -244.3676300048828,
      "logps/ref_chosen": -238.3917999267578,
      "logps/ref_rejected": -243.70272827148438,
      "logps/rejected": -275.921630859375,
      "loss": 4.8679,
      "margin_dpo/margin_mean": 26.24307632446289,
      "margin_dpo/margin_std": 44.835594177246094,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -12.41189193725586,
      "KL/mean": -23.2750186920166,
      "KL/rejected_KL_mean": -34.13813781738281,
      "KL/std": 35.38755798339844,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.008221767842769623,
      "fcm_dpo/delta": -0.007616392336785793,
      "fcm_dpo/margin": 21.72624969482422,
      "fcm_dpo/q_t": 0.4566434621810913,
      "grad_norm": 20.782564163208008,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.4401674270629883,
      "logits/rejected": 1.5130709409713745,
      "logps/chosen": -269.04888916015625,
      "logps/ref_chosen": -256.6369934082031,
      "logps/ref_rejected": -256.8894348144531,
      "logps/rejected": -291.027587890625,
      "loss": 5.0166,
      "margin_dpo/margin_mean": 21.72625160217285,
      "margin_dpo/margin_std": 45.12196731567383,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -13.669319152832031,
      "KL/mean": -25.652135848999023,
      "KL/rejected_KL_mean": -37.634944915771484,
      "KL/std": 37.09321212768555,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.008210784755647182,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.96562385559082,
      "fcm_dpo/q_t": 0.45244431495666504,
      "grad_norm": 22.49500274658203,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": 1.648001790046692,
      "logits/rejected": 1.7479485273361206,
      "logps/chosen": -286.468994140625,
      "logps/ref_chosen": -272.7996826171875,
      "logps/ref_rejected": -289.2391052246094,
      "logps/rejected": -326.8740539550781,
      "loss": 4.9227,
      "margin_dpo/margin_mean": 23.965625762939453,
      "margin_dpo/margin_std": 42.74507141113281,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -12.051790237426758,
      "KL/mean": -21.242891311645508,
      "KL/rejected_KL_mean": -30.43399429321289,
      "KL/std": 36.95978546142578,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.008207402192056179,
      "fcm_dpo/delta": -0.0005885101272724569,
      "fcm_dpo/margin": 18.382205963134766,
      "fcm_dpo/q_t": 0.4636900722980499,
      "grad_norm": 22.108482360839844,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": 1.2758780717849731,
      "logits/rejected": 1.3141889572143555,
      "logps/chosen": -293.36083984375,
      "logps/ref_chosen": -281.30902099609375,
      "logps/ref_rejected": -246.3243408203125,
      "logps/rejected": -276.7583312988281,
      "loss": 5.0878,
      "margin_dpo/margin_mean": 18.382205963134766,
      "margin_dpo/margin_std": 41.4659423828125,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -12.519378662109375,
      "KL/mean": -21.443214416503906,
      "KL/rejected_KL_mean": -30.367046356201172,
      "KL/std": 36.74597930908203,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.008212346583604813,
      "fcm_dpo/delta": 0.0009443212184123695,
      "fcm_dpo/margin": 17.84766960144043,
      "fcm_dpo/q_t": 0.46460726857185364,
      "grad_norm": 26.74915885925293,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": 1.6083052158355713,
      "logits/rejected": 1.7155585289001465,
      "logps/chosen": -279.37750244140625,
      "logps/ref_chosen": -266.8581237792969,
      "logps/ref_rejected": -276.0855712890625,
      "logps/rejected": -306.45257568359375,
      "loss": 5.1128,
      "margin_dpo/margin_mean": 17.84766960144043,
      "margin_dpo/margin_std": 43.32354736328125,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -2.700063705444336,
      "KL/mean": -17.782421112060547,
      "KL/rejected_KL_mean": -32.86478805541992,
      "KL/std": 35.061065673828125,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.008145892061293125,
      "fcm_dpo/delta": -0.017433583736419678,
      "fcm_dpo/margin": 30.16472053527832,
      "fcm_dpo/q_t": 0.440427303314209,
      "grad_norm": 21.47694206237793,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": 1.2188905477523804,
      "logits/rejected": 1.3164379596710205,
      "logps/chosen": -269.69500732421875,
      "logps/ref_chosen": -266.99493408203125,
      "logps/ref_rejected": -248.60513305664062,
      "logps/rejected": -281.4699401855469,
      "loss": 4.7314,
      "margin_dpo/margin_mean": 30.164718627929688,
      "margin_dpo/margin_std": 39.10826110839844,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -0.34409332275390625,
      "KL/mean": -19.59783363342285,
      "KL/rejected_KL_mean": -38.85157012939453,
      "KL/std": 38.29167938232422,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.007983393967151642,
      "fcm_dpo/delta": -0.05041225999593735,
      "fcm_dpo/margin": 38.507484436035156,
      "fcm_dpo/q_t": 0.4255077838897705,
      "grad_norm": 22.038917541503906,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": 1.4749455451965332,
      "logits/rejected": 1.4524145126342773,
      "logps/chosen": -303.3338928222656,
      "logps/ref_chosen": -302.98980712890625,
      "logps/ref_rejected": -262.8002624511719,
      "logps/rejected": -301.6518249511719,
      "loss": 4.5319,
      "margin_dpo/margin_mean": 38.507484436035156,
      "margin_dpo/margin_std": 41.13785171508789,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -2.530008316040039,
      "KL/mean": -16.886911392211914,
      "KL/rejected_KL_mean": -31.243806838989258,
      "KL/std": 38.14051055908203,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.007750402670353651,
      "fcm_dpo/delta": -0.006185724399983883,
      "fcm_dpo/margin": 28.713788986206055,
      "fcm_dpo/q_t": 0.44627559185028076,
      "grad_norm": 24.01470947265625,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": 1.5197107791900635,
      "logits/rejected": 1.6091481447219849,
      "logps/chosen": -298.74176025390625,
      "logps/ref_chosen": -296.2117614746094,
      "logps/ref_rejected": -289.8357238769531,
      "logps/rejected": -321.0795593261719,
      "loss": 4.8548,
      "margin_dpo/margin_mean": 28.713790893554688,
      "margin_dpo/margin_std": 48.71750259399414,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -3.047151565551758,
      "KL/mean": -17.014198303222656,
      "KL/rejected_KL_mean": -30.981237411499023,
      "KL/std": 36.06683349609375,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.007740070577710867,
      "fcm_dpo/delta": -0.0009701316594146192,
      "fcm_dpo/margin": 27.93408203125,
      "fcm_dpo/q_t": 0.44766533374786377,
      "grad_norm": 23.824769973754883,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": 1.3609297275543213,
      "logits/rejected": 1.3020771741867065,
      "logps/chosen": -258.33978271484375,
      "logps/ref_chosen": -255.29263305664062,
      "logps/ref_rejected": -212.69761657714844,
      "logps/rejected": -243.67886352539062,
      "loss": 4.851,
      "margin_dpo/margin_mean": 27.934083938598633,
      "margin_dpo/margin_std": 44.772151947021484,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -2.0598583221435547,
      "KL/mean": -14.186567306518555,
      "KL/rejected_KL_mean": -26.313278198242188,
      "KL/std": 37.449859619140625,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.007703812327235937,
      "fcm_dpo/delta": -0.010060962289571762,
      "fcm_dpo/margin": 24.25342559814453,
      "fcm_dpo/q_t": 0.45511072874069214,
      "grad_norm": 19.888334274291992,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 1.5627918243408203,
      "logits/rejected": 1.5573118925094604,
      "logps/chosen": -315.5962829589844,
      "logps/ref_chosen": -313.5364074707031,
      "logps/ref_rejected": -274.349609375,
      "logps/rejected": -300.662841796875,
      "loss": 5.0021,
      "margin_dpo/margin_mean": 24.25342559814453,
      "margin_dpo/margin_std": 52.56910705566406,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_KL/chosen_KL_mean": -6.554592609405518,
      "eval_KL/mean": -20.041173934936523,
      "eval_KL/rejected_KL_mean": -33.52775573730469,
      "eval_KL/std": 37.361328125,
      "eval_fcm_dpo/beta": 0.0076728095300495625,
      "eval_logits/chosen": 1.5835438966751099,
      "eval_logits/rejected": 1.62919282913208,
      "eval_logps/chosen": -286.9712829589844,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -298.17327880859375,
      "eval_loss": 0.6114903688430786,
      "eval_margin_dpo/margin_mean": 26.973159790039062,
      "eval_margin_dpo/margin_std": 43.414180755615234,
      "eval_runtime": 88.9666,
      "eval_samples_per_second": 22.48,
      "eval_steps_per_second": 2.81,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -5.014675140380859,
      "KL/mean": -19.579740524291992,
      "KL/rejected_KL_mean": -34.144805908203125,
      "KL/std": 38.54046630859375,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.007650283165276051,
      "fcm_dpo/delta": -0.01317515131086111,
      "fcm_dpo/margin": 29.130126953125,
      "fcm_dpo/q_t": 0.4459375739097595,
      "grad_norm": 27.600378036499023,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 1.3249129056930542,
      "logits/rejected": 1.364820957183838,
      "logps/chosen": -267.72186279296875,
      "logps/ref_chosen": -262.7071838378906,
      "logps/ref_rejected": -243.85879516601562,
      "logps/rejected": -278.00360107421875,
      "loss": 4.8217,
      "margin_dpo/margin_mean": 29.130126953125,
      "margin_dpo/margin_std": 43.46732711791992,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -7.115703582763672,
      "KL/mean": -21.52239418029785,
      "KL/rejected_KL_mean": -35.92908477783203,
      "KL/std": 38.69028854370117,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.007564978674054146,
      "fcm_dpo/delta": -0.006849254947155714,
      "fcm_dpo/margin": 28.813379287719727,
      "fcm_dpo/q_t": 0.44742870330810547,
      "grad_norm": 21.363086700439453,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": 1.519167423248291,
      "logits/rejected": 1.5006718635559082,
      "logps/chosen": -271.4024658203125,
      "logps/ref_chosen": -264.2867431640625,
      "logps/ref_rejected": -231.6894989013672,
      "logps/rejected": -267.6185607910156,
      "loss": 4.8691,
      "margin_dpo/margin_mean": 28.813379287719727,
      "margin_dpo/margin_std": 49.108612060546875,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -7.224494934082031,
      "KL/mean": -18.505355834960938,
      "KL/rejected_KL_mean": -29.786218643188477,
      "KL/std": 36.8636474609375,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.007550873328000307,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.56171226501465,
      "fcm_dpo/q_t": 0.4585086405277252,
      "grad_norm": 26.808414459228516,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": 1.7054312229156494,
      "logits/rejected": 1.698226809501648,
      "logps/chosen": -278.5086669921875,
      "logps/ref_chosen": -271.2842102050781,
      "logps/ref_rejected": -247.95462036132812,
      "logps/rejected": -277.7408447265625,
      "loss": 4.9809,
      "margin_dpo/margin_mean": 22.56171417236328,
      "margin_dpo/margin_std": 37.829463958740234,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -4.949108123779297,
      "KL/mean": -21.54059600830078,
      "KL/rejected_KL_mean": -38.13208770751953,
      "KL/std": 38.21521759033203,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.007524519227445126,
      "fcm_dpo/delta": -0.004370654467493296,
      "fcm_dpo/margin": 33.1829948425293,
      "fcm_dpo/q_t": 0.4396628141403198,
      "grad_norm": 23.20747184753418,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": 1.5162684917449951,
      "logits/rejected": 1.4890050888061523,
      "logps/chosen": -315.07952880859375,
      "logps/ref_chosen": -310.13043212890625,
      "logps/ref_rejected": -267.3905334472656,
      "logps/rejected": -305.5225830078125,
      "loss": 4.7254,
      "margin_dpo/margin_mean": 33.1829948425293,
      "margin_dpo/margin_std": 45.3623046875,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -2.4428176879882812,
      "KL/mean": -18.19799041748047,
      "KL/rejected_KL_mean": -33.953155517578125,
      "KL/std": 38.190223693847656,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.007505797781050205,
      "fcm_dpo/delta": -0.012502540834248066,
      "fcm_dpo/margin": 31.510343551635742,
      "fcm_dpo/q_t": 0.44278115034103394,
      "grad_norm": 20.580886840820312,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 1.475374460220337,
      "logits/rejected": 1.4744048118591309,
      "logps/chosen": -279.6034851074219,
      "logps/ref_chosen": -277.16064453125,
      "logps/ref_rejected": -292.91815185546875,
      "logps/rejected": -326.871337890625,
      "loss": 4.7863,
      "margin_dpo/margin_mean": 31.510343551635742,
      "margin_dpo/margin_std": 46.046226501464844,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -2.1376590728759766,
      "KL/mean": -16.975322723388672,
      "KL/rejected_KL_mean": -31.812990188598633,
      "KL/std": 37.96232223510742,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.007367079611867666,
      "fcm_dpo/delta": -0.017158741131424904,
      "fcm_dpo/margin": 29.675323486328125,
      "fcm_dpo/q_t": 0.4467848837375641,
      "grad_norm": 21.148208618164062,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": 1.4753514528274536,
      "logits/rejected": 1.5842045545578003,
      "logps/chosen": -288.9202575683594,
      "logps/ref_chosen": -286.7825927734375,
      "logps/ref_rejected": -289.08270263671875,
      "logps/rejected": -320.89569091796875,
      "loss": 4.8313,
      "margin_dpo/margin_mean": 29.675323486328125,
      "margin_dpo/margin_std": 44.045982360839844,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -6.676973342895508,
      "KL/mean": -19.416250228881836,
      "KL/rejected_KL_mean": -32.15553283691406,
      "KL/std": 41.20140075683594,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.0073263756930828094,
      "fcm_dpo/delta": -0.004930622410029173,
      "fcm_dpo/margin": 25.478565216064453,
      "fcm_dpo/q_t": 0.45514976978302,
      "grad_norm": 23.370859146118164,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": 1.3772979974746704,
      "logits/rejected": 1.4831854104995728,
      "logps/chosen": -273.2091979980469,
      "logps/ref_chosen": -266.5322570800781,
      "logps/ref_rejected": -275.20672607421875,
      "logps/rejected": -307.3622741699219,
      "loss": 4.9936,
      "margin_dpo/margin_mean": 25.478565216064453,
      "margin_dpo/margin_std": 53.219600677490234,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -5.503427505493164,
      "KL/mean": -20.244443893432617,
      "KL/rejected_KL_mean": -34.98546600341797,
      "KL/std": 40.30657196044922,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.007285946048796177,
      "fcm_dpo/delta": -0.00968192983418703,
      "fcm_dpo/margin": 29.482038497924805,
      "fcm_dpo/q_t": 0.44801706075668335,
      "grad_norm": 31.833921432495117,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": 1.7134777307510376,
      "logits/rejected": 1.7242817878723145,
      "logps/chosen": -278.3047790527344,
      "logps/ref_chosen": -272.80133056640625,
      "logps/ref_rejected": -288.4167785644531,
      "logps/rejected": -323.4022521972656,
      "loss": 4.8537,
      "margin_dpo/margin_mean": 29.482038497924805,
      "margin_dpo/margin_std": 46.23191833496094,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -11.791582107543945,
      "KL/mean": -23.793174743652344,
      "KL/rejected_KL_mean": -35.79478073120117,
      "KL/std": 40.42003631591797,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.007234956603497267,
      "fcm_dpo/delta": -0.007655359338968992,
      "fcm_dpo/margin": 24.003206253051758,
      "fcm_dpo/q_t": 0.4581032693386078,
      "grad_norm": 23.961259841918945,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": 1.6640757322311401,
      "logits/rejected": 1.5408504009246826,
      "logps/chosen": -322.47100830078125,
      "logps/ref_chosen": -310.679443359375,
      "logps/ref_rejected": -259.3792419433594,
      "logps/rejected": -295.17401123046875,
      "loss": 5.0088,
      "margin_dpo/margin_mean": 24.003206253051758,
      "margin_dpo/margin_std": 46.57493591308594,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -5.952775955200195,
      "KL/mean": -21.351032257080078,
      "KL/rejected_KL_mean": -36.74929428100586,
      "KL/std": 41.47489547729492,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.007138494402170181,
      "fcm_dpo/delta": -0.018570544198155403,
      "fcm_dpo/margin": 30.796512603759766,
      "fcm_dpo/q_t": 0.4466511011123657,
      "grad_norm": 22.066524505615234,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 1.4228332042694092,
      "logits/rejected": 1.46645188331604,
      "logps/chosen": -280.9209289550781,
      "logps/ref_chosen": -274.9681701660156,
      "logps/ref_rejected": -283.346923828125,
      "logps/rejected": -320.09625244140625,
      "loss": 4.8428,
      "margin_dpo/margin_mean": 30.7965087890625,
      "margin_dpo/margin_std": 48.242340087890625,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -2.4511280059814453,
      "KL/mean": -22.217527389526367,
      "KL/rejected_KL_mean": -41.983924865722656,
      "KL/std": 41.03608322143555,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.007026024628430605,
      "fcm_dpo/delta": -0.030917393043637276,
      "fcm_dpo/margin": 39.53279113769531,
      "fcm_dpo/q_t": 0.432537317276001,
      "grad_norm": 22.979124069213867,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.5288876295089722,
      "logits/rejected": 1.5303130149841309,
      "logps/chosen": -268.730712890625,
      "logps/ref_chosen": -266.2795715332031,
      "logps/ref_rejected": -249.7120819091797,
      "logps/rejected": -291.6960144042969,
      "loss": 4.6144,
      "margin_dpo/margin_mean": 39.53279113769531,
      "margin_dpo/margin_std": 43.33306121826172,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -6.355155944824219,
      "KL/mean": -21.053218841552734,
      "KL/rejected_KL_mean": -35.75128173828125,
      "KL/std": 40.66847229003906,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.0069381361827254295,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.3961181640625,
      "fcm_dpo/q_t": 0.4503456652164459,
      "grad_norm": 46.38639450073242,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": 1.3768949508666992,
      "logits/rejected": 1.4181287288665771,
      "logps/chosen": -271.10797119140625,
      "logps/ref_chosen": -264.7528076171875,
      "logps/ref_rejected": -241.02842712402344,
      "logps/rejected": -276.77972412109375,
      "loss": 4.8927,
      "margin_dpo/margin_mean": 29.3961181640625,
      "margin_dpo/margin_std": 48.89656066894531,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -11.253694534301758,
      "KL/mean": -25.430131912231445,
      "KL/rejected_KL_mean": -39.60657501220703,
      "KL/std": 41.475341796875,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.006939054466784,
      "fcm_dpo/delta": 0.0006614904850721359,
      "fcm_dpo/margin": 28.352882385253906,
      "fcm_dpo/q_t": 0.45271652936935425,
      "grad_norm": 26.31064224243164,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": 1.4263278245925903,
      "logits/rejected": 1.54822838306427,
      "logps/chosen": -305.3923034667969,
      "logps/ref_chosen": -294.13861083984375,
      "logps/ref_rejected": -303.11669921875,
      "logps/rejected": -342.7232971191406,
      "loss": 4.9155,
      "margin_dpo/margin_mean": 28.352882385253906,
      "margin_dpo/margin_std": 48.291927337646484,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -21.486934661865234,
      "KL/mean": -34.478458404541016,
      "KL/rejected_KL_mean": -47.4699821472168,
      "KL/std": 39.824058532714844,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.00693837646394968,
      "fcm_dpo/delta": -0.004954638425260782,
      "fcm_dpo/margin": 25.9830379486084,
      "fcm_dpo/q_t": 0.4563639760017395,
      "grad_norm": 29.38641357421875,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": 1.595127820968628,
      "logits/rejected": 1.7498384714126587,
      "logps/chosen": -288.0636901855469,
      "logps/ref_chosen": -266.5767517089844,
      "logps/ref_rejected": -291.2264709472656,
      "logps/rejected": -338.6964416503906,
      "loss": 4.9701,
      "margin_dpo/margin_mean": 25.9830379486084,
      "margin_dpo/margin_std": 46.93110275268555,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -15.83352279663086,
      "KL/mean": -29.35395622253418,
      "KL/rejected_KL_mean": -42.87439727783203,
      "KL/std": 44.724021911621094,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.006894505582749844,
      "fcm_dpo/delta": -0.0072785308584570885,
      "fcm_dpo/margin": 27.04085922241211,
      "fcm_dpo/q_t": 0.4551750123500824,
      "grad_norm": 26.059274673461914,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": 1.5595961809158325,
      "logits/rejected": 1.571595311164856,
      "logps/chosen": -300.874267578125,
      "logps/ref_chosen": -285.040771484375,
      "logps/ref_rejected": -258.0254821777344,
      "logps/rejected": -300.89990234375,
      "loss": 4.973,
      "margin_dpo/margin_mean": 27.040861129760742,
      "margin_dpo/margin_std": 52.56559371948242,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -16.333953857421875,
      "KL/mean": -31.913618087768555,
      "KL/rejected_KL_mean": -47.4932861328125,
      "KL/std": 45.794639587402344,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.006855614949017763,
      "fcm_dpo/delta": -0.005035098642110825,
      "fcm_dpo/margin": 31.159334182739258,
      "fcm_dpo/q_t": 0.44818347692489624,
      "grad_norm": 30.4041690826416,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": 1.5893064737319946,
      "logits/rejected": 1.6082733869552612,
      "logps/chosen": -349.0853271484375,
      "logps/ref_chosen": -332.7513732910156,
      "logps/ref_rejected": -302.9167785644531,
      "logps/rejected": -350.4100341796875,
      "loss": 4.892,
      "margin_dpo/margin_mean": 31.159337997436523,
      "margin_dpo/margin_std": 55.64927673339844,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -17.419002532958984,
      "KL/mean": -38.159366607666016,
      "KL/rejected_KL_mean": -58.899723052978516,
      "KL/std": 44.92721939086914,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.006788069382309914,
      "fcm_dpo/delta": -0.016210366040468216,
      "fcm_dpo/margin": 41.4807243347168,
      "fcm_dpo/q_t": 0.43247485160827637,
      "grad_norm": 22.87995147705078,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": 1.5631740093231201,
      "logits/rejected": 1.5671734809875488,
      "logps/chosen": -303.886962890625,
      "logps/ref_chosen": -286.46795654296875,
      "logps/ref_rejected": -253.99298095703125,
      "logps/rejected": -312.8927307128906,
      "loss": 4.6379,
      "margin_dpo/margin_mean": 41.4807243347168,
      "margin_dpo/margin_std": 54.90333557128906,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -13.912851333618164,
      "KL/mean": -30.289539337158203,
      "KL/rejected_KL_mean": -46.666236877441406,
      "KL/std": 44.88377380371094,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.006740156561136246,
      "fcm_dpo/delta": -0.007797584403306246,
      "fcm_dpo/margin": 32.75338363647461,
      "fcm_dpo/q_t": 0.44646579027175903,
      "grad_norm": 25.346580505371094,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 1.5309090614318848,
      "logits/rejected": 1.5880589485168457,
      "logps/chosen": -325.2914123535156,
      "logps/ref_chosen": -311.3785400390625,
      "logps/ref_rejected": -290.3092956542969,
      "logps/rejected": -336.97552490234375,
      "loss": 4.8249,
      "margin_dpo/margin_mean": 32.753379821777344,
      "margin_dpo/margin_std": 49.0080680847168,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -13.844461441040039,
      "KL/mean": -30.365615844726562,
      "KL/rejected_KL_mean": -46.88676452636719,
      "KL/std": 43.221805572509766,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.006678058300167322,
      "fcm_dpo/delta": -0.010624059475958347,
      "fcm_dpo/margin": 33.04231262207031,
      "fcm_dpo/q_t": 0.44639432430267334,
      "grad_norm": 24.73733901977539,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": 1.2813258171081543,
      "logits/rejected": 1.3411719799041748,
      "logps/chosen": -273.1773986816406,
      "logps/ref_chosen": -259.33294677734375,
      "logps/ref_rejected": -261.7661437988281,
      "logps/rejected": -308.65289306640625,
      "loss": 4.8359,
      "margin_dpo/margin_mean": 33.04231262207031,
      "margin_dpo/margin_std": 51.55025100708008,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -15.776079177856445,
      "KL/mean": -33.864540100097656,
      "KL/rejected_KL_mean": -51.95301055908203,
      "KL/std": 45.68415832519531,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.006659484934061766,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 36.17692947387695,
      "fcm_dpo/q_t": 0.4417183995246887,
      "grad_norm": 23.191158294677734,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 1.4746791124343872,
      "logits/rejected": 1.3971139192581177,
      "logps/chosen": -321.9804382324219,
      "logps/ref_chosen": -306.2043762207031,
      "logps/ref_rejected": -256.7926330566406,
      "logps/rejected": -308.7456359863281,
      "loss": 4.7718,
      "margin_dpo/margin_mean": 36.17693328857422,
      "margin_dpo/margin_std": 54.384552001953125,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -17.602924346923828,
      "KL/mean": -31.619911193847656,
      "KL/rejected_KL_mean": -45.636905670166016,
      "KL/std": 45.704620361328125,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.006646092981100082,
      "fcm_dpo/delta": -0.010095274075865746,
      "fcm_dpo/margin": 28.033981323242188,
      "fcm_dpo/q_t": 0.45468056201934814,
      "grad_norm": 21.323795318603516,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 1.2537236213684082,
      "logits/rejected": 1.3744418621063232,
      "logps/chosen": -310.93487548828125,
      "logps/ref_chosen": -293.3319396972656,
      "logps/ref_rejected": -282.7599182128906,
      "logps/rejected": -328.3968505859375,
      "loss": 4.9629,
      "margin_dpo/margin_mean": 28.033981323242188,
      "margin_dpo/margin_std": 51.64204406738281,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -13.29167366027832,
      "KL/mean": -31.25895881652832,
      "KL/rejected_KL_mean": -49.22623062133789,
      "KL/std": 44.00951385498047,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.0065589649602770805,
      "fcm_dpo/delta": -0.012526572681963444,
      "fcm_dpo/margin": 35.93455505371094,
      "fcm_dpo/q_t": 0.44276848435401917,
      "grad_norm": 30.335094451904297,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": 1.5930958986282349,
      "logits/rejected": 1.5219813585281372,
      "logps/chosen": -300.2978515625,
      "logps/ref_chosen": -287.0061340332031,
      "logps/ref_rejected": -233.23130798339844,
      "logps/rejected": -282.4575500488281,
      "loss": 4.7678,
      "margin_dpo/margin_mean": 35.93455505371094,
      "margin_dpo/margin_std": 49.12702178955078,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -15.622907638549805,
      "KL/mean": -35.50263214111328,
      "KL/rejected_KL_mean": -55.382354736328125,
      "KL/std": 47.70505905151367,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.00647671427577734,
      "fcm_dpo/delta": -0.015679936856031418,
      "fcm_dpo/margin": 39.75946044921875,
      "fcm_dpo/q_t": 0.43771272897720337,
      "grad_norm": 20.584640502929688,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": 1.2599544525146484,
      "logits/rejected": 1.1481380462646484,
      "logps/chosen": -291.4382019042969,
      "logps/ref_chosen": -275.8153076171875,
      "logps/ref_rejected": -238.54286193847656,
      "logps/rejected": -293.9252014160156,
      "loss": 4.7161,
      "margin_dpo/margin_mean": 39.759456634521484,
      "margin_dpo/margin_std": 55.50542068481445,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -15.428489685058594,
      "KL/mean": -35.80807876586914,
      "KL/rejected_KL_mean": -56.18767547607422,
      "KL/std": 46.21131134033203,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.006439714226871729,
      "fcm_dpo/delta": -0.021290991455316544,
      "fcm_dpo/margin": 40.759185791015625,
      "fcm_dpo/q_t": 0.4363047480583191,
      "grad_norm": 31.15467071533203,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": 1.427111268043518,
      "logits/rejected": 1.4988987445831299,
      "logps/chosen": -254.52886962890625,
      "logps/ref_chosen": -239.1003875732422,
      "logps/ref_rejected": -230.31900024414062,
      "logps/rejected": -286.50665283203125,
      "loss": 4.6799,
      "margin_dpo/margin_mean": 40.759185791015625,
      "margin_dpo/margin_std": 50.552947998046875,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -27.02050018310547,
      "KL/mean": -40.923126220703125,
      "KL/rejected_KL_mean": -54.82576370239258,
      "KL/std": 47.11695098876953,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.0063389018177986145,
      "fcm_dpo/delta": -0.0020778256002813578,
      "fcm_dpo/margin": 27.80525779724121,
      "fcm_dpo/q_t": 0.4576857388019562,
      "grad_norm": 28.944660186767578,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": 1.2866885662078857,
      "logits/rejected": 1.306793451309204,
      "logps/chosen": -327.3498840332031,
      "logps/ref_chosen": -300.3293762207031,
      "logps/ref_rejected": -332.132568359375,
      "logps/rejected": -386.9583435058594,
      "loss": 5.0285,
      "margin_dpo/margin_mean": 27.80525779724121,
      "margin_dpo/margin_std": 61.599735260009766,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -20.779296875,
      "KL/mean": -34.89369201660156,
      "KL/rejected_KL_mean": -49.008079528808594,
      "KL/std": 45.75469970703125,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.0063389018177986145,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.228778839111328,
      "fcm_dpo/q_t": 0.45669010281562805,
      "grad_norm": 32.95003128051758,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": 1.3824142217636108,
      "logits/rejected": 1.4477512836456299,
      "logps/chosen": -266.19146728515625,
      "logps/ref_chosen": -245.41213989257812,
      "logps/ref_rejected": -251.72999572753906,
      "logps/rejected": -300.73809814453125,
      "loss": 4.9817,
      "margin_dpo/margin_mean": 28.228782653808594,
      "margin_dpo/margin_std": 53.72325897216797,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -20.61986541748047,
      "KL/mean": -40.518943786621094,
      "KL/rejected_KL_mean": -60.41801071166992,
      "KL/std": 48.1357536315918,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.006294009275734425,
      "fcm_dpo/delta": -0.010339262895286083,
      "fcm_dpo/margin": 39.79815673828125,
      "fcm_dpo/q_t": 0.43934309482574463,
      "grad_norm": 23.352819442749023,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": 1.5281569957733154,
      "logits/rejected": 1.5819910764694214,
      "logps/chosen": -297.49566650390625,
      "logps/ref_chosen": -276.87579345703125,
      "logps/ref_rejected": -259.0455017089844,
      "logps/rejected": -319.4635009765625,
      "loss": 4.7349,
      "margin_dpo/margin_mean": 39.79815673828125,
      "margin_dpo/margin_std": 55.89205551147461,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -17.969255447387695,
      "KL/mean": -38.4671630859375,
      "KL/rejected_KL_mean": -58.965065002441406,
      "KL/std": 47.44749069213867,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.006264185532927513,
      "fcm_dpo/delta": -0.018955906853079796,
      "fcm_dpo/margin": 40.995811462402344,
      "fcm_dpo/q_t": 0.4378714859485626,
      "grad_norm": 30.164653778076172,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": 1.4733021259307861,
      "logits/rejected": 1.4698607921600342,
      "logps/chosen": -321.9402770996094,
      "logps/ref_chosen": -303.9710693359375,
      "logps/ref_rejected": -255.62033081054688,
      "logps/rejected": -314.5854187011719,
      "loss": 4.7077,
      "margin_dpo/margin_mean": 40.995811462402344,
      "margin_dpo/margin_std": 54.84269332885742,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -18.99317169189453,
      "KL/mean": -44.51826858520508,
      "KL/rejected_KL_mean": -70.04336547851562,
      "KL/std": 49.864437103271484,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.006055726204067469,
      "fcm_dpo/delta": -0.04315026104450226,
      "fcm_dpo/margin": 51.050209045410156,
      "fcm_dpo/q_t": 0.4250105023384094,
      "grad_norm": 32.363014221191406,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": 1.405127763748169,
      "logits/rejected": 1.404304027557373,
      "logps/chosen": -300.6890869140625,
      "logps/ref_chosen": -281.6959533691406,
      "logps/ref_rejected": -258.93353271484375,
      "logps/rejected": -328.9768981933594,
      "loss": 4.5304,
      "margin_dpo/margin_mean": 51.050209045410156,
      "margin_dpo/margin_std": 56.5584602355957,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -19.8351993560791,
      "KL/mean": -39.123294830322266,
      "KL/rejected_KL_mean": -58.41139221191406,
      "KL/std": 48.12479019165039,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.0059678805992007256,
      "fcm_dpo/delta": -0.005540241952985525,
      "fcm_dpo/margin": 38.576175689697266,
      "fcm_dpo/q_t": 0.44414374232292175,
      "grad_norm": 33.902530670166016,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 1.2827942371368408,
      "logits/rejected": 1.3769925832748413,
      "logps/chosen": -243.791748046875,
      "logps/ref_chosen": -223.95655822753906,
      "logps/ref_rejected": -238.95187377929688,
      "logps/rejected": -297.3632507324219,
      "loss": 4.8025,
      "margin_dpo/margin_mean": 38.576175689697266,
      "margin_dpo/margin_std": 57.882598876953125,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -28.286428451538086,
      "KL/mean": -43.748836517333984,
      "KL/rejected_KL_mean": -59.21125030517578,
      "KL/std": 47.58973693847656,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.005955680273473263,
      "fcm_dpo/delta": 5.6711611250648275e-05,
      "fcm_dpo/margin": 30.924816131591797,
      "fcm_dpo/q_t": 0.4551432728767395,
      "grad_norm": 26.73813819885254,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 1.3211333751678467,
      "logits/rejected": 1.387269377708435,
      "logps/chosen": -343.2110290527344,
      "logps/ref_chosen": -314.92462158203125,
      "logps/ref_rejected": -279.6347351074219,
      "logps/rejected": -338.8459777832031,
      "loss": 4.9459,
      "margin_dpo/margin_mean": 30.924819946289062,
      "margin_dpo/margin_std": 52.34429168701172,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -17.190954208374023,
      "KL/mean": -35.5333366394043,
      "KL/rejected_KL_mean": -53.87571716308594,
      "KL/std": 50.35641098022461,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.005933224689215422,
      "fcm_dpo/delta": -0.0047430298291146755,
      "fcm_dpo/margin": 36.68476486206055,
      "fcm_dpo/q_t": 0.4473300576210022,
      "grad_norm": 18.70887565612793,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": 1.3253000974655151,
      "logits/rejected": 1.3900870084762573,
      "logps/chosen": -280.51055908203125,
      "logps/ref_chosen": -263.3196105957031,
      "logps/ref_rejected": -247.13612365722656,
      "logps/rejected": -301.0118408203125,
      "loss": 4.8379,
      "margin_dpo/margin_mean": 36.68476486206055,
      "margin_dpo/margin_std": 55.60708236694336,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -17.79711151123047,
      "KL/mean": -36.819671630859375,
      "KL/rejected_KL_mean": -55.842227935791016,
      "KL/std": 48.893150329589844,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.005931841675192118,
      "fcm_dpo/delta": -0.000300525949569419,
      "fcm_dpo/margin": 38.04512023925781,
      "fcm_dpo/q_t": 0.44547054171562195,
      "grad_norm": 41.95193862915039,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": 1.4256671667099,
      "logits/rejected": 1.5037790536880493,
      "logps/chosen": -297.15936279296875,
      "logps/ref_chosen": -279.36224365234375,
      "logps/ref_rejected": -264.46112060546875,
      "logps/rejected": -320.3033447265625,
      "loss": 4.8089,
      "margin_dpo/margin_mean": 38.04512023925781,
      "margin_dpo/margin_std": 56.3094367980957,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -15.290029525756836,
      "KL/mean": -35.22977828979492,
      "KL/rejected_KL_mean": -55.16952133178711,
      "KL/std": 51.99163818359375,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.005923133343458176,
      "fcm_dpo/delta": -0.00339376088231802,
      "fcm_dpo/margin": 39.879486083984375,
      "fcm_dpo/q_t": 0.4430829584598541,
      "grad_norm": 27.847883224487305,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": 1.4091912508010864,
      "logits/rejected": 1.494905948638916,
      "logps/chosen": -301.7391357421875,
      "logps/ref_chosen": -286.4491271972656,
      "logps/ref_rejected": -294.5469665527344,
      "logps/rejected": -349.71649169921875,
      "loss": 4.8143,
      "margin_dpo/margin_mean": 39.879486083984375,
      "margin_dpo/margin_std": 64.82295227050781,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -15.785848617553711,
      "KL/mean": -34.47373962402344,
      "KL/rejected_KL_mean": -53.161643981933594,
      "KL/std": 52.24916458129883,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.0059035648591816425,
      "fcm_dpo/delta": -0.013154011219739914,
      "fcm_dpo/margin": 37.375789642333984,
      "fcm_dpo/q_t": 0.44654592871665955,
      "grad_norm": 29.40359115600586,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": 1.343457818031311,
      "logits/rejected": 1.3375635147094727,
      "logps/chosen": -298.1459045410156,
      "logps/ref_chosen": -282.36004638671875,
      "logps/ref_rejected": -243.79052734375,
      "logps/rejected": -296.9521789550781,
      "loss": 4.8536,
      "margin_dpo/margin_mean": 37.375789642333984,
      "margin_dpo/margin_std": 62.480804443359375,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -8.098560333251953,
      "KL/mean": -29.460729598999023,
      "KL/rejected_KL_mean": -50.822898864746094,
      "KL/std": 49.86714553833008,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.005831598769873381,
      "fcm_dpo/delta": -0.009792954660952091,
      "fcm_dpo/margin": 42.72433090209961,
      "fcm_dpo/q_t": 0.43949559330940247,
      "grad_norm": 17.708133697509766,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": 1.2868651151657104,
      "logits/rejected": 1.2997386455535889,
      "logps/chosen": -287.23797607421875,
      "logps/ref_chosen": -279.139404296875,
      "logps/ref_rejected": -240.5850067138672,
      "logps/rejected": -291.4079284667969,
      "loss": 4.7275,
      "margin_dpo/margin_mean": 42.72432327270508,
      "margin_dpo/margin_std": 57.29695510864258,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -11.607168197631836,
      "KL/mean": -29.622222900390625,
      "KL/rejected_KL_mean": -47.63728332519531,
      "KL/std": 46.839500427246094,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.005805491004139185,
      "fcm_dpo/delta": -0.0022551105357706547,
      "fcm_dpo/margin": 36.030120849609375,
      "fcm_dpo/q_t": 0.44947537779808044,
      "grad_norm": 20.612361907958984,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": 1.421472191810608,
      "logits/rejected": 1.5023185014724731,
      "logps/chosen": -270.4186706542969,
      "logps/ref_chosen": -258.8114929199219,
      "logps/ref_rejected": -266.3898010253906,
      "logps/rejected": -314.0270690917969,
      "loss": 4.8779,
      "margin_dpo/margin_mean": 36.030120849609375,
      "margin_dpo/margin_std": 59.41435241699219,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -12.212844848632812,
      "KL/mean": -32.20032501220703,
      "KL/rejected_KL_mean": -52.18780517578125,
      "KL/std": 51.616214752197266,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.005777006037533283,
      "fcm_dpo/delta": -0.005956320092082024,
      "fcm_dpo/margin": 39.97496032714844,
      "fcm_dpo/q_t": 0.44440504908561707,
      "grad_norm": 29.856430053710938,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": 1.3500447273254395,
      "logits/rejected": 1.2798025608062744,
      "logps/chosen": -314.27899169921875,
      "logps/ref_chosen": -302.0661926269531,
      "logps/ref_rejected": -265.5518798828125,
      "logps/rejected": -317.73968505859375,
      "loss": 4.8087,
      "margin_dpo/margin_mean": 39.97496032714844,
      "margin_dpo/margin_std": 61.058292388916016,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -7.640283584594727,
      "KL/mean": -24.7486629486084,
      "KL/rejected_KL_mean": -41.85704803466797,
      "KL/std": 49.51959228515625,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.005770079325884581,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.21677017211914,
      "fcm_dpo/q_t": 0.4524189233779907,
      "grad_norm": 23.82918357849121,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": 1.4158596992492676,
      "logits/rejected": 1.5180094242095947,
      "logps/chosen": -266.6386413574219,
      "logps/ref_chosen": -258.99835205078125,
      "logps/ref_rejected": -273.7834777832031,
      "logps/rejected": -315.6405029296875,
      "loss": 4.9293,
      "margin_dpo/margin_mean": 34.216766357421875,
      "margin_dpo/margin_std": 63.23017501831055,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -4.72981071472168,
      "KL/mean": -26.458637237548828,
      "KL/rejected_KL_mean": -48.18747329711914,
      "KL/std": 49.00332260131836,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.005733635276556015,
      "fcm_dpo/delta": -0.007919962517917156,
      "fcm_dpo/margin": 43.45765686035156,
      "fcm_dpo/q_t": 0.43962374329566956,
      "grad_norm": 33.54020690917969,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 1.3982484340667725,
      "logits/rejected": 1.4985620975494385,
      "logps/chosen": -280.25286865234375,
      "logps/ref_chosen": -275.5230407714844,
      "logps/ref_rejected": -255.50784301757812,
      "logps/rejected": -303.6953125,
      "loss": 4.7225,
      "margin_dpo/margin_mean": 43.45765686035156,
      "margin_dpo/margin_std": 57.205360412597656,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -11.40134048461914,
      "KL/mean": -28.270343780517578,
      "KL/rejected_KL_mean": -45.139339447021484,
      "KL/std": 50.00520706176758,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.0057302797213196754,
      "fcm_dpo/delta": -0.0009758807718753815,
      "fcm_dpo/margin": 33.738006591796875,
      "fcm_dpo/q_t": 0.45337939262390137,
      "grad_norm": 27.756332397460938,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.5557098388671875,
      "logits/rejected": 1.609498381614685,
      "logps/chosen": -264.76483154296875,
      "logps/ref_chosen": -253.36349487304688,
      "logps/ref_rejected": -247.1916961669922,
      "logps/rejected": -292.3310546875,
      "loss": 4.9375,
      "margin_dpo/margin_mean": 33.738006591796875,
      "margin_dpo/margin_std": 60.77975845336914,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -8.794973373413086,
      "KL/mean": -28.885425567626953,
      "KL/rejected_KL_mean": -48.97587585449219,
      "KL/std": 49.459877014160156,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.005693924147635698,
      "fcm_dpo/delta": -0.007434740196913481,
      "fcm_dpo/margin": 40.1808967590332,
      "fcm_dpo/q_t": 0.44428327679634094,
      "grad_norm": 23.537521362304688,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": 1.4914366006851196,
      "logits/rejected": 1.4436402320861816,
      "logps/chosen": -273.7830505371094,
      "logps/ref_chosen": -264.9880676269531,
      "logps/ref_rejected": -243.07838439941406,
      "logps/rejected": -292.05426025390625,
      "loss": 4.7891,
      "margin_dpo/margin_mean": 40.1808967590332,
      "margin_dpo/margin_std": 56.223262786865234,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -3.733377456665039,
      "KL/mean": -26.795364379882812,
      "KL/rejected_KL_mean": -49.85735321044922,
      "KL/std": 49.22810363769531,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.005657509434968233,
      "fcm_dpo/delta": -0.02789243869483471,
      "fcm_dpo/margin": 46.12397384643555,
      "fcm_dpo/q_t": 0.4367283880710602,
      "grad_norm": 43.456268310546875,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": 1.4066061973571777,
      "logits/rejected": 1.4677698612213135,
      "logps/chosen": -319.60406494140625,
      "logps/ref_chosen": -315.8706970214844,
      "logps/ref_rejected": -304.3729248046875,
      "logps/rejected": -354.23028564453125,
      "loss": 4.6769,
      "margin_dpo/margin_mean": 46.12397384643555,
      "margin_dpo/margin_std": 53.863033294677734,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -8.17613410949707,
      "KL/mean": -31.878700256347656,
      "KL/rejected_KL_mean": -55.58129119873047,
      "KL/std": 52.06996154785156,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.005510547664016485,
      "fcm_dpo/delta": -0.019652821123600006,
      "fcm_dpo/margin": 47.405155181884766,
      "fcm_dpo/q_t": 0.4366835057735443,
      "grad_norm": 18.35883903503418,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 1.6468145847320557,
      "logits/rejected": 1.6163761615753174,
      "logps/chosen": -319.2733154296875,
      "logps/ref_chosen": -311.09716796875,
      "logps/ref_rejected": -266.01959228515625,
      "logps/rejected": -321.6008605957031,
      "loss": 4.6878,
      "margin_dpo/margin_mean": 47.405155181884766,
      "margin_dpo/margin_std": 60.39481735229492,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -8.778343200683594,
      "KL/mean": -32.535858154296875,
      "KL/rejected_KL_mean": -56.29338073730469,
      "KL/std": 51.80685043334961,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.00545636098831892,
      "fcm_dpo/delta": -0.016637688502669334,
      "fcm_dpo/margin": 47.51504135131836,
      "fcm_dpo/q_t": 0.43707552552223206,
      "grad_norm": 43.15507125854492,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": 1.474489450454712,
      "logits/rejected": 1.3690487146377563,
      "logps/chosen": -293.518798828125,
      "logps/ref_chosen": -284.740478515625,
      "logps/ref_rejected": -237.64117431640625,
      "logps/rejected": -293.9345397949219,
      "loss": 4.6953,
      "margin_dpo/margin_mean": 47.51504135131836,
      "margin_dpo/margin_std": 60.37356185913086,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -20.450580596923828,
      "KL/mean": -36.13612365722656,
      "KL/rejected_KL_mean": -51.82166290283203,
      "KL/std": 50.98601531982422,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.0054036458022892475,
      "fcm_dpo/delta": -0.0030045025050640106,
      "fcm_dpo/margin": 31.371070861816406,
      "fcm_dpo/q_t": 0.4591512084007263,
      "grad_norm": 41.21300506591797,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": 1.583566427230835,
      "logits/rejected": 1.6231988668441772,
      "logps/chosen": -293.0634765625,
      "logps/ref_chosen": -272.6128845214844,
      "logps/ref_rejected": -249.99725341796875,
      "logps/rejected": -301.8189392089844,
      "loss": 5.0213,
      "margin_dpo/margin_mean": 31.371070861816406,
      "margin_dpo/margin_std": 62.0515022277832,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -10.791646957397461,
      "KL/mean": -31.646896362304688,
      "KL/rejected_KL_mean": -52.50214767456055,
      "KL/std": 52.13120651245117,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.005395128391683102,
      "fcm_dpo/delta": -0.003642259631305933,
      "fcm_dpo/margin": 41.71050262451172,
      "fcm_dpo/q_t": 0.44542863965034485,
      "grad_norm": 34.43070983886719,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": 1.5228512287139893,
      "logits/rejected": 1.545363187789917,
      "logps/chosen": -285.2274475097656,
      "logps/ref_chosen": -274.435791015625,
      "logps/ref_rejected": -257.8125305175781,
      "logps/rejected": -310.3146667480469,
      "loss": 4.8035,
      "margin_dpo/margin_mean": 41.71050262451172,
      "margin_dpo/margin_std": 59.612083435058594,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -16.94788932800293,
      "KL/mean": -33.09451675415039,
      "KL/rejected_KL_mean": -49.24114990234375,
      "KL/std": 50.79631805419922,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.005359522067010403,
      "fcm_dpo/delta": -0.010227501392364502,
      "fcm_dpo/margin": 32.29326629638672,
      "fcm_dpo/q_t": 0.45795658230781555,
      "grad_norm": 28.7509765625,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": 1.2618658542633057,
      "logits/rejected": 1.2993965148925781,
      "logps/chosen": -278.25921630859375,
      "logps/ref_chosen": -261.3113098144531,
      "logps/ref_rejected": -231.152587890625,
      "logps/rejected": -280.39373779296875,
      "loss": 5.0021,
      "margin_dpo/margin_mean": 32.29326629638672,
      "margin_dpo/margin_std": 62.0631217956543,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -18.70046043395996,
      "KL/mean": -38.693565368652344,
      "KL/rejected_KL_mean": -58.686683654785156,
      "KL/std": 54.45020294189453,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.005325684789568186,
      "fcm_dpo/delta": -0.004471386782824993,
      "fcm_dpo/margin": 39.98623275756836,
      "fcm_dpo/q_t": 0.44843408465385437,
      "grad_norm": 44.733154296875,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": 1.145013451576233,
      "logits/rejected": 1.0835671424865723,
      "logps/chosen": -287.167724609375,
      "logps/ref_chosen": -268.46722412109375,
      "logps/ref_rejected": -240.27597045898438,
      "logps/rejected": -298.962646484375,
      "loss": 4.8561,
      "margin_dpo/margin_mean": 39.98623275756836,
      "margin_dpo/margin_std": 62.08204650878906,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -21.25681495666504,
      "KL/mean": -41.992523193359375,
      "KL/rejected_KL_mean": -62.728233337402344,
      "KL/std": 56.715824127197266,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.005270650144666433,
      "fcm_dpo/delta": -0.012929601594805717,
      "fcm_dpo/margin": 41.4714241027832,
      "fcm_dpo/q_t": 0.4470377266407013,
      "grad_norm": 25.59990882873535,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 1.5841245651245117,
      "logits/rejected": 1.7200686931610107,
      "logps/chosen": -306.362060546875,
      "logps/ref_chosen": -285.1051940917969,
      "logps/ref_rejected": -281.2331848144531,
      "logps/rejected": -343.96142578125,
      "loss": 4.853,
      "margin_dpo/margin_mean": 41.47142028808594,
      "margin_dpo/margin_std": 67.64434051513672,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -18.00657081604004,
      "KL/mean": -38.74930191040039,
      "KL/rejected_KL_mean": -59.492042541503906,
      "KL/std": 54.254005432128906,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.005256249103695154,
      "fcm_dpo/delta": -0.007166672497987747,
      "fcm_dpo/margin": 41.48546600341797,
      "fcm_dpo/q_t": 0.4473690688610077,
      "grad_norm": 34.787437438964844,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 1.3140625953674316,
      "logits/rejected": 1.2779964208602905,
      "logps/chosen": -292.11920166015625,
      "logps/ref_chosen": -274.1126403808594,
      "logps/ref_rejected": -256.7730407714844,
      "logps/rejected": -316.2651062011719,
      "loss": 4.8351,
      "margin_dpo/margin_mean": 41.48546600341797,
      "margin_dpo/margin_std": 61.314964294433594,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -27.64002227783203,
      "KL/mean": -49.92330551147461,
      "KL/rejected_KL_mean": -72.20659637451172,
      "KL/std": 56.884273529052734,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.005141837522387505,
      "fcm_dpo/delta": -0.02213863655924797,
      "fcm_dpo/margin": 44.56657409667969,
      "fcm_dpo/q_t": 0.44443100690841675,
      "grad_norm": 25.04254913330078,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": 1.4576672315597534,
      "logits/rejected": 1.4290910959243774,
      "logps/chosen": -317.20574951171875,
      "logps/ref_chosen": -289.56573486328125,
      "logps/ref_rejected": -277.9891662597656,
      "logps/rejected": -350.1957702636719,
      "loss": 4.8185,
      "margin_dpo/margin_mean": 44.56657409667969,
      "margin_dpo/margin_std": 69.65999603271484,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -23.642297744750977,
      "KL/mean": -41.09128189086914,
      "KL/rejected_KL_mean": -58.54026794433594,
      "KL/std": 56.1010856628418,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.005141837522387505,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.897972106933594,
      "fcm_dpo/q_t": 0.4565576910972595,
      "grad_norm": 26.461227416992188,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": 1.5789790153503418,
      "logits/rejected": 1.5835169553756714,
      "logps/chosen": -293.82623291015625,
      "logps/ref_chosen": -270.1839294433594,
      "logps/ref_rejected": -245.86949157714844,
      "logps/rejected": -304.40972900390625,
      "loss": 4.9798,
      "margin_dpo/margin_mean": 34.897972106933594,
      "margin_dpo/margin_std": 65.26042175292969,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -20.0852108001709,
      "KL/mean": -42.109413146972656,
      "KL/rejected_KL_mean": -64.13360595703125,
      "KL/std": 56.21087646484375,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.005131649784743786,
      "fcm_dpo/delta": -0.019971460103988647,
      "fcm_dpo/margin": 44.04839324951172,
      "fcm_dpo/q_t": 0.4449615776538849,
      "grad_norm": 27.832529067993164,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": 1.38333261013031,
      "logits/rejected": 1.3831310272216797,
      "logps/chosen": -280.7716369628906,
      "logps/ref_chosen": -260.6864318847656,
      "logps/ref_rejected": -245.99508666992188,
      "logps/rejected": -310.1286926269531,
      "loss": 4.7941,
      "margin_dpo/margin_mean": 44.04839324951172,
      "margin_dpo/margin_std": 57.58948516845703,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -18.736474990844727,
      "KL/mean": -42.689109802246094,
      "KL/rejected_KL_mean": -66.64175415039062,
      "KL/std": 56.458091735839844,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.005051720887422562,
      "fcm_dpo/delta": -0.003938022535294294,
      "fcm_dpo/margin": 47.90529251098633,
      "fcm_dpo/q_t": 0.4414996802806854,
      "grad_norm": 32.32161331176758,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": 1.537187099456787,
      "logits/rejected": 1.4527195692062378,
      "logps/chosen": -316.6502990722656,
      "logps/ref_chosen": -297.9138488769531,
      "logps/ref_rejected": -253.39952087402344,
      "logps/rejected": -320.041259765625,
      "loss": 4.766,
      "margin_dpo/margin_mean": 47.90529251098633,
      "margin_dpo/margin_std": 70.53783416748047,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -16.660253524780273,
      "KL/mean": -40.885108947753906,
      "KL/rejected_KL_mean": -65.10997009277344,
      "KL/std": 57.11183547973633,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.005012368783354759,
      "fcm_dpo/delta": -0.024699721485376358,
      "fcm_dpo/margin": 48.44971466064453,
      "fcm_dpo/q_t": 0.441072940826416,
      "grad_norm": 22.287565231323242,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": 1.572829008102417,
      "logits/rejected": 1.568257212638855,
      "logps/chosen": -326.58660888671875,
      "logps/ref_chosen": -309.9263916015625,
      "logps/ref_rejected": -253.452392578125,
      "logps/rejected": -318.5623779296875,
      "loss": 4.7757,
      "margin_dpo/margin_mean": 48.44971466064453,
      "margin_dpo/margin_std": 71.91654205322266,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -22.89225959777832,
      "KL/mean": -44.69786071777344,
      "KL/rejected_KL_mean": -66.50346374511719,
      "KL/std": 58.156864166259766,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.004919106140732765,
      "fcm_dpo/delta": -0.007633425761014223,
      "fcm_dpo/margin": 43.6112060546875,
      "fcm_dpo/q_t": 0.44809433817863464,
      "grad_norm": 39.61440658569336,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 1.166211485862732,
      "logits/rejected": 1.19886314868927,
      "logps/chosen": -282.7059020996094,
      "logps/ref_chosen": -259.8136291503906,
      "logps/ref_rejected": -245.84947204589844,
      "logps/rejected": -312.3529357910156,
      "loss": 4.8454,
      "margin_dpo/margin_mean": 43.6112060546875,
      "margin_dpo/margin_std": 66.2654037475586,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -19.254642486572266,
      "KL/mean": -43.92586135864258,
      "KL/rejected_KL_mean": -68.59708404541016,
      "KL/std": 58.71088790893555,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.0049090259708464146,
      "fcm_dpo/delta": -0.0038778134621679783,
      "fcm_dpo/margin": 49.342437744140625,
      "fcm_dpo/q_t": 0.4409804344177246,
      "grad_norm": 22.32662010192871,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": 1.610338568687439,
      "logits/rejected": 1.7077364921569824,
      "logps/chosen": -284.6083068847656,
      "logps/ref_chosen": -265.35369873046875,
      "logps/ref_rejected": -247.9174346923828,
      "logps/rejected": -316.5145263671875,
      "loss": 4.7302,
      "margin_dpo/margin_mean": 49.34244155883789,
      "margin_dpo/margin_std": 61.70582962036133,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -19.378826141357422,
      "KL/mean": -41.73649597167969,
      "KL/rejected_KL_mean": -64.09414672851562,
      "KL/std": 57.740272521972656,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.004900387022644281,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.7153205871582,
      "fcm_dpo/q_t": 0.4467147886753082,
      "grad_norm": 20.408735275268555,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": 1.4549899101257324,
      "logits/rejected": 1.4599144458770752,
      "logps/chosen": -307.60205078125,
      "logps/ref_chosen": -288.2232360839844,
      "logps/ref_rejected": -270.7300109863281,
      "logps/rejected": -334.8241882324219,
      "loss": 4.8268,
      "margin_dpo/margin_mean": 44.7153205871582,
      "margin_dpo/margin_std": 67.61615753173828,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -24.650798797607422,
      "KL/mean": -34.8856086730957,
      "KL/rejected_KL_mean": -45.12040710449219,
      "KL/std": 55.774139404296875,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.004900387022644281,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 20.4696044921875,
      "fcm_dpo/q_t": 0.47563889622688293,
      "grad_norm": 20.872560501098633,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 1.4797863960266113,
      "logits/rejected": 1.5249698162078857,
      "logps/chosen": -306.2705078125,
      "logps/ref_chosen": -281.61968994140625,
      "logps/ref_rejected": -271.03973388671875,
      "logps/rejected": -316.16015625,
      "loss": 5.2637,
      "margin_dpo/margin_mean": 20.469606399536133,
      "margin_dpo/margin_std": 65.8192367553711,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -30.460447311401367,
      "KL/mean": -49.89323425292969,
      "KL/rejected_KL_mean": -69.3260269165039,
      "KL/std": 57.98685073852539,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.004900387022644281,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.865570068359375,
      "fcm_dpo/q_t": 0.453850120306015,
      "grad_norm": 45.724246978759766,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 1.09812331199646,
      "logits/rejected": 1.2451034784317017,
      "logps/chosen": -272.9236145019531,
      "logps/ref_chosen": -242.4631805419922,
      "logps/ref_rejected": -244.0020751953125,
      "logps/rejected": -313.3280944824219,
      "loss": 4.9491,
      "margin_dpo/margin_mean": 38.865570068359375,
      "margin_dpo/margin_std": 72.50148010253906,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -20.111948013305664,
      "KL/mean": -44.619728088378906,
      "KL/rejected_KL_mean": -69.12750244140625,
      "KL/std": 59.31889343261719,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.00489090196788311,
      "fcm_dpo/delta": -0.0064683947712183,
      "fcm_dpo/margin": 49.01555633544922,
      "fcm_dpo/q_t": 0.4417862892150879,
      "grad_norm": 29.030298233032227,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": 1.4831479787826538,
      "logits/rejected": 1.5133428573608398,
      "logps/chosen": -312.1005554199219,
      "logps/ref_chosen": -291.98858642578125,
      "logps/ref_rejected": -273.35955810546875,
      "logps/rejected": -342.487060546875,
      "loss": 4.7494,
      "margin_dpo/margin_mean": 49.01555633544922,
      "margin_dpo/margin_std": 65.5273666381836,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -29.34044647216797,
      "KL/mean": -49.22222900390625,
      "KL/rejected_KL_mean": -69.10401916503906,
      "KL/std": 60.262638092041016,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.004873760510236025,
      "fcm_dpo/delta": -0.0013689089100807905,
      "fcm_dpo/margin": 39.7635612487793,
      "fcm_dpo/q_t": 0.4531114399433136,
      "grad_norm": 20.277210235595703,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": 1.2746118307113647,
      "logits/rejected": 1.2540702819824219,
      "logps/chosen": -304.32891845703125,
      "logps/ref_chosen": -274.98846435546875,
      "logps/ref_rejected": -254.75782775878906,
      "logps/rejected": -323.8618469238281,
      "loss": 4.9232,
      "margin_dpo/margin_mean": 39.7635612487793,
      "margin_dpo/margin_std": 67.70934295654297,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -18.63523292541504,
      "KL/mean": -43.786231994628906,
      "KL/rejected_KL_mean": -68.93724060058594,
      "KL/std": 59.106224060058594,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.004857831634581089,
      "fcm_dpo/delta": -0.003171622520312667,
      "fcm_dpo/margin": 50.3020133972168,
      "fcm_dpo/q_t": 0.4408496618270874,
      "grad_norm": 17.28449821472168,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": 1.086753487586975,
      "logits/rejected": 1.1523369550704956,
      "logps/chosen": -270.26141357421875,
      "logps/ref_chosen": -251.62615966796875,
      "logps/ref_rejected": -251.9549560546875,
      "logps/rejected": -320.8922119140625,
      "loss": 4.7323,
      "margin_dpo/margin_mean": 50.3020133972168,
      "margin_dpo/margin_std": 65.24363708496094,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -26.276391983032227,
      "KL/mean": -50.282135009765625,
      "KL/rejected_KL_mean": -74.28787994384766,
      "KL/std": 61.92739486694336,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.0048460159450769424,
      "fcm_dpo/delta": -0.003930447157472372,
      "fcm_dpo/margin": 48.01148223876953,
      "fcm_dpo/q_t": 0.4436649978160858,
      "grad_norm": 14.025216102600098,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": 1.2417588233947754,
      "logits/rejected": 1.29044771194458,
      "logps/chosen": -304.2130126953125,
      "logps/ref_chosen": -277.9366149902344,
      "logps/ref_rejected": -262.689453125,
      "logps/rejected": -336.9773254394531,
      "loss": 4.7919,
      "margin_dpo/margin_mean": 48.01148223876953,
      "margin_dpo/margin_std": 70.6418685913086,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -15.229087829589844,
      "KL/mean": -38.93570327758789,
      "KL/rejected_KL_mean": -62.64231872558594,
      "KL/std": 59.347415924072266,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.004809689242392778,
      "fcm_dpo/delta": -0.020862078294157982,
      "fcm_dpo/margin": 47.413230895996094,
      "fcm_dpo/q_t": 0.4443405866622925,
      "grad_norm": 26.679866790771484,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": 1.4691436290740967,
      "logits/rejected": 1.4648163318634033,
      "logps/chosen": -285.67401123046875,
      "logps/ref_chosen": -270.4449157714844,
      "logps/ref_rejected": -259.5942077636719,
      "logps/rejected": -322.2365417480469,
      "loss": 4.7885,
      "margin_dpo/margin_mean": 47.413230895996094,
      "margin_dpo/margin_std": 62.96893310546875,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -26.065441131591797,
      "KL/mean": -46.49700164794922,
      "KL/rejected_KL_mean": -66.92855834960938,
      "KL/std": 61.69449996948242,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.0047574336640536785,
      "fcm_dpo/delta": -0.0013780706794932485,
      "fcm_dpo/margin": 40.86311721801758,
      "fcm_dpo/q_t": 0.4529418349266052,
      "grad_norm": 34.731204986572266,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": 1.386948823928833,
      "logits/rejected": 1.4774832725524902,
      "logps/chosen": -295.9053649902344,
      "logps/ref_chosen": -269.83990478515625,
      "logps/ref_rejected": -246.33746337890625,
      "logps/rejected": -313.2659912109375,
      "loss": 4.9394,
      "margin_dpo/margin_mean": 40.86311721801758,
      "margin_dpo/margin_std": 75.67585754394531,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -20.262197494506836,
      "KL/mean": -42.44255828857422,
      "KL/rejected_KL_mean": -64.6229248046875,
      "KL/std": 60.31898498535156,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.004756778012961149,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.36071014404297,
      "fcm_dpo/q_t": 0.44891396164894104,
      "grad_norm": 20.415266036987305,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": 1.2808576822280884,
      "logits/rejected": 1.2620887756347656,
      "logps/chosen": -317.796142578125,
      "logps/ref_chosen": -297.533935546875,
      "logps/ref_rejected": -270.3407897949219,
      "logps/rejected": -334.9637145996094,
      "loss": 4.8528,
      "margin_dpo/margin_mean": 44.36071014404297,
      "margin_dpo/margin_std": 67.61516571044922,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -24.747509002685547,
      "KL/mean": -46.30746078491211,
      "KL/rejected_KL_mean": -67.86742401123047,
      "KL/std": 61.336124420166016,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.004756778012961149,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.119903564453125,
      "fcm_dpo/q_t": 0.4501405954360962,
      "grad_norm": 29.791580200195312,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": 1.1873263120651245,
      "logits/rejected": 1.234531044960022,
      "logps/chosen": -265.08636474609375,
      "logps/ref_chosen": -240.33883666992188,
      "logps/ref_rejected": -220.71717834472656,
      "logps/rejected": -288.5845947265625,
      "loss": 4.8965,
      "margin_dpo/margin_mean": 43.119903564453125,
      "margin_dpo/margin_std": 74.95491027832031,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -21.7313232421875,
      "KL/mean": -48.08092498779297,
      "KL/rejected_KL_mean": -74.43052673339844,
      "KL/std": 64.36347961425781,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.004752114415168762,
      "fcm_dpo/delta": -0.0019622333347797394,
      "fcm_dpo/margin": 52.69917678833008,
      "fcm_dpo/q_t": 0.4393257200717926,
      "grad_norm": 34.03675079345703,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 1.6121935844421387,
      "logits/rejected": 1.6214523315429688,
      "logps/chosen": -330.6244201660156,
      "logps/ref_chosen": -308.8930969238281,
      "logps/ref_rejected": -278.47869873046875,
      "logps/rejected": -352.90924072265625,
      "loss": 4.7496,
      "margin_dpo/margin_mean": 52.69917678833008,
      "margin_dpo/margin_std": 78.26819610595703,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -22.324020385742188,
      "KL/mean": -46.72215270996094,
      "KL/rejected_KL_mean": -71.12028503417969,
      "KL/std": 61.11709213256836,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.004728853236883879,
      "fcm_dpo/delta": -0.014445129781961441,
      "fcm_dpo/margin": 48.79625701904297,
      "fcm_dpo/q_t": 0.44407954812049866,
      "grad_norm": 25.317625045776367,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.3800427913665771,
      "logits/rejected": 1.4031487703323364,
      "logps/chosen": -297.75225830078125,
      "logps/ref_chosen": -275.42822265625,
      "logps/ref_rejected": -251.50906372070312,
      "logps/rejected": -322.6293640136719,
      "loss": 4.7853,
      "margin_dpo/margin_mean": 48.79624938964844,
      "margin_dpo/margin_std": 68.38352966308594,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -21.60759162902832,
      "KL/mean": -44.6788444519043,
      "KL/rejected_KL_mean": -67.75011444091797,
      "KL/std": 61.39951705932617,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.004690962843596935,
      "fcm_dpo/delta": -0.0004940137732774019,
      "fcm_dpo/margin": 46.14250946044922,
      "fcm_dpo/q_t": 0.4475138187408447,
      "grad_norm": 42.43691635131836,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": 1.4158202409744263,
      "logits/rejected": 1.4358092546463013,
      "logps/chosen": -310.9903259277344,
      "logps/ref_chosen": -289.38275146484375,
      "logps/ref_rejected": -274.895751953125,
      "logps/rejected": -342.6458435058594,
      "loss": 4.8351,
      "margin_dpo/margin_mean": 46.14250946044922,
      "margin_dpo/margin_std": 67.42285919189453,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -22.468345642089844,
      "KL/mean": -47.6202392578125,
      "KL/rejected_KL_mean": -72.77214050292969,
      "KL/std": 62.072052001953125,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.004655232187360525,
      "fcm_dpo/delta": -0.011517275124788284,
      "fcm_dpo/margin": 50.30379104614258,
      "fcm_dpo/q_t": 0.44327303767204285,
      "grad_norm": 53.49825668334961,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": 1.4047025442123413,
      "logits/rejected": 1.525120735168457,
      "logps/chosen": -307.45684814453125,
      "logps/ref_chosen": -284.98846435546875,
      "logps/ref_rejected": -273.5077819824219,
      "logps/rejected": -346.2798767089844,
      "loss": 4.7926,
      "margin_dpo/margin_mean": 50.30379104614258,
      "margin_dpo/margin_std": 75.65714263916016,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -25.054706573486328,
      "KL/mean": -52.047157287597656,
      "KL/rejected_KL_mean": -79.03961944580078,
      "KL/std": 65.66096496582031,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.004619365558028221,
      "fcm_dpo/delta": -0.011681661009788513,
      "fcm_dpo/margin": 53.98490524291992,
      "fcm_dpo/q_t": 0.4394109845161438,
      "grad_norm": 32.09884262084961,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": 1.5334891080856323,
      "logits/rejected": 1.6311089992523193,
      "logps/chosen": -287.73431396484375,
      "logps/ref_chosen": -262.6795959472656,
      "logps/ref_rejected": -236.13792419433594,
      "logps/rejected": -315.1775207519531,
      "loss": 4.7273,
      "margin_dpo/margin_mean": 53.98490524291992,
      "margin_dpo/margin_std": 71.97262573242188,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -19.451385498046875,
      "KL/mean": -49.810264587402344,
      "KL/rejected_KL_mean": -80.16914367675781,
      "KL/std": 63.36285400390625,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.004568762145936489,
      "fcm_dpo/delta": -0.013221175409853458,
      "fcm_dpo/margin": 60.7177619934082,
      "fcm_dpo/q_t": 0.4327259063720703,
      "grad_norm": 19.935415267944336,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": 1.0856351852416992,
      "logits/rejected": 1.1892577409744263,
      "logps/chosen": -265.85089111328125,
      "logps/ref_chosen": -246.39950561523438,
      "logps/ref_rejected": -247.4836883544922,
      "logps/rejected": -327.65283203125,
      "loss": 4.626,
      "margin_dpo/margin_mean": 60.7177619934082,
      "margin_dpo/margin_std": 72.81011199951172,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -17.262720108032227,
      "KL/mean": -46.84621810913086,
      "KL/rejected_KL_mean": -76.42971801757812,
      "KL/std": 64.48748016357422,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.0045245056971907616,
      "fcm_dpo/delta": -0.013115347363054752,
      "fcm_dpo/margin": 59.1669921875,
      "fcm_dpo/q_t": 0.4352185130119324,
      "grad_norm": 18.843891143798828,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": 1.4079735279083252,
      "logits/rejected": 1.522764801979065,
      "logps/chosen": -306.1827697753906,
      "logps/ref_chosen": -288.9200439453125,
      "logps/ref_rejected": -291.69305419921875,
      "logps/rejected": -368.12274169921875,
      "loss": 4.6633,
      "margin_dpo/margin_mean": 59.16699981689453,
      "margin_dpo/margin_std": 74.64692687988281,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -34.042415618896484,
      "KL/mean": -52.952964782714844,
      "KL/rejected_KL_mean": -71.863525390625,
      "KL/std": 63.20186233520508,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.0045105572789907455,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.82111358642578,
      "fcm_dpo/q_t": 0.45854708552360535,
      "grad_norm": 38.761680603027344,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": 1.09867525100708,
      "logits/rejected": 1.1397531032562256,
      "logps/chosen": -291.1272888183594,
      "logps/ref_chosen": -257.08489990234375,
      "logps/ref_rejected": -251.61532592773438,
      "logps/rejected": -323.4788818359375,
      "loss": 5.0282,
      "margin_dpo/margin_mean": 37.82111358642578,
      "margin_dpo/margin_std": 80.87380981445312,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -39.81092071533203,
      "KL/mean": -57.12293243408203,
      "KL/rejected_KL_mean": -74.43495178222656,
      "KL/std": 64.91429901123047,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.004512595012784004,
      "fcm_dpo/delta": 0.0009030746296048164,
      "fcm_dpo/margin": 34.62403106689453,
      "fcm_dpo/q_t": 0.4621298015117645,
      "grad_norm": 34.532527923583984,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": 1.3962581157684326,
      "logits/rejected": 1.3683218955993652,
      "logps/chosen": -300.5522766113281,
      "logps/ref_chosen": -260.7413635253906,
      "logps/ref_rejected": -227.4325408935547,
      "logps/rejected": -301.86749267578125,
      "loss": 5.0893,
      "margin_dpo/margin_mean": 34.62403106689453,
      "margin_dpo/margin_std": 80.49478912353516,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -27.686656951904297,
      "KL/mean": -56.293182373046875,
      "KL/rejected_KL_mean": -84.89971160888672,
      "KL/std": 67.04974365234375,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.004473809618502855,
      "fcm_dpo/delta": -0.01613171026110649,
      "fcm_dpo/margin": 57.213050842285156,
      "fcm_dpo/q_t": 0.43794018030166626,
      "grad_norm": 34.26251220703125,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": 1.223738193511963,
      "logits/rejected": 1.3191523551940918,
      "logps/chosen": -330.6341857910156,
      "logps/ref_chosen": -302.947509765625,
      "logps/ref_rejected": -294.1771545410156,
      "logps/rejected": -379.076904296875,
      "loss": 4.6995,
      "margin_dpo/margin_mean": 57.21304702758789,
      "margin_dpo/margin_std": 72.50021362304688,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -30.430696487426758,
      "KL/mean": -55.44902801513672,
      "KL/rejected_KL_mean": -80.46735382080078,
      "KL/std": 64.27362823486328,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.004457960370928049,
      "fcm_dpo/delta": 0.001133347861468792,
      "fcm_dpo/margin": 50.03666687011719,
      "fcm_dpo/q_t": 0.4459591805934906,
      "grad_norm": 28.231075286865234,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 1.30367112159729,
      "logits/rejected": 1.3837306499481201,
      "logps/chosen": -288.32879638671875,
      "logps/ref_chosen": -257.8981018066406,
      "logps/ref_rejected": -244.9084014892578,
      "logps/rejected": -325.375732421875,
      "loss": 4.8302,
      "margin_dpo/margin_mean": 50.03666687011719,
      "margin_dpo/margin_std": 79.5361557006836,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -40.557918548583984,
      "KL/mean": -67.02166748046875,
      "KL/rejected_KL_mean": -93.48542785644531,
      "KL/std": 66.38741302490234,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.004437134135514498,
      "fcm_dpo/delta": -0.011196678504347801,
      "fcm_dpo/margin": 52.9275016784668,
      "fcm_dpo/q_t": 0.44308292865753174,
      "grad_norm": 32.53326416015625,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 1.1773613691329956,
      "logits/rejected": 1.2400239706039429,
      "logps/chosen": -330.7525634765625,
      "logps/ref_chosen": -290.1946716308594,
      "logps/ref_rejected": -290.98883056640625,
      "logps/rejected": -384.4742431640625,
      "loss": 4.7678,
      "margin_dpo/margin_mean": 52.9275016784668,
      "margin_dpo/margin_std": 70.56356048583984,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -40.77531814575195,
      "KL/mean": -62.49406433105469,
      "KL/rejected_KL_mean": -84.21281433105469,
      "KL/std": 67.94178771972656,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.004421152640134096,
      "fcm_dpo/delta": 0.0002664776402525604,
      "fcm_dpo/margin": 43.4375114440918,
      "fcm_dpo/q_t": 0.4537264406681061,
      "grad_norm": 29.236297607421875,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": 1.4015604257583618,
      "logits/rejected": 1.444056510925293,
      "logps/chosen": -349.6927490234375,
      "logps/ref_chosen": -308.91741943359375,
      "logps/ref_rejected": -272.2177734375,
      "logps/rejected": -356.4305725097656,
      "loss": 4.9591,
      "margin_dpo/margin_mean": 43.4375114440918,
      "margin_dpo/margin_std": 85.44091796875,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -44.8071403503418,
      "KL/mean": -70.00708770751953,
      "KL/rejected_KL_mean": -95.20703887939453,
      "KL/std": 62.773651123046875,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.004405615851283073,
      "fcm_dpo/delta": -0.011769093573093414,
      "fcm_dpo/margin": 50.399898529052734,
      "fcm_dpo/q_t": 0.44614121317863464,
      "grad_norm": 30.119125366210938,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 1.7217158079147339,
      "logits/rejected": 1.8126791715621948,
      "logps/chosen": -292.82916259765625,
      "logps/ref_chosen": -248.02200317382812,
      "logps/ref_rejected": -234.78541564941406,
      "logps/rejected": -329.99249267578125,
      "loss": 4.82,
      "margin_dpo/margin_mean": 50.39990234375,
      "margin_dpo/margin_std": 73.72570037841797,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -42.80335235595703,
      "KL/mean": -68.65229034423828,
      "KL/rejected_KL_mean": -94.501220703125,
      "KL/std": 69.01512145996094,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.004350032191723585,
      "fcm_dpo/delta": -0.009891406632959843,
      "fcm_dpo/margin": 51.6978645324707,
      "fcm_dpo/q_t": 0.4459015727043152,
      "grad_norm": 23.92279815673828,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": 1.3646740913391113,
      "logits/rejected": 1.3824570178985596,
      "logps/chosen": -307.2925720214844,
      "logps/ref_chosen": -264.4892578125,
      "logps/ref_rejected": -255.43092346191406,
      "logps/rejected": -349.9321594238281,
      "loss": 4.8486,
      "margin_dpo/margin_mean": 51.6978645324707,
      "margin_dpo/margin_std": 86.31040954589844,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -44.26451110839844,
      "KL/mean": -63.2172966003418,
      "KL/rejected_KL_mean": -82.1700668334961,
      "KL/std": 65.42170715332031,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.0043395040556788445,
      "fcm_dpo/delta": -0.006573015823960304,
      "fcm_dpo/margin": 37.90556335449219,
      "fcm_dpo/q_t": 0.46003422141075134,
      "grad_norm": 26.278900146484375,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": 1.5798490047454834,
      "logits/rejected": 1.6110665798187256,
      "logps/chosen": -345.7331848144531,
      "logps/ref_chosen": -301.46868896484375,
      "logps/ref_rejected": -274.2380676269531,
      "logps/rejected": -356.40814208984375,
      "loss": 5.0383,
      "margin_dpo/margin_mean": 37.90556335449219,
      "margin_dpo/margin_std": 76.93035888671875,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -46.72309112548828,
      "KL/mean": -67.91627502441406,
      "KL/rejected_KL_mean": -89.10946655273438,
      "KL/std": 64.8916015625,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.0043224128894507885,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.386383056640625,
      "fcm_dpo/q_t": 0.45563966035842896,
      "grad_norm": 22.88116455078125,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": 1.3460297584533691,
      "logits/rejected": 1.4205471277236938,
      "logps/chosen": -313.28216552734375,
      "logps/ref_chosen": -266.5590515136719,
      "logps/ref_rejected": -280.12066650390625,
      "logps/rejected": -369.2301330566406,
      "loss": 4.9732,
      "margin_dpo/margin_mean": 42.386383056640625,
      "margin_dpo/margin_std": 81.49267578125,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -45.9259147644043,
      "KL/mean": -68.34357452392578,
      "KL/rejected_KL_mean": -90.76123809814453,
      "KL/std": 60.805137634277344,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.004324114881455898,
      "fcm_dpo/delta": 0.0007875416777096689,
      "fcm_dpo/margin": 44.835323333740234,
      "fcm_dpo/q_t": 0.45277613401412964,
      "grad_norm": 50.52631378173828,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": 1.1059064865112305,
      "logits/rejected": 1.1593595743179321,
      "logps/chosen": -315.86224365234375,
      "logps/ref_chosen": -269.93634033203125,
      "logps/ref_rejected": -232.32330322265625,
      "logps/rejected": -323.08453369140625,
      "loss": 4.9238,
      "margin_dpo/margin_mean": 44.835323333740234,
      "margin_dpo/margin_std": 76.86017608642578,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -44.96957778930664,
      "KL/mean": -67.43724822998047,
      "KL/rejected_KL_mean": -89.90492248535156,
      "KL/std": 66.4072494506836,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.004320498555898666,
      "fcm_dpo/delta": -0.010769463144242764,
      "fcm_dpo/margin": 44.93536376953125,
      "fcm_dpo/q_t": 0.45310139656066895,
      "grad_norm": 28.669755935668945,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": 1.5005367994308472,
      "logits/rejected": 1.477102518081665,
      "logps/chosen": -329.01507568359375,
      "logps/ref_chosen": -284.04547119140625,
      "logps/ref_rejected": -261.19378662109375,
      "logps/rejected": -351.0987243652344,
      "loss": 4.9384,
      "margin_dpo/margin_mean": 44.93536376953125,
      "margin_dpo/margin_std": 82.00439453125,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -51.29305648803711,
      "KL/mean": -67.19605255126953,
      "KL/rejected_KL_mean": -83.09906005859375,
      "KL/std": 67.37464141845703,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.004288033116608858,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.806007385253906,
      "fcm_dpo/q_t": 0.46691325306892395,
      "grad_norm": 27.18282127380371,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": 1.3973357677459717,
      "logits/rejected": 1.382079839706421,
      "logps/chosen": -374.5743103027344,
      "logps/ref_chosen": -323.28125,
      "logps/ref_rejected": -274.2093200683594,
      "logps/rejected": -357.3083801269531,
      "loss": 5.1407,
      "margin_dpo/margin_mean": 31.806007385253906,
      "margin_dpo/margin_std": 80.26555633544922,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -44.79306411743164,
      "KL/mean": -63.81265640258789,
      "KL/rejected_KL_mean": -82.8322525024414,
      "KL/std": 66.64505004882812,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.004286645445972681,
      "fcm_dpo/delta": -0.0005395765183493495,
      "fcm_dpo/margin": 38.039188385009766,
      "fcm_dpo/q_t": 0.4604703187942505,
      "grad_norm": 36.115478515625,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 1.193994164466858,
      "logits/rejected": 1.1565959453582764,
      "logps/chosen": -362.22845458984375,
      "logps/ref_chosen": -317.4353942871094,
      "logps/ref_rejected": -309.32403564453125,
      "logps/rejected": -392.1562805175781,
      "loss": 5.0355,
      "margin_dpo/margin_mean": 38.039188385009766,
      "margin_dpo/margin_std": 77.6399154663086,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -37.74793243408203,
      "KL/mean": -66.97676849365234,
      "KL/rejected_KL_mean": -96.20561218261719,
      "KL/std": 70.90298461914062,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.004265831783413887,
      "fcm_dpo/delta": -0.010897559113800526,
      "fcm_dpo/margin": 58.45766830444336,
      "fcm_dpo/q_t": 0.4395430386066437,
      "grad_norm": 42.119606018066406,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 1.1978421211242676,
      "logits/rejected": 1.3700828552246094,
      "logps/chosen": -303.25616455078125,
      "logps/ref_chosen": -265.50823974609375,
      "logps/ref_rejected": -275.1380310058594,
      "logps/rejected": -371.3436279296875,
      "loss": 4.7381,
      "margin_dpo/margin_mean": 58.457672119140625,
      "margin_dpo/margin_std": 82.14814758300781,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -31.5036563873291,
      "KL/mean": -55.08972930908203,
      "KL/rejected_KL_mean": -78.67579650878906,
      "KL/std": 66.80307006835938,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.004226091783493757,
      "fcm_dpo/delta": -0.00900965929031372,
      "fcm_dpo/margin": 47.172142028808594,
      "fcm_dpo/q_t": 0.45140957832336426,
      "grad_norm": 28.07166862487793,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": 1.29680335521698,
      "logits/rejected": 1.4240124225616455,
      "logps/chosen": -281.40496826171875,
      "logps/ref_chosen": -249.9013214111328,
      "logps/ref_rejected": -274.58245849609375,
      "logps/rejected": -353.2582702636719,
      "loss": 4.8983,
      "margin_dpo/margin_mean": 47.17214584350586,
      "margin_dpo/margin_std": 74.5413818359375,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -40.41657257080078,
      "KL/mean": -65.62700653076172,
      "KL/rejected_KL_mean": -90.83744049072266,
      "KL/std": 70.43557739257812,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.004211593419313431,
      "fcm_dpo/delta": -0.00423895288258791,
      "fcm_dpo/margin": 50.42085647583008,
      "fcm_dpo/q_t": 0.44849881529808044,
      "grad_norm": 36.46430587768555,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": 1.6114522218704224,
      "logits/rejected": 1.6038446426391602,
      "logps/chosen": -336.7225036621094,
      "logps/ref_chosen": -296.30596923828125,
      "logps/ref_rejected": -268.0640869140625,
      "logps/rejected": -358.9014892578125,
      "loss": 4.8476,
      "margin_dpo/margin_mean": 50.42085647583008,
      "margin_dpo/margin_std": 75.41331481933594,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -34.22224426269531,
      "KL/mean": -56.34048080444336,
      "KL/rejected_KL_mean": -78.45870971679688,
      "KL/std": 67.74516296386719,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.0042041814886033535,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.236473083496094,
      "fcm_dpo/q_t": 0.4548218250274658,
      "grad_norm": 28.696266174316406,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": 1.3955225944519043,
      "logits/rejected": 1.4542133808135986,
      "logps/chosen": -268.105224609375,
      "logps/ref_chosen": -233.8829803466797,
      "logps/ref_rejected": -224.14334106445312,
      "logps/rejected": -302.6020812988281,
      "loss": 4.9537,
      "margin_dpo/margin_mean": 44.23646926879883,
      "margin_dpo/margin_std": 79.65289306640625,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -34.84636688232422,
      "KL/mean": -59.73717498779297,
      "KL/rejected_KL_mean": -84.62798309326172,
      "KL/std": 67.07266998291016,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.0042041814886033535,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.7816162109375,
      "fcm_dpo/q_t": 0.44940096139907837,
      "grad_norm": 26.853879928588867,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": 1.3590420484542847,
      "logits/rejected": 1.3767759799957275,
      "logps/chosen": -278.95770263671875,
      "logps/ref_chosen": -244.111328125,
      "logps/ref_rejected": -240.18373107910156,
      "logps/rejected": -324.81170654296875,
      "loss": 4.8752,
      "margin_dpo/margin_mean": 49.7816162109375,
      "margin_dpo/margin_std": 82.17388916015625,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -30.360498428344727,
      "KL/mean": -63.16857147216797,
      "KL/rejected_KL_mean": -95.97663116455078,
      "KL/std": 69.53978729248047,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.004164664540439844,
      "fcm_dpo/delta": -0.01857716590166092,
      "fcm_dpo/margin": 65.61615753173828,
      "fcm_dpo/q_t": 0.43357765674591064,
      "grad_norm": 23.006004333496094,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 1.355375051498413,
      "logits/rejected": 1.534814476966858,
      "logps/chosen": -315.04107666015625,
      "logps/ref_chosen": -284.6805725097656,
      "logps/ref_rejected": -282.1923828125,
      "logps/rejected": -378.1689453125,
      "loss": 4.6499,
      "margin_dpo/margin_mean": 65.61615753173828,
      "margin_dpo/margin_std": 82.957275390625,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -32.585960388183594,
      "KL/mean": -58.152099609375,
      "KL/rejected_KL_mean": -83.7182388305664,
      "KL/std": 67.38668823242188,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.004121565725654364,
      "fcm_dpo/delta": -0.007001765072345734,
      "fcm_dpo/margin": 51.132259368896484,
      "fcm_dpo/q_t": 0.448840856552124,
      "grad_norm": 22.238370895385742,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": 1.4639967679977417,
      "logits/rejected": 1.473618984222412,
      "logps/chosen": -312.8145751953125,
      "logps/ref_chosen": -280.2286376953125,
      "logps/ref_rejected": -280.8172912597656,
      "logps/rejected": -364.5355224609375,
      "loss": 4.8563,
      "margin_dpo/margin_mean": 51.132259368896484,
      "margin_dpo/margin_std": 78.27207946777344,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -25.93964958190918,
      "KL/mean": -54.860321044921875,
      "KL/rejected_KL_mean": -83.78099060058594,
      "KL/std": 71.21715545654297,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.004096931777894497,
      "fcm_dpo/delta": -0.014958172105252743,
      "fcm_dpo/margin": 57.84134292602539,
      "fcm_dpo/q_t": 0.4429057538509369,
      "grad_norm": 22.162282943725586,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": 1.3619986772537231,
      "logits/rejected": 1.3755991458892822,
      "logps/chosen": -322.9399719238281,
      "logps/ref_chosen": -297.0003356933594,
      "logps/ref_rejected": -279.1561279296875,
      "logps/rejected": -362.9371337890625,
      "loss": 4.7835,
      "margin_dpo/margin_mean": 57.84134292602539,
      "margin_dpo/margin_std": 85.39665222167969,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -36.36292266845703,
      "KL/mean": -59.93888473510742,
      "KL/rejected_KL_mean": -83.51484680175781,
      "KL/std": 69.19416809082031,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.004066410940140486,
      "fcm_dpo/delta": -0.0015857742400839925,
      "fcm_dpo/margin": 47.15193176269531,
      "fcm_dpo/q_t": 0.4532583951950073,
      "grad_norm": 26.17923927307129,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": 1.366503119468689,
      "logits/rejected": 1.470059871673584,
      "logps/chosen": -307.8343505859375,
      "logps/ref_chosen": -271.471435546875,
      "logps/ref_rejected": -268.0870056152344,
      "logps/rejected": -351.60186767578125,
      "loss": 4.9096,
      "margin_dpo/margin_mean": 47.15193557739258,
      "margin_dpo/margin_std": 71.60176849365234,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -37.46304702758789,
      "KL/mean": -61.87550354003906,
      "KL/rejected_KL_mean": -86.2879638671875,
      "KL/std": 68.84837341308594,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.0040648686699569225,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.82493591308594,
      "fcm_dpo/q_t": 0.4519917070865631,
      "grad_norm": 30.296241760253906,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 1.3365329504013062,
      "logits/rejected": 1.3320995569229126,
      "logps/chosen": -343.8209228515625,
      "logps/ref_chosen": -306.35784912109375,
      "logps/ref_rejected": -268.64776611328125,
      "logps/rejected": -354.93572998046875,
      "loss": 4.9147,
      "margin_dpo/margin_mean": 48.82493591308594,
      "margin_dpo/margin_std": 85.29371643066406,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -36.709136962890625,
      "KL/mean": -60.71306610107422,
      "KL/rejected_KL_mean": -84.71700286865234,
      "KL/std": 69.09561920166016,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.0040648686699569225,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.00785827636719,
      "fcm_dpo/q_t": 0.4525894522666931,
      "grad_norm": 34.80900192260742,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.14371919631958,
      "logits/rejected": 1.1913278102874756,
      "logps/chosen": -306.70159912109375,
      "logps/ref_chosen": -269.9924621582031,
      "logps/ref_rejected": -271.3135681152344,
      "logps/rejected": -356.03057861328125,
      "loss": 4.9104,
      "margin_dpo/margin_mean": 48.00785827636719,
      "margin_dpo/margin_std": 79.8358154296875,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -33.48521423339844,
      "KL/mean": -62.557342529296875,
      "KL/rejected_KL_mean": -91.62948608398438,
      "KL/std": 68.52920532226562,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.004041476175189018,
      "fcm_dpo/delta": -0.006993370596319437,
      "fcm_dpo/margin": 58.14426040649414,
      "fcm_dpo/q_t": 0.4427790641784668,
      "grad_norm": 21.615293502807617,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": 1.1951918601989746,
      "logits/rejected": 1.272159457206726,
      "logps/chosen": -293.9459228515625,
      "logps/ref_chosen": -260.460693359375,
      "logps/ref_rejected": -254.2005615234375,
      "logps/rejected": -345.8300476074219,
      "loss": 4.7775,
      "margin_dpo/margin_mean": 58.14426040649414,
      "margin_dpo/margin_std": 82.3933334350586,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -33.315982818603516,
      "KL/mean": -59.34415817260742,
      "KL/rejected_KL_mean": -85.372314453125,
      "KL/std": 68.67239379882812,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.004030800890177488,
      "fcm_dpo/delta": -0.0048316288739442825,
      "fcm_dpo/margin": 52.05633544921875,
      "fcm_dpo/q_t": 0.4488368630409241,
      "grad_norm": 31.296024322509766,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": 1.2147839069366455,
      "logits/rejected": 1.264925479888916,
      "logps/chosen": -329.4122314453125,
      "logps/ref_chosen": -296.09625244140625,
      "logps/ref_rejected": -264.7806701660156,
      "logps/rejected": -350.1529846191406,
      "loss": 4.8704,
      "margin_dpo/margin_mean": 52.05634307861328,
      "margin_dpo/margin_std": 82.28915405273438,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -46.2089729309082,
      "KL/mean": -69.05396270751953,
      "KL/rejected_KL_mean": -91.89895629882812,
      "KL/std": 72.41987609863281,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.004027293063700199,
      "fcm_dpo/delta": 0.00034591404255479574,
      "fcm_dpo/margin": 45.689971923828125,
      "fcm_dpo/q_t": 0.4552707374095917,
      "grad_norm": 25.77277946472168,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": 1.2982745170593262,
      "logits/rejected": 1.313848853111267,
      "logps/chosen": -328.0584716796875,
      "logps/ref_chosen": -281.8494873046875,
      "logps/ref_rejected": -234.3673858642578,
      "logps/rejected": -326.266357421875,
      "loss": 4.9564,
      "margin_dpo/margin_mean": 45.68997573852539,
      "margin_dpo/margin_std": 82.60110473632812,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -26.901674270629883,
      "KL/mean": -60.61687088012695,
      "KL/rejected_KL_mean": -94.33207702636719,
      "KL/std": 72.72174835205078,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.003974398132413626,
      "fcm_dpo/delta": -0.0269442331045866,
      "fcm_dpo/margin": 67.43041229248047,
      "fcm_dpo/q_t": 0.4349156320095062,
      "grad_norm": 21.982114791870117,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": 1.3066025972366333,
      "logits/rejected": 1.4130481481552124,
      "logps/chosen": -274.9444580078125,
      "logps/ref_chosen": -248.0427703857422,
      "logps/ref_rejected": -263.464599609375,
      "logps/rejected": -357.79669189453125,
      "loss": 4.6624,
      "margin_dpo/margin_mean": 67.43041229248047,
      "margin_dpo/margin_std": 83.17764282226562,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -25.44013786315918,
      "KL/mean": -55.096893310546875,
      "KL/rejected_KL_mean": -84.75364685058594,
      "KL/std": 69.17593383789062,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.003930878825485706,
      "fcm_dpo/delta": -0.004289279226213694,
      "fcm_dpo/margin": 59.31350326538086,
      "fcm_dpo/q_t": 0.44320034980773926,
      "grad_norm": 24.75372886657715,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": 1.4009541273117065,
      "logits/rejected": 1.3296130895614624,
      "logps/chosen": -336.72607421875,
      "logps/ref_chosen": -311.2859191894531,
      "logps/ref_rejected": -268.4949645996094,
      "logps/rejected": -353.2486267089844,
      "loss": 4.7737,
      "margin_dpo/margin_mean": 59.313499450683594,
      "margin_dpo/margin_std": 79.95907592773438,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -33.308738708496094,
      "KL/mean": -64.83464813232422,
      "KL/rejected_KL_mean": -96.36056518554688,
      "KL/std": 72.08885955810547,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.003924451768398285,
      "fcm_dpo/delta": -0.007943897508084774,
      "fcm_dpo/margin": 63.05182647705078,
      "fcm_dpo/q_t": 0.4396139085292816,
      "grad_norm": 34.679622650146484,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": 1.3536103963851929,
      "logits/rejected": 1.3708926439285278,
      "logps/chosen": -303.52117919921875,
      "logps/ref_chosen": -270.21246337890625,
      "logps/ref_rejected": -239.1932830810547,
      "logps/rejected": -335.5538330078125,
      "loss": 4.7157,
      "margin_dpo/margin_mean": 63.05182647705078,
      "margin_dpo/margin_std": 77.1610107421875,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -41.373069763183594,
      "KL/mean": -62.410579681396484,
      "KL/rejected_KL_mean": -83.4480972290039,
      "KL/std": 69.45616149902344,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.003907492849975824,
      "fcm_dpo/delta": 0.0012996502919122577,
      "fcm_dpo/margin": 42.07502365112305,
      "fcm_dpo/q_t": 0.4599437415599823,
      "grad_norm": 20.657968521118164,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": 1.4415968656539917,
      "logits/rejected": 1.456695318222046,
      "logps/chosen": -329.8958740234375,
      "logps/ref_chosen": -288.5228271484375,
      "logps/ref_rejected": -253.90597534179688,
      "logps/rejected": -337.3540954589844,
      "loss": 5.0195,
      "margin_dpo/margin_mean": 42.07502746582031,
      "margin_dpo/margin_std": 80.3031234741211,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -38.390403747558594,
      "KL/mean": -69.86437225341797,
      "KL/rejected_KL_mean": -101.33831787109375,
      "KL/std": 73.71216583251953,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.0038987554144114256,
      "fcm_dpo/delta": -0.00925364438444376,
      "fcm_dpo/margin": 62.947906494140625,
      "fcm_dpo/q_t": 0.44066518545150757,
      "grad_norm": 29.97333335876465,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 1.289689540863037,
      "logits/rejected": 1.4189997911453247,
      "logps/chosen": -310.2851257324219,
      "logps/ref_chosen": -271.8947448730469,
      "logps/ref_rejected": -260.5709228515625,
      "logps/rejected": -361.90924072265625,
      "loss": 4.7462,
      "margin_dpo/margin_mean": 62.947906494140625,
      "margin_dpo/margin_std": 87.69436645507812,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -41.82401657104492,
      "KL/mean": -67.83028411865234,
      "KL/rejected_KL_mean": -93.83656311035156,
      "KL/std": 70.22896575927734,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.0038792260456830263,
      "fcm_dpo/delta": 0.001425105263479054,
      "fcm_dpo/margin": 52.012542724609375,
      "fcm_dpo/q_t": 0.4509691596031189,
      "grad_norm": 17.642919540405273,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 1.3056867122650146,
      "logits/rejected": 1.4215185642242432,
      "logps/chosen": -311.99951171875,
      "logps/ref_chosen": -270.17547607421875,
      "logps/ref_rejected": -257.6781311035156,
      "logps/rejected": -351.51470947265625,
      "loss": 4.8733,
      "margin_dpo/margin_mean": 52.01254653930664,
      "margin_dpo/margin_std": 77.01701354980469,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -40.84086608886719,
      "KL/mean": -68.33751678466797,
      "KL/rejected_KL_mean": -95.83415222167969,
      "KL/std": 73.27204895019531,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.0038795368745923042,
      "fcm_dpo/delta": -0.004050515592098236,
      "fcm_dpo/margin": 54.99327087402344,
      "fcm_dpo/q_t": 0.44809120893478394,
      "grad_norm": 15.816947937011719,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 1.1713835000991821,
      "logits/rejected": 1.3251903057098389,
      "logps/chosen": -304.6031494140625,
      "logps/ref_chosen": -263.76226806640625,
      "logps/ref_rejected": -263.8758850097656,
      "logps/rejected": -359.71002197265625,
      "loss": 4.8668,
      "margin_dpo/margin_mean": 54.99327087402344,
      "margin_dpo/margin_std": 89.631103515625,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -38.10146713256836,
      "KL/mean": -65.74610900878906,
      "KL/rejected_KL_mean": -93.39075469970703,
      "KL/std": 69.32891082763672,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.003848535241559148,
      "fcm_dpo/delta": -0.011419926770031452,
      "fcm_dpo/margin": 55.2892951965332,
      "fcm_dpo/q_t": 0.44809532165527344,
      "grad_norm": 28.016109466552734,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": 1.2274868488311768,
      "logits/rejected": 1.196799397468567,
      "logps/chosen": -312.77392578125,
      "logps/ref_chosen": -274.67242431640625,
      "logps/ref_rejected": -242.1553955078125,
      "logps/rejected": -335.546142578125,
      "loss": 4.851,
      "margin_dpo/margin_mean": 55.2892951965332,
      "margin_dpo/margin_std": 83.24381256103516,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -42.066837310791016,
      "KL/mean": -67.20684814453125,
      "KL/rejected_KL_mean": -92.34686279296875,
      "KL/std": 74.03976440429688,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.0038281409069895744,
      "fcm_dpo/delta": -0.018899880349636078,
      "fcm_dpo/margin": 50.2800178527832,
      "fcm_dpo/q_t": 0.45321422815322876,
      "grad_norm": 21.685569763183594,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": 1.4938710927963257,
      "logits/rejected": 1.5979026556015015,
      "logps/chosen": -337.2547302246094,
      "logps/ref_chosen": -295.1878662109375,
      "logps/ref_rejected": -271.3431091308594,
      "logps/rejected": -363.69000244140625,
      "loss": 4.9355,
      "margin_dpo/margin_mean": 50.2800178527832,
      "margin_dpo/margin_std": 87.26283264160156,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -38.41937255859375,
      "KL/mean": -65.4301986694336,
      "KL/rejected_KL_mean": -92.44102478027344,
      "KL/std": 72.63717651367188,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.003777781268581748,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.02167510986328,
      "fcm_dpo/q_t": 0.45029619336128235,
      "grad_norm": 23.136240005493164,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": 1.227036714553833,
      "logits/rejected": 1.2757822275161743,
      "logps/chosen": -328.18719482421875,
      "logps/ref_chosen": -289.767822265625,
      "logps/ref_rejected": -267.0824279785156,
      "logps/rejected": -359.5234375,
      "loss": 4.8856,
      "margin_dpo/margin_mean": 54.02167510986328,
      "margin_dpo/margin_std": 87.43756103515625,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -34.78050231933594,
      "KL/mean": -64.88722229003906,
      "KL/rejected_KL_mean": -94.99395751953125,
      "KL/std": 72.9035415649414,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.0037749430630356073,
      "fcm_dpo/delta": -0.0018531241221353412,
      "fcm_dpo/margin": 60.21345520019531,
      "fcm_dpo/q_t": 0.444881409406662,
      "grad_norm": 31.45768165588379,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": 1.6590012311935425,
      "logits/rejected": 1.695084571838379,
      "logps/chosen": -333.1995544433594,
      "logps/ref_chosen": -298.4190673828125,
      "logps/ref_rejected": -266.64447021484375,
      "logps/rejected": -361.638427734375,
      "loss": 4.7936,
      "margin_dpo/margin_mean": 60.21345138549805,
      "margin_dpo/margin_std": 85.1278076171875,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -41.15290069580078,
      "KL/mean": -72.2494125366211,
      "KL/rejected_KL_mean": -103.34593200683594,
      "KL/std": 75.14649963378906,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.003762506414204836,
      "fcm_dpo/delta": -0.0032112684566527605,
      "fcm_dpo/margin": 62.193031311035156,
      "fcm_dpo/q_t": 0.44330793619155884,
      "grad_norm": 28.033740997314453,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": 1.4360030889511108,
      "logits/rejected": 1.4453506469726562,
      "logps/chosen": -343.38275146484375,
      "logps/ref_chosen": -302.2298583984375,
      "logps/ref_rejected": -272.7162170410156,
      "logps/rejected": -376.0621337890625,
      "loss": 4.7774,
      "margin_dpo/margin_mean": 62.193031311035156,
      "margin_dpo/margin_std": 88.42662048339844,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -32.11273193359375,
      "KL/mean": -60.81684112548828,
      "KL/rejected_KL_mean": -89.52095031738281,
      "KL/std": 70.97887420654297,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.0037522483617067337,
      "fcm_dpo/delta": -0.0039009428583085537,
      "fcm_dpo/margin": 57.40822219848633,
      "fcm_dpo/q_t": 0.4475215673446655,
      "grad_norm": 25.648094177246094,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": 1.2365297079086304,
      "logits/rejected": 1.2107276916503906,
      "logps/chosen": -278.9661560058594,
      "logps/ref_chosen": -246.85342407226562,
      "logps/ref_rejected": -216.5264892578125,
      "logps/rejected": -306.04742431640625,
      "loss": 4.8278,
      "margin_dpo/margin_mean": 57.40821838378906,
      "margin_dpo/margin_std": 81.15754699707031,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -36.14681625366211,
      "KL/mean": -65.88819885253906,
      "KL/rejected_KL_mean": -95.62959289550781,
      "KL/std": 72.37037658691406,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.003739651059731841,
      "fcm_dpo/delta": -0.00484112836420536,
      "fcm_dpo/margin": 59.482791900634766,
      "fcm_dpo/q_t": 0.44558480381965637,
      "grad_norm": 38.77082061767578,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": 1.2492884397506714,
      "logits/rejected": 1.335770845413208,
      "logps/chosen": -298.79644775390625,
      "logps/ref_chosen": -262.649658203125,
      "logps/ref_rejected": -251.24945068359375,
      "logps/rejected": -346.8790283203125,
      "loss": 4.799,
      "margin_dpo/margin_mean": 59.482791900634766,
      "margin_dpo/margin_std": 79.76242065429688,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -33.609230041503906,
      "KL/mean": -65.16617584228516,
      "KL/rejected_KL_mean": -96.72313690185547,
      "KL/std": 71.05553436279297,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.0037287170998752117,
      "fcm_dpo/delta": -0.004279743414372206,
      "fcm_dpo/margin": 63.1138916015625,
      "fcm_dpo/q_t": 0.4424435496330261,
      "grad_norm": 26.169889450073242,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": 1.3643083572387695,
      "logits/rejected": 1.4147653579711914,
      "logps/chosen": -326.9955749511719,
      "logps/ref_chosen": -293.3863220214844,
      "logps/ref_rejected": -238.86026000976562,
      "logps/rejected": -335.5834045410156,
      "loss": 4.7349,
      "margin_dpo/margin_mean": 63.1138916015625,
      "margin_dpo/margin_std": 72.3996810913086,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -32.760833740234375,
      "KL/mean": -61.89388656616211,
      "KL/rejected_KL_mean": -91.02693939208984,
      "KL/std": 71.0989990234375,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.003724002977833152,
      "fcm_dpo/delta": -0.001423541223630309,
      "fcm_dpo/margin": 58.266090393066406,
      "fcm_dpo/q_t": 0.44715023040771484,
      "grad_norm": 27.365583419799805,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 1.4122627973556519,
      "logits/rejected": 1.4529802799224854,
      "logps/chosen": -328.2646484375,
      "logps/ref_chosen": -295.5038146972656,
      "logps/ref_rejected": -278.524169921875,
      "logps/rejected": -369.5511474609375,
      "loss": 4.8432,
      "margin_dpo/margin_mean": 58.266090393066406,
      "margin_dpo/margin_std": 90.42108917236328,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -35.895790100097656,
      "KL/mean": -61.337730407714844,
      "KL/rejected_KL_mean": -86.7796630859375,
      "KL/std": 71.69768524169922,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.003719276050105691,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.88385772705078,
      "fcm_dpo/q_t": 0.45384901762008667,
      "grad_norm": 28.481828689575195,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 1.6092890501022339,
      "logits/rejected": 1.5551338195800781,
      "logps/chosen": -316.6511535644531,
      "logps/ref_chosen": -280.7553405761719,
      "logps/ref_rejected": -237.47286987304688,
      "logps/rejected": -324.25250244140625,
      "loss": 4.921,
      "margin_dpo/margin_mean": 50.88385772705078,
      "margin_dpo/margin_std": 82.6810302734375,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -37.419639587402344,
      "KL/mean": -67.68118286132812,
      "KL/rejected_KL_mean": -97.94271850585938,
      "KL/std": 73.34646606445312,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.0037213361356407404,
      "fcm_dpo/delta": -0.00031929946271702647,
      "fcm_dpo/margin": 60.523075103759766,
      "fcm_dpo/q_t": 0.44503477215766907,
      "grad_norm": 40.07929992675781,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 1.3795121908187866,
      "logits/rejected": 1.3915975093841553,
      "logps/chosen": -336.5109558105469,
      "logps/ref_chosen": -299.09130859375,
      "logps/ref_rejected": -268.9637756347656,
      "logps/rejected": -366.9064636230469,
      "loss": 4.7896,
      "margin_dpo/margin_mean": 60.523075103759766,
      "margin_dpo/margin_std": 80.9197006225586,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -46.54456329345703,
      "KL/mean": -66.1429214477539,
      "KL/rejected_KL_mean": -85.74125671386719,
      "KL/std": 71.34706115722656,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.003718326333910227,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.196693420410156,
      "fcm_dpo/q_t": 0.46447882056236267,
      "grad_norm": 20.844757080078125,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": 1.3957551717758179,
      "logits/rejected": 1.339003562927246,
      "logps/chosen": -323.2406005859375,
      "logps/ref_chosen": -276.696044921875,
      "logps/ref_rejected": -225.78790283203125,
      "logps/rejected": -311.5291748046875,
      "loss": 5.1075,
      "margin_dpo/margin_mean": 39.196693420410156,
      "margin_dpo/margin_std": 92.16028594970703,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -31.785404205322266,
      "KL/mean": -65.27703094482422,
      "KL/rejected_KL_mean": -98.7686538696289,
      "KL/std": 72.33475494384766,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.0036729485727846622,
      "fcm_dpo/delta": -0.015634149312973022,
      "fcm_dpo/margin": 66.98324584960938,
      "fcm_dpo/q_t": 0.4399413466453552,
      "grad_norm": 19.249759674072266,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": 1.5020745992660522,
      "logits/rejected": 1.6019726991653442,
      "logps/chosen": -262.37042236328125,
      "logps/ref_chosen": -230.58499145507812,
      "logps/ref_rejected": -229.8074493408203,
      "logps/rejected": -328.57611083984375,
      "loss": 4.7081,
      "margin_dpo/margin_mean": 66.98324584960938,
      "margin_dpo/margin_std": 76.41722106933594,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -36.84132766723633,
      "KL/mean": -65.03240203857422,
      "KL/rejected_KL_mean": -93.22348022460938,
      "KL/std": 76.39358520507812,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.0036614236887544394,
      "fcm_dpo/delta": -0.004164143465459347,
      "fcm_dpo/margin": 56.38215637207031,
      "fcm_dpo/q_t": 0.45011106133461,
      "grad_norm": 23.528684616088867,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": 1.1532700061798096,
      "logits/rejected": 1.185892105102539,
      "logps/chosen": -307.3973693847656,
      "logps/ref_chosen": -270.5559997558594,
      "logps/ref_rejected": -266.328369140625,
      "logps/rejected": -359.5518798828125,
      "loss": 4.8922,
      "margin_dpo/margin_mean": 56.38215637207031,
      "margin_dpo/margin_std": 97.08711242675781,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -36.50991439819336,
      "KL/mean": -64.57628631591797,
      "KL/rejected_KL_mean": -92.64266967773438,
      "KL/std": 71.62165069580078,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.0036603519693017006,
      "fcm_dpo/delta": 0.0012418776750564575,
      "fcm_dpo/margin": 56.13277816772461,
      "fcm_dpo/q_t": 0.44982704520225525,
      "grad_norm": 25.0255126953125,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": 1.0707982778549194,
      "logits/rejected": 1.1598451137542725,
      "logps/chosen": -269.74853515625,
      "logps/ref_chosen": -233.2386016845703,
      "logps/ref_rejected": -258.154541015625,
      "logps/rejected": -350.79718017578125,
      "loss": 4.8869,
      "margin_dpo/margin_mean": 56.13277816772461,
      "margin_dpo/margin_std": 94.2267074584961,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -24.1115779876709,
      "KL/mean": -56.50189971923828,
      "KL/rejected_KL_mean": -88.89224243164062,
      "KL/std": 72.01570129394531,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.0036340218503028154,
      "fcm_dpo/delta": -0.017394088208675385,
      "fcm_dpo/margin": 64.78065490722656,
      "fcm_dpo/q_t": 0.4426022171974182,
      "grad_norm": 20.25372886657715,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": 1.4274934530258179,
      "logits/rejected": 1.4670301675796509,
      "logps/chosen": -292.6577453613281,
      "logps/ref_chosen": -268.5461730957031,
      "logps/ref_rejected": -266.5316467285156,
      "logps/rejected": -355.4238586425781,
      "loss": 4.771,
      "margin_dpo/margin_mean": 64.78064727783203,
      "margin_dpo/margin_std": 88.85862731933594,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -37.14639663696289,
      "KL/mean": -67.29112243652344,
      "KL/rejected_KL_mean": -97.43585205078125,
      "KL/std": 73.50383758544922,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.0036049624904990196,
      "fcm_dpo/delta": -0.003638308960944414,
      "fcm_dpo/margin": 60.289451599121094,
      "fcm_dpo/q_t": 0.44700950384140015,
      "grad_norm": 39.587547302246094,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": 1.3424164056777954,
      "logits/rejected": 1.3738272190093994,
      "logps/chosen": -332.023193359375,
      "logps/ref_chosen": -294.8768005371094,
      "logps/ref_rejected": -281.09771728515625,
      "logps/rejected": -378.5335693359375,
      "loss": 4.8174,
      "margin_dpo/margin_mean": 60.289451599121094,
      "margin_dpo/margin_std": 82.55860137939453,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -49.84102249145508,
      "KL/mean": -74.86270904541016,
      "KL/rejected_KL_mean": -99.88438415527344,
      "KL/std": 77.95404052734375,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.003602408105507493,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.04336929321289,
      "fcm_dpo/q_t": 0.4562312662601471,
      "grad_norm": 23.28908920288086,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": 1.4512102603912354,
      "logits/rejected": 1.4713666439056396,
      "logps/chosen": -362.1387939453125,
      "logps/ref_chosen": -312.2977600097656,
      "logps/ref_rejected": -304.6536560058594,
      "logps/rejected": -404.53802490234375,
      "loss": 4.9822,
      "margin_dpo/margin_mean": 50.04336929321289,
      "margin_dpo/margin_std": 96.0155029296875,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -48.282135009765625,
      "KL/mean": -69.26779174804688,
      "KL/rejected_KL_mean": -90.2534408569336,
      "KL/std": 75.18617248535156,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.003602408105507493,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.971309661865234,
      "fcm_dpo/q_t": 0.46323472261428833,
      "grad_norm": 21.66948890686035,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 1.3172247409820557,
      "logits/rejected": 1.3869045972824097,
      "logps/chosen": -342.4863586425781,
      "logps/ref_chosen": -294.2042236328125,
      "logps/ref_rejected": -296.2102355957031,
      "logps/rejected": -386.4637145996094,
      "loss": 5.0748,
      "margin_dpo/margin_mean": 41.971309661865234,
      "margin_dpo/margin_std": 91.01898193359375,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -38.96791458129883,
      "KL/mean": -65.38485717773438,
      "KL/rejected_KL_mean": -91.80180358886719,
      "KL/std": 73.67510223388672,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.0036032593343406916,
      "fcm_dpo/delta": 0.0002952196809928864,
      "fcm_dpo/margin": 52.83389663696289,
      "fcm_dpo/q_t": 0.45363372564315796,
      "grad_norm": 41.75129699707031,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.3374618291854858,
      "logits/rejected": 1.3522100448608398,
      "logps/chosen": -307.28057861328125,
      "logps/ref_chosen": -268.3126525878906,
      "logps/ref_rejected": -255.5543212890625,
      "logps/rejected": -347.3561096191406,
      "loss": 4.9237,
      "margin_dpo/margin_mean": 52.83389663696289,
      "margin_dpo/margin_std": 86.43743896484375,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -42.476131439208984,
      "KL/mean": -75.61743927001953,
      "KL/rejected_KL_mean": -108.75875854492188,
      "KL/std": 75.02903747558594,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.0035763438791036606,
      "fcm_dpo/delta": -0.013716969639062881,
      "fcm_dpo/margin": 66.28263092041016,
      "fcm_dpo/q_t": 0.4421843886375427,
      "grad_norm": 26.113079071044922,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": 1.3270248174667358,
      "logits/rejected": 1.3678098917007446,
      "logps/chosen": -306.2265930175781,
      "logps/ref_chosen": -263.75042724609375,
      "logps/ref_rejected": -234.5144500732422,
      "logps/rejected": -343.273193359375,
      "loss": 4.769,
      "margin_dpo/margin_mean": 66.28263092041016,
      "margin_dpo/margin_std": 92.22808837890625,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -41.69737243652344,
      "KL/mean": -65.6533203125,
      "KL/rejected_KL_mean": -89.60924530029297,
      "KL/std": 72.80133056640625,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.0035639347042888403,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.911869049072266,
      "fcm_dpo/q_t": 0.45821112394332886,
      "grad_norm": 23.505001068115234,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": 1.5514143705368042,
      "logits/rejected": 1.5452572107315063,
      "logps/chosen": -328.5138244628906,
      "logps/ref_chosen": -286.81646728515625,
      "logps/ref_rejected": -260.4169616699219,
      "logps/rejected": -350.0262145996094,
      "loss": 4.9865,
      "margin_dpo/margin_mean": 47.911869049072266,
      "margin_dpo/margin_std": 83.8459243774414,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -39.54972839355469,
      "KL/mean": -68.1318130493164,
      "KL/rejected_KL_mean": -96.71390533447266,
      "KL/std": 75.41263580322266,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.0035475054755806923,
      "fcm_dpo/delta": -0.015461431816220284,
      "fcm_dpo/margin": 57.164180755615234,
      "fcm_dpo/q_t": 0.4506651759147644,
      "grad_norm": 26.11410903930664,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": 1.2712123394012451,
      "logits/rejected": 1.271366834640503,
      "logps/chosen": -332.86065673828125,
      "logps/ref_chosen": -293.3109130859375,
      "logps/ref_rejected": -264.1449279785156,
      "logps/rejected": -360.85882568359375,
      "loss": 4.8942,
      "margin_dpo/margin_mean": 57.1641845703125,
      "margin_dpo/margin_std": 92.12812805175781,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -59.39787292480469,
      "KL/mean": -83.51630401611328,
      "KL/rejected_KL_mean": -107.6347427368164,
      "KL/std": 72.62464904785156,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.0035094409249722958,
      "fcm_dpo/delta": -0.005068112164735794,
      "fcm_dpo/margin": 48.23686981201172,
      "fcm_dpo/q_t": 0.4587838351726532,
      "grad_norm": 32.70136260986328,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 1.165753722190857,
      "logits/rejected": 1.2550374269485474,
      "logps/chosen": -329.15423583984375,
      "logps/ref_chosen": -269.75634765625,
      "logps/ref_rejected": -241.08438110351562,
      "logps/rejected": -348.7191162109375,
      "loss": 5.013,
      "margin_dpo/margin_mean": 48.23686981201172,
      "margin_dpo/margin_std": 94.28709411621094,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -43.197303771972656,
      "KL/mean": -67.86194610595703,
      "KL/rejected_KL_mean": -92.52659606933594,
      "KL/std": 72.93277740478516,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.0035058800131082535,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.32929992675781,
      "fcm_dpo/q_t": 0.457876592874527,
      "grad_norm": 19.66707992553711,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": 1.2120617628097534,
      "logits/rejected": 1.2428627014160156,
      "logps/chosen": -280.569091796875,
      "logps/ref_chosen": -237.37179565429688,
      "logps/ref_rejected": -253.5463409423828,
      "logps/rejected": -346.07293701171875,
      "loss": 4.973,
      "margin_dpo/margin_mean": 49.32929611206055,
      "margin_dpo/margin_std": 84.10586547851562,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -52.6077880859375,
      "KL/mean": -80.27420806884766,
      "KL/rejected_KL_mean": -107.94062042236328,
      "KL/std": 73.11258697509766,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.003505683969706297,
      "fcm_dpo/delta": -0.0005592982633970678,
      "fcm_dpo/margin": 55.332847595214844,
      "fcm_dpo/q_t": 0.4528258442878723,
      "grad_norm": 19.880237579345703,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": 1.0469619035720825,
      "logits/rejected": 1.097562313079834,
      "logps/chosen": -313.9154357910156,
      "logps/ref_chosen": -261.3076477050781,
      "logps/ref_rejected": -252.3170166015625,
      "logps/rejected": -360.25762939453125,
      "loss": 4.922,
      "margin_dpo/margin_mean": 55.332847595214844,
      "margin_dpo/margin_std": 95.31304931640625,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -30.250011444091797,
      "KL/mean": -63.98081588745117,
      "KL/rejected_KL_mean": -97.71161651611328,
      "KL/std": 75.55657958984375,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.0035062034148722887,
      "fcm_dpo/delta": 0.0010792623506858945,
      "fcm_dpo/margin": 67.46160888671875,
      "fcm_dpo/q_t": 0.442409485578537,
      "grad_norm": 26.25409507751465,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": 1.2521647214889526,
      "logits/rejected": 1.2221202850341797,
      "logps/chosen": -299.0217590332031,
      "logps/ref_chosen": -268.7717590332031,
      "logps/ref_rejected": -232.3352813720703,
      "logps/rejected": -330.04693603515625,
      "loss": 4.7557,
      "margin_dpo/margin_mean": 67.46160888671875,
      "margin_dpo/margin_std": 89.89640045166016,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -42.991973876953125,
      "KL/mean": -75.847412109375,
      "KL/rejected_KL_mean": -108.70285034179688,
      "KL/std": 77.86981201171875,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.00349692371673882,
      "fcm_dpo/delta": -0.010045051574707031,
      "fcm_dpo/margin": 65.71089935302734,
      "fcm_dpo/q_t": 0.4440435767173767,
      "grad_norm": 26.751659393310547,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": 1.1007888317108154,
      "logits/rejected": 1.0614851713180542,
      "logps/chosen": -336.6051025390625,
      "logps/ref_chosen": -293.6131286621094,
      "logps/ref_rejected": -281.87738037109375,
      "logps/rejected": -390.5802001953125,
      "loss": 4.7895,
      "margin_dpo/margin_mean": 65.71089935302734,
      "margin_dpo/margin_std": 92.96116638183594,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -44.959617614746094,
      "KL/mean": -72.2796859741211,
      "KL/rejected_KL_mean": -99.59976196289062,
      "KL/std": 75.87923431396484,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.003476228564977646,
      "fcm_dpo/delta": -0.0021849393378943205,
      "fcm_dpo/margin": 54.64013671875,
      "fcm_dpo/q_t": 0.4537637233734131,
      "grad_norm": 25.92624855041504,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 1.0750155448913574,
      "logits/rejected": 1.192997694015503,
      "logps/chosen": -296.7166748046875,
      "logps/ref_chosen": -251.75704956054688,
      "logps/ref_rejected": -255.76441955566406,
      "logps/rejected": -355.36419677734375,
      "loss": 4.9232,
      "margin_dpo/margin_mean": 54.64013671875,
      "margin_dpo/margin_std": 89.72686767578125,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -61.31831741333008,
      "KL/mean": -85.0216064453125,
      "KL/rejected_KL_mean": -108.72489929199219,
      "KL/std": 78.9588394165039,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.003469163551926613,
      "fcm_dpo/delta": -0.003870386863127351,
      "fcm_dpo/margin": 47.406585693359375,
      "fcm_dpo/q_t": 0.4600060284137726,
      "grad_norm": 26.93791961669922,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 1.3836538791656494,
      "logits/rejected": 1.4956365823745728,
      "logps/chosen": -367.9659118652344,
      "logps/ref_chosen": -306.64764404296875,
      "logps/ref_rejected": -301.4248352050781,
      "logps/rejected": -410.14971923828125,
      "loss": 5.0209,
      "margin_dpo/margin_mean": 47.406585693359375,
      "margin_dpo/margin_std": 91.28639221191406,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -41.23699951171875,
      "KL/mean": -70.06922912597656,
      "KL/rejected_KL_mean": -98.90147399902344,
      "KL/std": 75.11622619628906,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.003461285727098584,
      "fcm_dpo/delta": -0.003375059925019741,
      "fcm_dpo/margin": 57.664485931396484,
      "fcm_dpo/q_t": 0.45127934217453003,
      "grad_norm": 29.541269302368164,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": 1.643196702003479,
      "logits/rejected": 1.721242904663086,
      "logps/chosen": -303.9059143066406,
      "logps/ref_chosen": -262.6689147949219,
      "logps/ref_rejected": -260.8409118652344,
      "logps/rejected": -359.7424011230469,
      "loss": 4.8768,
      "margin_dpo/margin_mean": 57.664485931396484,
      "margin_dpo/margin_std": 84.35206604003906,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -45.57520294189453,
      "KL/mean": -70.36051940917969,
      "KL/rejected_KL_mean": -95.14582824707031,
      "KL/std": 75.61854553222656,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.0034531166311353445,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.57064437866211,
      "fcm_dpo/q_t": 0.4582403898239136,
      "grad_norm": 30.22859764099121,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": 1.4821414947509766,
      "logits/rejected": 1.4963527917861938,
      "logps/chosen": -345.2011413574219,
      "logps/ref_chosen": -299.6259765625,
      "logps/ref_rejected": -261.2701416015625,
      "logps/rejected": -356.41595458984375,
      "loss": 4.9888,
      "margin_dpo/margin_mean": 49.57063674926758,
      "margin_dpo/margin_std": 89.47786712646484,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -47.18052291870117,
      "KL/mean": -78.13189697265625,
      "KL/rejected_KL_mean": -109.0832748413086,
      "KL/std": 77.5177001953125,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.0034431039821356535,
      "fcm_dpo/delta": -0.01458237785845995,
      "fcm_dpo/margin": 61.90274429321289,
      "fcm_dpo/q_t": 0.44810208678245544,
      "grad_norm": 31.48516845703125,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": 1.5791652202606201,
      "logits/rejected": 1.6656800508499146,
      "logps/chosen": -338.1309509277344,
      "logps/ref_chosen": -290.950439453125,
      "logps/ref_rejected": -255.15374755859375,
      "logps/rejected": -364.23699951171875,
      "loss": 4.8479,
      "margin_dpo/margin_mean": 61.902748107910156,
      "margin_dpo/margin_std": 92.29537963867188,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -40.52460861206055,
      "KL/mean": -64.14592742919922,
      "KL/rejected_KL_mean": -87.76724243164062,
      "KL/std": 73.77255249023438,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.003412774996832013,
      "fcm_dpo/delta": -0.00021404886501841247,
      "fcm_dpo/margin": 47.24263000488281,
      "fcm_dpo/q_t": 0.4605637788772583,
      "grad_norm": 34.9588508605957,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": 1.3149529695510864,
      "logits/rejected": 1.3967843055725098,
      "logps/chosen": -313.7904968261719,
      "logps/ref_chosen": -273.2658996582031,
      "logps/ref_rejected": -263.9664306640625,
      "logps/rejected": -351.733642578125,
      "loss": 5.0068,
      "margin_dpo/margin_mean": 47.24263000488281,
      "margin_dpo/margin_std": 78.90423583984375,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -56.19938659667969,
      "KL/mean": -86.85615539550781,
      "KL/rejected_KL_mean": -117.51292419433594,
      "KL/std": 76.91390991210938,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.0033779798541218042,
      "fcm_dpo/delta": -0.016539718955755234,
      "fcm_dpo/margin": 61.31352615356445,
      "fcm_dpo/q_t": 0.4493855834007263,
      "grad_norm": 15.324016571044922,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": 1.4744255542755127,
      "logits/rejected": 1.5635322332382202,
      "logps/chosen": -318.8815612792969,
      "logps/ref_chosen": -262.6821594238281,
      "logps/ref_rejected": -226.46859741210938,
      "logps/rejected": -343.9815368652344,
      "loss": 4.8618,
      "margin_dpo/margin_mean": 61.313533782958984,
      "margin_dpo/margin_std": 86.9046630859375,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -54.128910064697266,
      "KL/mean": -80.29754638671875,
      "KL/rejected_KL_mean": -106.46620178222656,
      "KL/std": 76.21686553955078,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.0033676268067210913,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.3372802734375,
      "fcm_dpo/q_t": 0.4568735361099243,
      "grad_norm": 22.626510620117188,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": 1.3744761943817139,
      "logits/rejected": 1.4067660570144653,
      "logps/chosen": -317.9534912109375,
      "logps/ref_chosen": -263.8246154785156,
      "logps/ref_rejected": -237.9121551513672,
      "logps/rejected": -344.37835693359375,
      "loss": 4.9631,
      "margin_dpo/margin_mean": 52.337276458740234,
      "margin_dpo/margin_std": 88.17259979248047,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -46.68338394165039,
      "KL/mean": -69.73628997802734,
      "KL/rejected_KL_mean": -92.78919982910156,
      "KL/std": 76.2786865234375,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.0033676268067210913,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.10582733154297,
      "fcm_dpo/q_t": 0.4620835781097412,
      "grad_norm": 21.343595504760742,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 1.3980164527893066,
      "logits/rejected": 1.4621376991271973,
      "logps/chosen": -324.8367004394531,
      "logps/ref_chosen": -278.1533203125,
      "logps/ref_rejected": -263.39691162109375,
      "logps/rejected": -356.1861267089844,
      "loss": 5.0345,
      "margin_dpo/margin_mean": 46.1058235168457,
      "margin_dpo/margin_std": 85.6088638305664,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -51.23295593261719,
      "KL/mean": -82.2686767578125,
      "KL/rejected_KL_mean": -113.30438995361328,
      "KL/std": 79.2357406616211,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.003365444950759411,
      "fcm_dpo/delta": -0.0017438313225284219,
      "fcm_dpo/margin": 62.071434020996094,
      "fcm_dpo/q_t": 0.4490716755390167,
      "grad_norm": 25.503881454467773,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": 1.4671545028686523,
      "logits/rejected": 1.5145776271820068,
      "logps/chosen": -369.6880187988281,
      "logps/ref_chosen": -318.455078125,
      "logps/ref_rejected": -288.5691833496094,
      "logps/rejected": -401.87353515625,
      "loss": 4.8487,
      "margin_dpo/margin_mean": 62.071434020996094,
      "margin_dpo/margin_std": 89.12194061279297,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -54.86925506591797,
      "KL/mean": -83.15248107910156,
      "KL/rejected_KL_mean": -111.43571472167969,
      "KL/std": 77.8895263671875,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.0033640326000750065,
      "fcm_dpo/delta": 0.0010901456698775291,
      "fcm_dpo/margin": 56.56646728515625,
      "fcm_dpo/q_t": 0.4535086452960968,
      "grad_norm": 29.125232696533203,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 1.4809000492095947,
      "logits/rejected": 1.5150684118270874,
      "logps/chosen": -339.0584411621094,
      "logps/ref_chosen": -284.1891784667969,
      "logps/ref_rejected": -247.94589233398438,
      "logps/rejected": -359.381591796875,
      "loss": 4.9258,
      "margin_dpo/margin_mean": 56.56647491455078,
      "margin_dpo/margin_std": 92.23213195800781,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -56.2276725769043,
      "KL/mean": -74.49805450439453,
      "KL/rejected_KL_mean": -92.76841735839844,
      "KL/std": 77.12947082519531,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.0033657194580882788,
      "fcm_dpo/delta": -0.0004365870845504105,
      "fcm_dpo/margin": 36.54075241088867,
      "fcm_dpo/q_t": 0.4701828062534332,
      "grad_norm": 27.386497497558594,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 1.4092917442321777,
      "logits/rejected": 1.3878355026245117,
      "logps/chosen": -359.5357666015625,
      "logps/ref_chosen": -303.30810546875,
      "logps/ref_rejected": -260.1866455078125,
      "logps/rejected": -352.955078125,
      "loss": 5.174,
      "margin_dpo/margin_mean": 36.540748596191406,
      "margin_dpo/margin_std": 93.19281005859375,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -46.157508850097656,
      "KL/mean": -78.2077865600586,
      "KL/rejected_KL_mean": -110.25807189941406,
      "KL/std": 77.94124603271484,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.0033579848241060972,
      "fcm_dpo/delta": -0.006571314297616482,
      "fcm_dpo/margin": 64.1005630493164,
      "fcm_dpo/q_t": 0.4474737048149109,
      "grad_norm": 19.78268051147461,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": 1.358485221862793,
      "logits/rejected": 1.543984055519104,
      "logps/chosen": -292.9646301269531,
      "logps/ref_chosen": -246.80709838867188,
      "logps/ref_rejected": -253.13223266601562,
      "logps/rejected": -363.39031982421875,
      "loss": 4.8254,
      "margin_dpo/margin_mean": 64.1005630493164,
      "margin_dpo/margin_std": 87.1874008178711,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -53.06743621826172,
      "KL/mean": -85.31784057617188,
      "KL/rejected_KL_mean": -117.56826782226562,
      "KL/std": 77.3055648803711,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.003341167699545622,
      "fcm_dpo/delta": -0.011824660934507847,
      "fcm_dpo/margin": 64.50082397460938,
      "fcm_dpo/q_t": 0.447360634803772,
      "grad_norm": 19.483903884887695,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": 1.1569479703903198,
      "logits/rejected": 1.2405478954315186,
      "logps/chosen": -314.66888427734375,
      "logps/ref_chosen": -261.6014709472656,
      "logps/ref_rejected": -262.74761962890625,
      "logps/rejected": -380.31585693359375,
      "loss": 4.8215,
      "margin_dpo/margin_mean": 64.50082397460938,
      "margin_dpo/margin_std": 86.99037170410156,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -53.94029998779297,
      "KL/mean": -74.45891571044922,
      "KL/rejected_KL_mean": -94.9775390625,
      "KL/std": 77.73728942871094,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.0033155360724776983,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.0372200012207,
      "fcm_dpo/q_t": 0.4669164717197418,
      "grad_norm": 22.525739669799805,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": 1.3699513673782349,
      "logits/rejected": 1.496642827987671,
      "logps/chosen": -306.0887451171875,
      "logps/ref_chosen": -252.14840698242188,
      "logps/ref_rejected": -244.1017608642578,
      "logps/rejected": -339.07928466796875,
      "loss": 5.133,
      "margin_dpo/margin_mean": 41.03722381591797,
      "margin_dpo/margin_std": 98.45577239990234,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -58.49189758300781,
      "KL/mean": -87.15135192871094,
      "KL/rejected_KL_mean": -115.81080627441406,
      "KL/std": 77.88475799560547,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.0033155360724776983,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.31890869140625,
      "fcm_dpo/q_t": 0.45356622338294983,
      "grad_norm": 27.57345962524414,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": 1.464728593826294,
      "logits/rejected": 1.5862468481063843,
      "logps/chosen": -344.6544494628906,
      "logps/ref_chosen": -286.16253662109375,
      "logps/ref_rejected": -272.3275146484375,
      "logps/rejected": -388.1383361816406,
      "loss": 4.915,
      "margin_dpo/margin_mean": 57.31890869140625,
      "margin_dpo/margin_std": 90.1369857788086,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -51.041778564453125,
      "KL/mean": -75.57719421386719,
      "KL/rejected_KL_mean": -100.11262512207031,
      "KL/std": 74.35294342041016,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.0033155360724776983,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.07083511352539,
      "fcm_dpo/q_t": 0.46016615629196167,
      "grad_norm": 35.502197265625,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": 1.3562461137771606,
      "logits/rejected": 1.4173487424850464,
      "logps/chosen": -249.19454956054688,
      "logps/ref_chosen": -198.1527862548828,
      "logps/ref_rejected": -209.80569458007812,
      "logps/rejected": -309.9183044433594,
      "loss": 5.0109,
      "margin_dpo/margin_mean": 49.07083511352539,
      "margin_dpo/margin_std": 86.48873901367188,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -60.71533966064453,
      "KL/mean": -88.5450210571289,
      "KL/rejected_KL_mean": -116.37471771240234,
      "KL/std": 76.30548095703125,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.0033164650667458773,
      "fcm_dpo/delta": 0.0009338111849501729,
      "fcm_dpo/margin": 55.65937805175781,
      "fcm_dpo/q_t": 0.4551228880882263,
      "grad_norm": 24.87114143371582,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": 1.377108097076416,
      "logits/rejected": 1.4484338760375977,
      "logps/chosen": -317.22247314453125,
      "logps/ref_chosen": -256.50714111328125,
      "logps/ref_rejected": -237.40769958496094,
      "logps/rejected": -353.78240966796875,
      "loss": 4.9371,
      "margin_dpo/margin_mean": 55.659385681152344,
      "margin_dpo/margin_std": 90.34854888916016,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -45.229034423828125,
      "KL/mean": -77.3415298461914,
      "KL/rejected_KL_mean": -109.45403289794922,
      "KL/std": 77.7447509765625,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.003318003611639142,
      "fcm_dpo/delta": -0.000413521978771314,
      "fcm_dpo/margin": 64.2249984741211,
      "fcm_dpo/q_t": 0.44800660014152527,
      "grad_norm": 23.181842803955078,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": 1.5458502769470215,
      "logits/rejected": 1.5060445070266724,
      "logps/chosen": -337.67626953125,
      "logps/ref_chosen": -292.447265625,
      "logps/ref_rejected": -255.47872924804688,
      "logps/rejected": -364.9327392578125,
      "loss": 4.8365,
      "margin_dpo/margin_mean": 64.22499084472656,
      "margin_dpo/margin_std": 91.29795837402344,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -51.896820068359375,
      "KL/mean": -80.55307006835938,
      "KL/rejected_KL_mean": -109.2093276977539,
      "KL/std": 78.42959594726562,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.0033169162925332785,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.31250762939453,
      "fcm_dpo/q_t": 0.4535343647003174,
      "grad_norm": 23.673067092895508,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": 1.5036903619766235,
      "logits/rejected": 1.4851536750793457,
      "logps/chosen": -345.65679931640625,
      "logps/ref_chosen": -293.7599792480469,
      "logps/ref_rejected": -259.4142761230469,
      "logps/rejected": -368.62359619140625,
      "loss": 4.9159,
      "margin_dpo/margin_mean": 57.31250762939453,
      "margin_dpo/margin_std": 91.00225830078125,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -55.825435638427734,
      "KL/mean": -82.72126770019531,
      "KL/rejected_KL_mean": -109.61709594726562,
      "KL/std": 76.8468017578125,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.003317145397886634,
      "fcm_dpo/delta": -0.002289965283125639,
      "fcm_dpo/margin": 53.791656494140625,
      "fcm_dpo/q_t": 0.4563436508178711,
      "grad_norm": 22.19236946105957,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 1.4091564416885376,
      "logits/rejected": 1.3725320100784302,
      "logps/chosen": -356.73919677734375,
      "logps/ref_chosen": -300.91375732421875,
      "logps/ref_rejected": -256.4080810546875,
      "logps/rejected": -366.025146484375,
      "loss": 4.9406,
      "margin_dpo/margin_mean": 53.791656494140625,
      "margin_dpo/margin_std": 79.75872039794922,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -42.27953338623047,
      "KL/mean": -78.18563842773438,
      "KL/rejected_KL_mean": -114.09175109863281,
      "KL/std": 79.39582061767578,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.0033092102967202663,
      "fcm_dpo/delta": -0.0014256327413022518,
      "fcm_dpo/margin": 71.81221771240234,
      "fcm_dpo/q_t": 0.442095011472702,
      "grad_norm": 26.42469596862793,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 1.0468591451644897,
      "logits/rejected": 1.137787938117981,
      "logps/chosen": -298.83984375,
      "logps/ref_chosen": -256.5603332519531,
      "logps/ref_rejected": -271.043701171875,
      "logps/rejected": -385.1354064941406,
      "loss": 4.7628,
      "margin_dpo/margin_mean": 71.81221771240234,
      "margin_dpo/margin_std": 100.29558563232422,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -50.225826263427734,
      "KL/mean": -81.47354125976562,
      "KL/rejected_KL_mean": -112.72126770019531,
      "KL/std": 75.64615631103516,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.0033045364543795586,
      "fcm_dpo/delta": -0.001095408108085394,
      "fcm_dpo/margin": 62.495445251464844,
      "fcm_dpo/q_t": 0.4496321678161621,
      "grad_norm": 24.555025100708008,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": 1.427524209022522,
      "logits/rejected": 1.352337121963501,
      "logps/chosen": -344.9263000488281,
      "logps/ref_chosen": -294.700439453125,
      "logps/ref_rejected": -239.67922973632812,
      "logps/rejected": -352.4005432128906,
      "loss": 4.8557,
      "margin_dpo/margin_mean": 62.49544906616211,
      "margin_dpo/margin_std": 89.48477172851562,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -47.10271453857422,
      "KL/mean": -81.6268539428711,
      "KL/rejected_KL_mean": -116.15098571777344,
      "KL/std": 81.02608489990234,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.0033049543853849173,
      "fcm_dpo/delta": -0.0009921613382175565,
      "fcm_dpo/margin": 69.04827880859375,
      "fcm_dpo/q_t": 0.4444260597229004,
      "grad_norm": 20.71644401550293,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": 1.3383458852767944,
      "logits/rejected": 1.3542650938034058,
      "logps/chosen": -340.778564453125,
      "logps/ref_chosen": -293.67584228515625,
      "logps/ref_rejected": -260.85675048828125,
      "logps/rejected": -377.0077209472656,
      "loss": 4.7798,
      "margin_dpo/margin_mean": 69.04827880859375,
      "margin_dpo/margin_std": 91.94550323486328,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -68.14653015136719,
      "KL/mean": -93.97543334960938,
      "KL/rejected_KL_mean": -119.80433654785156,
      "KL/std": 76.39247131347656,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.003301552962511778,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.65779495239258,
      "fcm_dpo/q_t": 0.45839154720306396,
      "grad_norm": 21.557758331298828,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": 1.1817994117736816,
      "logits/rejected": 1.2469911575317383,
      "logps/chosen": -347.04193115234375,
      "logps/ref_chosen": -278.8953857421875,
      "logps/ref_rejected": -283.8898620605469,
      "logps/rejected": -403.6941833496094,
      "loss": 4.9907,
      "margin_dpo/margin_mean": 51.65779495239258,
      "margin_dpo/margin_std": 91.19343566894531,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -45.47368240356445,
      "KL/mean": -78.93256378173828,
      "KL/rejected_KL_mean": -112.39144134521484,
      "KL/std": 79.95598602294922,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.003301856340840459,
      "fcm_dpo/delta": 0.000918581907171756,
      "fcm_dpo/margin": 66.91775512695312,
      "fcm_dpo/q_t": 0.44612759351730347,
      "grad_norm": 18.9466609954834,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": 1.4341872930526733,
      "logits/rejected": 1.5067832469940186,
      "logps/chosen": -311.11761474609375,
      "logps/ref_chosen": -265.6439208984375,
      "logps/ref_rejected": -255.40138244628906,
      "logps/rejected": -367.7928466796875,
      "loss": 4.8043,
      "margin_dpo/margin_mean": 66.91775512695312,
      "margin_dpo/margin_std": 91.59461975097656,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -55.38150405883789,
      "KL/mean": -86.90545654296875,
      "KL/rejected_KL_mean": -118.42941284179688,
      "KL/std": 79.66348266601562,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.0032985431607812643,
      "fcm_dpo/delta": -0.002352825365960598,
      "fcm_dpo/margin": 63.047889709472656,
      "fcm_dpo/q_t": 0.44913098216056824,
      "grad_norm": 24.762001037597656,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": 1.3500516414642334,
      "logits/rejected": 1.3686127662658691,
      "logps/chosen": -335.60601806640625,
      "logps/ref_chosen": -280.2244873046875,
      "logps/ref_rejected": -248.72366333007812,
      "logps/rejected": -367.1530456542969,
      "loss": 4.8472,
      "margin_dpo/margin_mean": 63.04789733886719,
      "margin_dpo/margin_std": 88.69403076171875,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -50.35454177856445,
      "KL/mean": -83.74942779541016,
      "KL/rejected_KL_mean": -117.14432525634766,
      "KL/std": 79.27290344238281,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.003283771686255932,
      "fcm_dpo/delta": -0.0064624096266925335,
      "fcm_dpo/margin": 66.7897720336914,
      "fcm_dpo/q_t": 0.4464291036128998,
      "grad_norm": 19.767654418945312,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": 1.3092267513275146,
      "logits/rejected": 1.3245837688446045,
      "logps/chosen": -343.29119873046875,
      "logps/ref_chosen": -292.9366760253906,
      "logps/ref_rejected": -261.88763427734375,
      "logps/rejected": -379.0319519042969,
      "loss": 4.8001,
      "margin_dpo/margin_mean": 66.7897720336914,
      "margin_dpo/margin_std": 86.00994110107422,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -42.20712661743164,
      "KL/mean": -79.65823364257812,
      "KL/rejected_KL_mean": -117.1093521118164,
      "KL/std": 80.84276580810547,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.003260695608332753,
      "fcm_dpo/delta": -0.01176365464925766,
      "fcm_dpo/margin": 74.90223693847656,
      "fcm_dpo/q_t": 0.44028425216674805,
      "grad_norm": 23.74356460571289,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": 1.098871111869812,
      "logits/rejected": 1.1739897727966309,
      "logps/chosen": -327.1128845214844,
      "logps/ref_chosen": -284.90576171875,
      "logps/ref_rejected": -256.59197998046875,
      "logps/rejected": -373.7013244628906,
      "loss": 4.714,
      "margin_dpo/margin_mean": 74.90223693847656,
      "margin_dpo/margin_std": 86.85099029541016,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -51.827266693115234,
      "KL/mean": -77.18627166748047,
      "KL/rejected_KL_mean": -102.54527282714844,
      "KL/std": 77.58454895019531,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.0032486312557011843,
      "fcm_dpo/delta": -0.001436402089893818,
      "fcm_dpo/margin": 50.71799850463867,
      "fcm_dpo/q_t": 0.45973673462867737,
      "grad_norm": 24.99881935119629,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": 1.3955183029174805,
      "logits/rejected": 1.4104790687561035,
      "logps/chosen": -327.7431945800781,
      "logps/ref_chosen": -275.91595458984375,
      "logps/ref_rejected": -264.79736328125,
      "logps/rejected": -367.3426208496094,
      "loss": 5.0121,
      "margin_dpo/margin_mean": 50.71799850463867,
      "margin_dpo/margin_std": 93.11578369140625,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -45.66680145263672,
      "KL/mean": -78.9039306640625,
      "KL/rejected_KL_mean": -112.14105224609375,
      "KL/std": 79.91596221923828,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.003246298525482416,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.47425079345703,
      "fcm_dpo/q_t": 0.4475272297859192,
      "grad_norm": 16.57088279724121,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 1.0823299884796143,
      "logits/rejected": 1.1458933353424072,
      "logps/chosen": -290.60882568359375,
      "logps/ref_chosen": -244.9420166015625,
      "logps/ref_rejected": -235.61978149414062,
      "logps/rejected": -347.76080322265625,
      "loss": 4.8333,
      "margin_dpo/margin_mean": 66.47425079345703,
      "margin_dpo/margin_std": 98.18226623535156,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -51.80181121826172,
      "KL/mean": -79.74626159667969,
      "KL/rejected_KL_mean": -107.69070434570312,
      "KL/std": 78.42607116699219,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.003246288513764739,
      "fcm_dpo/delta": -3.09517745336052e-05,
      "fcm_dpo/margin": 55.888893127441406,
      "fcm_dpo/q_t": 0.45555856823921204,
      "grad_norm": 27.30246925354004,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 1.2252428531646729,
      "logits/rejected": 1.2251687049865723,
      "logps/chosen": -341.15167236328125,
      "logps/ref_chosen": -289.3498229980469,
      "logps/ref_rejected": -243.70205688476562,
      "logps/rejected": -351.39276123046875,
      "loss": 4.9432,
      "margin_dpo/margin_mean": 55.888893127441406,
      "margin_dpo/margin_std": 90.28512573242188,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -48.88127136230469,
      "KL/mean": -84.29096221923828,
      "KL/rejected_KL_mean": -119.7006607055664,
      "KL/std": 81.74325561523438,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.00322899641469121,
      "fcm_dpo/delta": -0.008873400278389454,
      "fcm_dpo/margin": 70.81936645507812,
      "fcm_dpo/q_t": 0.4441310167312622,
      "grad_norm": 25.090999603271484,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": 1.4054909944534302,
      "logits/rejected": 1.3996577262878418,
      "logps/chosen": -370.53472900390625,
      "logps/ref_chosen": -321.65350341796875,
      "logps/ref_rejected": -282.73773193359375,
      "logps/rejected": -402.4383544921875,
      "loss": 4.7812,
      "margin_dpo/margin_mean": 70.8193588256836,
      "margin_dpo/margin_std": 92.59941864013672,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -47.69096755981445,
      "KL/mean": -81.39285278320312,
      "KL/rejected_KL_mean": -115.09471130371094,
      "KL/std": 83.05015563964844,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.003205117303878069,
      "fcm_dpo/delta": -0.008065128698945045,
      "fcm_dpo/margin": 67.40376281738281,
      "fcm_dpo/q_t": 0.447279155254364,
      "grad_norm": 25.204622268676758,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": 1.236878514289856,
      "logits/rejected": 1.2037667036056519,
      "logps/chosen": -357.4414367675781,
      "logps/ref_chosen": -309.75048828125,
      "logps/ref_rejected": -267.41925048828125,
      "logps/rejected": -382.5139465332031,
      "loss": 4.826,
      "margin_dpo/margin_mean": 67.40376281738281,
      "margin_dpo/margin_std": 95.259521484375,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -53.15375900268555,
      "KL/mean": -75.93586730957031,
      "KL/rejected_KL_mean": -98.71798706054688,
      "KL/std": 80.3330078125,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.003202526131644845,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.564208984375,
      "fcm_dpo/q_t": 0.46444591879844666,
      "grad_norm": 27.224382400512695,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 1.5045759677886963,
      "logits/rejected": 1.4927321672439575,
      "logps/chosen": -364.1478576660156,
      "logps/ref_chosen": -310.9941101074219,
      "logps/ref_rejected": -279.94915771484375,
      "logps/rejected": -378.6671447753906,
      "loss": 5.0801,
      "margin_dpo/margin_mean": 45.564208984375,
      "margin_dpo/margin_std": 94.23603820800781,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -52.99443817138672,
      "KL/mean": -81.1201400756836,
      "KL/rejected_KL_mean": -109.24583435058594,
      "KL/std": 78.0033950805664,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.0032018551137298346,
      "fcm_dpo/delta": -0.002096818992868066,
      "fcm_dpo/margin": 56.25139236450195,
      "fcm_dpo/q_t": 0.4560784101486206,
      "grad_norm": 44.572628021240234,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": 1.4920945167541504,
      "logits/rejected": 1.428406000137329,
      "logps/chosen": -362.1778564453125,
      "logps/ref_chosen": -309.18341064453125,
      "logps/ref_rejected": -256.9963073730469,
      "logps/rejected": -366.24212646484375,
      "loss": 4.9514,
      "margin_dpo/margin_mean": 56.25139236450195,
      "margin_dpo/margin_std": 93.5705795288086,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -54.731571197509766,
      "KL/mean": -82.27603149414062,
      "KL/rejected_KL_mean": -109.82049560546875,
      "KL/std": 80.75271606445312,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.003196127712726593,
      "fcm_dpo/delta": -0.003228014102205634,
      "fcm_dpo/margin": 55.088905334472656,
      "fcm_dpo/q_t": 0.4570787847042084,
      "grad_norm": 21.084060668945312,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": 1.4325295686721802,
      "logits/rejected": 1.5137896537780762,
      "logps/chosen": -337.0067443847656,
      "logps/ref_chosen": -282.275146484375,
      "logps/ref_rejected": -255.17941284179688,
      "logps/rejected": -364.9999084472656,
      "loss": 4.9756,
      "margin_dpo/margin_mean": 55.08890914916992,
      "margin_dpo/margin_std": 95.9913330078125,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -49.95620346069336,
      "KL/mean": -75.90070343017578,
      "KL/rejected_KL_mean": -101.8451919555664,
      "KL/std": 75.8688735961914,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.0031889122910797596,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.888980865478516,
      "fcm_dpo/q_t": 0.4596155285835266,
      "grad_norm": 20.545408248901367,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": 1.326216697692871,
      "logits/rejected": 1.3728169202804565,
      "logps/chosen": -342.83343505859375,
      "logps/ref_chosen": -292.8772277832031,
      "logps/ref_rejected": -266.78509521484375,
      "logps/rejected": -368.6302795410156,
      "loss": 4.9963,
      "margin_dpo/margin_mean": 51.88898468017578,
      "margin_dpo/margin_std": 90.05303192138672,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -49.76266860961914,
      "KL/mean": -75.59111022949219,
      "KL/rejected_KL_mean": -101.41956329345703,
      "KL/std": 80.05342864990234,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.00318842101842165,
      "fcm_dpo/delta": -0.0007711012731306255,
      "fcm_dpo/margin": 51.65688705444336,
      "fcm_dpo/q_t": 0.4597935676574707,
      "grad_norm": 24.64509391784668,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": 1.25086510181427,
      "logits/rejected": 1.317776083946228,
      "logps/chosen": -322.6563415527344,
      "logps/ref_chosen": -272.8937072753906,
      "logps/ref_rejected": -293.058349609375,
      "logps/rejected": -394.4779052734375,
      "loss": 5.0013,
      "margin_dpo/margin_mean": 51.65688705444336,
      "margin_dpo/margin_std": 89.84577178955078,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -50.12177658081055,
      "KL/mean": -78.80418395996094,
      "KL/rejected_KL_mean": -107.48658752441406,
      "KL/std": 77.51163482666016,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.0031808498315513134,
      "fcm_dpo/delta": -0.003831584705039859,
      "fcm_dpo/margin": 57.36481857299805,
      "fcm_dpo/q_t": 0.455456405878067,
      "grad_norm": 30.93496322631836,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": 1.2815213203430176,
      "logits/rejected": 1.2952262163162231,
      "logps/chosen": -309.26336669921875,
      "logps/ref_chosen": -259.1416015625,
      "logps/ref_rejected": -229.64996337890625,
      "logps/rejected": -337.13653564453125,
      "loss": 4.9525,
      "margin_dpo/margin_mean": 57.36481475830078,
      "margin_dpo/margin_std": 99.58026123046875,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -58.31438064575195,
      "KL/mean": -85.06442260742188,
      "KL/rejected_KL_mean": -111.81446838378906,
      "KL/std": 81.7717514038086,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.0031771922949701548,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.50007247924805,
      "fcm_dpo/q_t": 0.4586583077907562,
      "grad_norm": 79.52574157714844,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 1.2629585266113281,
      "logits/rejected": 1.1896681785583496,
      "logps/chosen": -354.65234375,
      "logps/ref_chosen": -296.33795166015625,
      "logps/ref_rejected": -283.7359924316406,
      "logps/rejected": -395.5504150390625,
      "loss": 4.9866,
      "margin_dpo/margin_mean": 53.50007247924805,
      "margin_dpo/margin_std": 92.29588317871094,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -53.07571029663086,
      "KL/mean": -86.70103454589844,
      "KL/rejected_KL_mean": -120.32633972167969,
      "KL/std": 79.83550262451172,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.0031771922949701548,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.25062561035156,
      "fcm_dpo/q_t": 0.4478040337562561,
      "grad_norm": 50.429115295410156,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 1.1256216764450073,
      "logits/rejected": 1.1636962890625,
      "logps/chosen": -302.0489501953125,
      "logps/ref_chosen": -248.97320556640625,
      "logps/ref_rejected": -259.5140075683594,
      "logps/rejected": -379.84033203125,
      "loss": 4.8159,
      "margin_dpo/margin_mean": 67.25062561035156,
      "margin_dpo/margin_std": 88.97489929199219,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -39.25403594970703,
      "KL/mean": -78.25004577636719,
      "KL/rejected_KL_mean": -117.24604034423828,
      "KL/std": 81.05953216552734,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.0031646958086639643,
      "fcm_dpo/delta": -0.006866569630801678,
      "fcm_dpo/margin": 77.99199676513672,
      "fcm_dpo/q_t": 0.4397163689136505,
      "grad_norm": 25.52645492553711,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": 1.2023080587387085,
      "logits/rejected": 1.3568530082702637,
      "logps/chosen": -319.92633056640625,
      "logps/ref_chosen": -280.67230224609375,
      "logps/ref_rejected": -272.4359130859375,
      "logps/rejected": -389.68194580078125,
      "loss": 4.7165,
      "margin_dpo/margin_mean": 77.99199676513672,
      "margin_dpo/margin_std": 95.23932647705078,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -47.98863983154297,
      "KL/mean": -76.71472930908203,
      "KL/rejected_KL_mean": -105.44082641601562,
      "KL/std": 78.51671600341797,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.0031597872730344534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.452171325683594,
      "fcm_dpo/q_t": 0.45568060874938965,
      "grad_norm": 29.483720779418945,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": 0.8402943015098572,
      "logits/rejected": 0.8843772411346436,
      "logps/chosen": -315.6470947265625,
      "logps/ref_chosen": -267.658447265625,
      "logps/ref_rejected": -262.9810791015625,
      "logps/rejected": -368.4219055175781,
      "loss": 4.9426,
      "margin_dpo/margin_mean": 57.452171325683594,
      "margin_dpo/margin_std": 92.73298645019531,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -57.155975341796875,
      "KL/mean": -77.0462875366211,
      "KL/rejected_KL_mean": -96.93659973144531,
      "KL/std": 77.19548034667969,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.0031597872730344534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.78061294555664,
      "fcm_dpo/q_t": 0.46932417154312134,
      "grad_norm": 31.698734283447266,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": 1.262492060661316,
      "logits/rejected": 1.442999243736267,
      "logps/chosen": -323.40252685546875,
      "logps/ref_chosen": -266.2465515136719,
      "logps/ref_rejected": -281.1450500488281,
      "logps/rejected": -378.0816345214844,
      "loss": 5.1603,
      "margin_dpo/margin_mean": 39.78061294555664,
      "margin_dpo/margin_std": 99.6649169921875,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -51.0305290222168,
      "KL/mean": -78.44595336914062,
      "KL/rejected_KL_mean": -105.86136627197266,
      "KL/std": 78.32321166992188,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.0031597872730344534,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.830833435058594,
      "fcm_dpo/q_t": 0.4576951861381531,
      "grad_norm": 22.02092742919922,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": 1.3602886199951172,
      "logits/rejected": 1.3560171127319336,
      "logps/chosen": -325.33636474609375,
      "logps/ref_chosen": -274.30584716796875,
      "logps/ref_rejected": -242.94247436523438,
      "logps/rejected": -348.8038330078125,
      "loss": 4.9732,
      "margin_dpo/margin_mean": 54.830833435058594,
      "margin_dpo/margin_std": 93.27059936523438,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -42.66922378540039,
      "KL/mean": -77.46637725830078,
      "KL/rejected_KL_mean": -112.26351928710938,
      "KL/std": 83.31898498535156,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.0031259064562618732,
      "fcm_dpo/delta": -0.017999593168497086,
      "fcm_dpo/margin": 69.59428405761719,
      "fcm_dpo/q_t": 0.4467129111289978,
      "grad_norm": 19.12034797668457,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": 1.3047019243240356,
      "logits/rejected": 1.381069540977478,
      "logps/chosen": -351.9356689453125,
      "logps/ref_chosen": -309.2664489746094,
      "logps/ref_rejected": -298.4182434082031,
      "logps/rejected": -410.6817321777344,
      "loss": 4.8193,
      "margin_dpo/margin_mean": 69.59428405761719,
      "margin_dpo/margin_std": 93.4989013671875,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -46.019737243652344,
      "KL/mean": -78.29544830322266,
      "KL/rejected_KL_mean": -110.57117462158203,
      "KL/std": 79.43971252441406,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.0031087519600987434,
      "fcm_dpo/delta": -0.003769397735595703,
      "fcm_dpo/margin": 64.55143737792969,
      "fcm_dpo/q_t": 0.45103922486305237,
      "grad_norm": 22.891273498535156,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 1.2870815992355347,
      "logits/rejected": 1.3363697528839111,
      "logps/chosen": -310.0406494140625,
      "logps/ref_chosen": -264.0209045410156,
      "logps/ref_rejected": -264.3447570800781,
      "logps/rejected": -374.9159240722656,
      "loss": 4.8774,
      "margin_dpo/margin_mean": 64.55142974853516,
      "margin_dpo/margin_std": 95.06035614013672,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -58.4990234375,
      "KL/mean": -83.7007064819336,
      "KL/rejected_KL_mean": -108.90238952636719,
      "KL/std": 78.46406555175781,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.0030927436891943216,
      "fcm_dpo/delta": -0.005760009400546551,
      "fcm_dpo/margin": 50.40336608886719,
      "fcm_dpo/q_t": 0.4619749188423157,
      "grad_norm": 40.386962890625,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": 1.1393028497695923,
      "logits/rejected": 1.1738042831420898,
      "logps/chosen": -321.7178649902344,
      "logps/ref_chosen": -263.2188415527344,
      "logps/ref_rejected": -236.08546447753906,
      "logps/rejected": -344.9878234863281,
      "loss": 5.0476,
      "margin_dpo/margin_mean": 50.40336608886719,
      "margin_dpo/margin_std": 99.36976623535156,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -48.984588623046875,
      "KL/mean": -80.84336853027344,
      "KL/rejected_KL_mean": -112.7021484375,
      "KL/std": 77.46011352539062,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.0030909592751413584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.717567443847656,
      "fcm_dpo/q_t": 0.4517747759819031,
      "grad_norm": 30.78275489807129,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": 1.2918577194213867,
      "logits/rejected": 1.273925542831421,
      "logps/chosen": -322.782470703125,
      "logps/ref_chosen": -273.79791259765625,
      "logps/ref_rejected": -242.4090118408203,
      "logps/rejected": -355.11114501953125,
      "loss": 4.8904,
      "margin_dpo/margin_mean": 63.717567443847656,
      "margin_dpo/margin_std": 96.11004638671875,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -50.432594299316406,
      "KL/mean": -81.83416748046875,
      "KL/rejected_KL_mean": -113.23574829101562,
      "KL/std": 77.79269409179688,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.0030905301682651043,
      "fcm_dpo/delta": -0.0004629777686204761,
      "fcm_dpo/margin": 62.80315399169922,
      "fcm_dpo/q_t": 0.452594131231308,
      "grad_norm": 25.367996215820312,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 1.3643503189086914,
      "logits/rejected": 1.4356598854064941,
      "logps/chosen": -325.7477722167969,
      "logps/ref_chosen": -275.3151550292969,
      "logps/ref_rejected": -278.1514587402344,
      "logps/rejected": -391.3872375488281,
      "loss": 4.9082,
      "margin_dpo/margin_mean": 62.80315399169922,
      "margin_dpo/margin_std": 100.80440521240234,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -57.311683654785156,
      "KL/mean": -87.48883819580078,
      "KL/rejected_KL_mean": -117.6659927368164,
      "KL/std": 81.78815460205078,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.003088284283876419,
      "fcm_dpo/delta": -0.000990866799838841,
      "fcm_dpo/margin": 60.35430908203125,
      "fcm_dpo/q_t": 0.45448651909828186,
      "grad_norm": 22.163450241088867,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.278942584991455,
      "logits/rejected": 1.2678638696670532,
      "logps/chosen": -332.5110168457031,
      "logps/ref_chosen": -275.1993408203125,
      "logps/ref_rejected": -261.32122802734375,
      "logps/rejected": -378.9871826171875,
      "loss": 4.9093,
      "margin_dpo/margin_mean": 60.354312896728516,
      "margin_dpo/margin_std": 84.11183166503906,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -49.66401672363281,
      "KL/mean": -89.32964324951172,
      "KL/rejected_KL_mean": -128.99525451660156,
      "KL/std": 81.27680206298828,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.0030665306840091944,
      "fcm_dpo/delta": -0.010800527408719063,
      "fcm_dpo/margin": 79.33124542236328,
      "fcm_dpo/q_t": 0.4404911994934082,
      "grad_norm": 22.130102157592773,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": 1.3700909614562988,
      "logits/rejected": 1.3970681428909302,
      "logps/chosen": -349.4917907714844,
      "logps/ref_chosen": -299.8277587890625,
      "logps/ref_rejected": -251.24798583984375,
      "logps/rejected": -380.2432556152344,
      "loss": 4.7074,
      "margin_dpo/margin_mean": 79.33124542236328,
      "margin_dpo/margin_std": 85.76410675048828,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -47.45423889160156,
      "KL/mean": -75.66732788085938,
      "KL/rejected_KL_mean": -103.88041687011719,
      "KL/std": 79.30508422851562,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.003060805145651102,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.4261589050293,
      "fcm_dpo/q_t": 0.45780038833618164,
      "grad_norm": 24.124319076538086,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": 1.2767376899719238,
      "logits/rejected": 1.2665152549743652,
      "logps/chosen": -327.91534423828125,
      "logps/ref_chosen": -280.4610900878906,
      "logps/ref_rejected": -228.34823608398438,
      "logps/rejected": -332.2286376953125,
      "loss": 4.9775,
      "margin_dpo/margin_mean": 56.4261589050293,
      "margin_dpo/margin_std": 98.70948028564453,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -42.189178466796875,
      "KL/mean": -71.13665008544922,
      "KL/rejected_KL_mean": -100.0841064453125,
      "KL/std": 78.60493469238281,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.003060805145651102,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.89493942260742,
      "fcm_dpo/q_t": 0.45673203468322754,
      "grad_norm": 27.59872055053711,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": 1.3638890981674194,
      "logits/rejected": 1.4763792753219604,
      "logps/chosen": -361.7892761230469,
      "logps/ref_chosen": -319.60009765625,
      "logps/ref_rejected": -316.42822265625,
      "logps/rejected": -416.5122985839844,
      "loss": 4.9469,
      "margin_dpo/margin_mean": 57.894935607910156,
      "margin_dpo/margin_std": 89.09048461914062,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -52.89837646484375,
      "KL/mean": -82.95077514648438,
      "KL/rejected_KL_mean": -113.00318145751953,
      "KL/std": 76.66319274902344,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.003062440315261483,
      "fcm_dpo/delta": 0.0006675279000774026,
      "fcm_dpo/margin": 60.10477828979492,
      "fcm_dpo/q_t": 0.4549895226955414,
      "grad_norm": 19.91802978515625,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": 1.0856108665466309,
      "logits/rejected": 1.209949254989624,
      "logps/chosen": -313.3161926269531,
      "logps/ref_chosen": -260.4178466796875,
      "logps/ref_rejected": -270.04498291015625,
      "logps/rejected": -383.04815673828125,
      "loss": 4.9196,
      "margin_dpo/margin_mean": 60.10478210449219,
      "margin_dpo/margin_std": 87.14879608154297,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -53.6054801940918,
      "KL/mean": -83.01910400390625,
      "KL/rejected_KL_mean": -112.43275451660156,
      "KL/std": 79.31015014648438,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.0030635048169642687,
      "fcm_dpo/delta": 0.0008686663932166994,
      "fcm_dpo/margin": 58.8272590637207,
      "fcm_dpo/q_t": 0.4558946490287781,
      "grad_norm": 35.57189178466797,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": 1.373162031173706,
      "logits/rejected": 1.3733060359954834,
      "logps/chosen": -334.3084716796875,
      "logps/ref_chosen": -280.7030029296875,
      "logps/ref_rejected": -250.76385498046875,
      "logps/rejected": -363.19659423828125,
      "loss": 4.9469,
      "margin_dpo/margin_mean": 58.8272590637207,
      "margin_dpo/margin_std": 95.9457778930664,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -47.18061447143555,
      "KL/mean": -74.65392303466797,
      "KL/rejected_KL_mean": -102.12724304199219,
      "KL/std": 80.8946762084961,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.0030624205246567726,
      "fcm_dpo/delta": -0.0014668111689388752,
      "fcm_dpo/margin": 54.946632385253906,
      "fcm_dpo/q_t": 0.45884475111961365,
      "grad_norm": 17.766738891601562,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": 1.1175850629806519,
      "logits/rejected": 1.225141167640686,
      "logps/chosen": -348.06707763671875,
      "logps/ref_chosen": -300.8864440917969,
      "logps/ref_rejected": -288.5682067871094,
      "logps/rejected": -390.69549560546875,
      "loss": 4.9912,
      "margin_dpo/margin_mean": 54.94662857055664,
      "margin_dpo/margin_std": 92.2708511352539,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -63.594451904296875,
      "KL/mean": -85.335205078125,
      "KL/rejected_KL_mean": -107.075927734375,
      "KL/std": 80.91473388671875,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.0030609751120209694,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.481475830078125,
      "fcm_dpo/q_t": 0.46755024790763855,
      "grad_norm": 33.36634063720703,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": 1.2853628396987915,
      "logits/rejected": 1.3548942804336548,
      "logps/chosen": -356.8968200683594,
      "logps/ref_chosen": -293.3023681640625,
      "logps/ref_rejected": -270.0457763671875,
      "logps/rejected": -377.1216735839844,
      "loss": 5.1347,
      "margin_dpo/margin_mean": 43.48147964477539,
      "margin_dpo/margin_std": 104.40238952636719,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -56.260169982910156,
      "KL/mean": -82.64054107666016,
      "KL/rejected_KL_mean": -109.02091979980469,
      "KL/std": 81.49649047851562,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.0030609751120209694,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.760746002197266,
      "fcm_dpo/q_t": 0.4604402780532837,
      "grad_norm": 26.541603088378906,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": 1.210934042930603,
      "logits/rejected": 1.2831346988677979,
      "logps/chosen": -339.659423828125,
      "logps/ref_chosen": -283.39923095703125,
      "logps/ref_rejected": -264.5092468261719,
      "logps/rejected": -373.5301513671875,
      "loss": 5.0055,
      "margin_dpo/margin_mean": 52.7607421875,
      "margin_dpo/margin_std": 89.31513977050781,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -49.83872985839844,
      "KL/mean": -81.5859375,
      "KL/rejected_KL_mean": -113.3331527709961,
      "KL/std": 77.20529174804688,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.0030576621647924185,
      "fcm_dpo/delta": -0.002708585700020194,
      "fcm_dpo/margin": 63.494422912597656,
      "fcm_dpo/q_t": 0.4524964988231659,
      "grad_norm": 26.394229888916016,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 1.082545518875122,
      "logits/rejected": 1.1480075120925903,
      "logps/chosen": -304.30657958984375,
      "logps/ref_chosen": -254.4678192138672,
      "logps/ref_rejected": -252.00326538085938,
      "logps/rejected": -365.3363952636719,
      "loss": 4.8925,
      "margin_dpo/margin_mean": 63.494422912597656,
      "margin_dpo/margin_std": 91.88121795654297,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_KL/chosen_KL_mean": -54.59788131713867,
      "eval_KL/mean": -82.77922058105469,
      "eval_KL/rejected_KL_mean": -110.9605484008789,
      "eval_KL/std": 78.74178314208984,
      "eval_fcm_dpo/beta": 0.0030543494503945112,
      "eval_logits/chosen": 1.4414118528366089,
      "eval_logits/rejected": 1.5117008686065674,
      "eval_logps/chosen": -335.0145568847656,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -375.60614013671875,
      "eval_loss": 0.622427225112915,
      "eval_margin_dpo/margin_mean": 56.36266326904297,
      "eval_margin_dpo/margin_std": 92.24838256835938,
      "eval_runtime": 88.926,
      "eval_samples_per_second": 22.491,
      "eval_steps_per_second": 2.811,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -46.134124755859375,
      "KL/mean": -82.72071075439453,
      "KL/rejected_KL_mean": -119.30729675292969,
      "KL/std": 82.62322998046875,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.0030543492175638676,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.17317962646484,
      "fcm_dpo/q_t": 0.4452967345714569,
      "grad_norm": 24.258255004882812,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 1.190069317817688,
      "logits/rejected": 1.2209234237670898,
      "logps/chosen": -356.733642578125,
      "logps/ref_chosen": -310.5995788574219,
      "logps/ref_rejected": -292.7754821777344,
      "logps/rejected": -412.082763671875,
      "loss": 4.7815,
      "margin_dpo/margin_mean": 73.17317962646484,
      "margin_dpo/margin_std": 91.3077392578125,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -48.435420989990234,
      "KL/mean": -78.62593078613281,
      "KL/rejected_KL_mean": -108.81643676757812,
      "KL/std": 80.6571273803711,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.0030543492175638676,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.38100051879883,
      "fcm_dpo/q_t": 0.4548862874507904,
      "grad_norm": 23.630855560302734,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": 1.2661728858947754,
      "logits/rejected": 1.192259430885315,
      "logps/chosen": -348.28680419921875,
      "logps/ref_chosen": -299.8513488769531,
      "logps/ref_rejected": -278.5664367675781,
      "logps/rejected": -387.3829040527344,
      "loss": 4.9289,
      "margin_dpo/margin_mean": 60.38100051879883,
      "margin_dpo/margin_std": 95.06654357910156,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -42.361759185791016,
      "KL/mean": -83.33432006835938,
      "KL/rejected_KL_mean": -124.30691528320312,
      "KL/std": 82.3029556274414,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.0030510914511978626,
      "fcm_dpo/delta": -0.010711705312132835,
      "fcm_dpo/margin": 81.94515991210938,
      "fcm_dpo/q_t": 0.4387422800064087,
      "grad_norm": 23.951541900634766,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": 1.3077610731124878,
      "logits/rejected": 1.352203369140625,
      "logps/chosen": -344.7933654785156,
      "logps/ref_chosen": -302.4316101074219,
      "logps/ref_rejected": -270.951171875,
      "logps/rejected": -395.258056640625,
      "loss": 4.6741,
      "margin_dpo/margin_mean": 81.94515991210938,
      "margin_dpo/margin_std": 82.05253601074219,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -57.71075439453125,
      "KL/mean": -87.08438110351562,
      "KL/rejected_KL_mean": -116.45799255371094,
      "KL/std": 83.10955810546875,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.0030319003853946924,
      "fcm_dpo/delta": 0.0014904225245118141,
      "fcm_dpo/margin": 58.74723434448242,
      "fcm_dpo/q_t": 0.4564915895462036,
      "grad_norm": 18.036270141601562,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": 1.226907730102539,
      "logits/rejected": 1.2609302997589111,
      "logps/chosen": -347.61383056640625,
      "logps/ref_chosen": -289.9031066894531,
      "logps/ref_rejected": -252.84918212890625,
      "logps/rejected": -369.30718994140625,
      "loss": 4.9557,
      "margin_dpo/margin_mean": 58.747230529785156,
      "margin_dpo/margin_std": 96.22080993652344,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -48.02851104736328,
      "KL/mean": -80.14710998535156,
      "KL/rejected_KL_mean": -112.26569366455078,
      "KL/std": 80.99962615966797,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.003028295934200287,
      "fcm_dpo/delta": -0.0019831396639347076,
      "fcm_dpo/margin": 64.2371826171875,
      "fcm_dpo/q_t": 0.45246168971061707,
      "grad_norm": 24.256715774536133,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": 1.2489477396011353,
      "logits/rejected": 1.433919906616211,
      "logps/chosen": -296.20257568359375,
      "logps/ref_chosen": -248.17404174804688,
      "logps/ref_rejected": -252.23764038085938,
      "logps/rejected": -364.50335693359375,
      "loss": 4.8813,
      "margin_dpo/margin_mean": 64.2371826171875,
      "margin_dpo/margin_std": 88.70677185058594,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -54.187278747558594,
      "KL/mean": -82.03688049316406,
      "KL/rejected_KL_mean": -109.886474609375,
      "KL/std": 81.11356353759766,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.0030270942952483892,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.69919204711914,
      "fcm_dpo/q_t": 0.45879003405570984,
      "grad_norm": 33.775230407714844,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": 1.0859718322753906,
      "logits/rejected": 1.1452538967132568,
      "logps/chosen": -330.6033935546875,
      "logps/ref_chosen": -276.4161376953125,
      "logps/ref_rejected": -264.0082702636719,
      "logps/rejected": -373.8947448730469,
      "loss": 4.9929,
      "margin_dpo/margin_mean": 55.69919204711914,
      "margin_dpo/margin_std": 97.21068572998047,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -64.49991607666016,
      "KL/mean": -95.04976654052734,
      "KL/rejected_KL_mean": -125.59962463378906,
      "KL/std": 83.55992126464844,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.0030270942952483892,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.099700927734375,
      "fcm_dpo/q_t": 0.4548070430755615,
      "grad_norm": 31.2412052154541,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": 1.2487804889678955,
      "logits/rejected": 1.2553848028182983,
      "logps/chosen": -321.937255859375,
      "logps/ref_chosen": -257.4373474121094,
      "logps/ref_rejected": -219.66390991210938,
      "logps/rejected": -345.2635192871094,
      "loss": 4.9235,
      "margin_dpo/margin_mean": 61.099708557128906,
      "margin_dpo/margin_std": 94.84451293945312,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -47.23867416381836,
      "KL/mean": -80.4483871459961,
      "KL/rejected_KL_mean": -113.65812683105469,
      "KL/std": 79.68524932861328,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.00302618439309299,
      "fcm_dpo/delta": -0.0030092885717749596,
      "fcm_dpo/margin": 66.41944885253906,
      "fcm_dpo/q_t": 0.4507253170013428,
      "grad_norm": 22.283071517944336,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": 1.0108540058135986,
      "logits/rejected": 1.104782223701477,
      "logps/chosen": -335.8920593261719,
      "logps/ref_chosen": -288.6533508300781,
      "logps/ref_rejected": -291.54107666015625,
      "logps/rejected": -405.19921875,
      "loss": 4.8612,
      "margin_dpo/margin_mean": 66.41944885253906,
      "margin_dpo/margin_std": 89.23849487304688,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -52.920223236083984,
      "KL/mean": -86.48065948486328,
      "KL/rejected_KL_mean": -120.04109191894531,
      "KL/std": 82.83065032958984,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.0030204616487026215,
      "fcm_dpo/delta": 0.00026751484256237745,
      "fcm_dpo/margin": 67.12086486816406,
      "fcm_dpo/q_t": 0.4505273997783661,
      "grad_norm": 30.827428817749023,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": 1.2510356903076172,
      "logits/rejected": 1.358864665031433,
      "logps/chosen": -330.6575927734375,
      "logps/ref_chosen": -277.7373962402344,
      "logps/ref_rejected": -272.988525390625,
      "logps/rejected": -393.0296325683594,
      "loss": 4.87,
      "margin_dpo/margin_mean": 67.12086486816406,
      "margin_dpo/margin_std": 99.05118560791016,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -55.641963958740234,
      "KL/mean": -83.86946868896484,
      "KL/rejected_KL_mean": -112.09697723388672,
      "KL/std": 82.95056915283203,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.0030204616487026215,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.45503234863281,
      "fcm_dpo/q_t": 0.45833486318588257,
      "grad_norm": 39.3583984375,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 1.5938812494277954,
      "logits/rejected": 1.5776993036270142,
      "logps/chosen": -329.86309814453125,
      "logps/ref_chosen": -274.2210998535156,
      "logps/ref_rejected": -264.33465576171875,
      "logps/rejected": -376.431640625,
      "loss": 4.9844,
      "margin_dpo/margin_mean": 56.45503234863281,
      "margin_dpo/margin_std": 98.42146301269531,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -47.82037353515625,
      "KL/mean": -81.95272827148438,
      "KL/rejected_KL_mean": -116.08509826660156,
      "KL/std": 82.48251342773438,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.0030021113343536854,
      "fcm_dpo/delta": -0.012330645695328712,
      "fcm_dpo/margin": 68.26471710205078,
      "fcm_dpo/q_t": 0.4498077630996704,
      "grad_norm": 22.29218101501465,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 1.3233897686004639,
      "logits/rejected": 1.3900409936904907,
      "logps/chosen": -320.6883850097656,
      "logps/ref_chosen": -272.86798095703125,
      "logps/ref_rejected": -250.35772705078125,
      "logps/rejected": -366.44281005859375,
      "loss": 4.8489,
      "margin_dpo/margin_mean": 68.26471710205078,
      "margin_dpo/margin_std": 88.70820617675781,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -62.97191619873047,
      "KL/mean": -91.14588928222656,
      "KL/rejected_KL_mean": -119.31986236572266,
      "KL/std": 82.80123901367188,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.0029892849270254374,
      "fcm_dpo/delta": -0.0006386071909219027,
      "fcm_dpo/margin": 56.34794616699219,
      "fcm_dpo/q_t": 0.45883727073669434,
      "grad_norm": 17.973936080932617,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": 1.1556928157806396,
      "logits/rejected": 1.261034369468689,
      "logps/chosen": -330.8911437988281,
      "logps/ref_chosen": -267.919189453125,
      "logps/ref_rejected": -271.73162841796875,
      "logps/rejected": -391.051513671875,
      "loss": 4.9919,
      "margin_dpo/margin_mean": 56.34794616699219,
      "margin_dpo/margin_std": 98.76493835449219,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -49.978431701660156,
      "KL/mean": -76.02570343017578,
      "KL/rejected_KL_mean": -102.07298278808594,
      "KL/std": 78.13495635986328,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.0029892849270254374,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.09455108642578,
      "fcm_dpo/q_t": 0.4619062542915344,
      "grad_norm": 32.96232604980469,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 1.0244016647338867,
      "logits/rejected": 1.0795246362686157,
      "logps/chosen": -314.9364929199219,
      "logps/ref_chosen": -264.95806884765625,
      "logps/ref_rejected": -267.40093994140625,
      "logps/rejected": -369.4739990234375,
      "loss": 5.0216,
      "margin_dpo/margin_mean": 52.09455108642578,
      "margin_dpo/margin_std": 90.24281311035156,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -59.29770278930664,
      "KL/mean": -89.8375244140625,
      "KL/rejected_KL_mean": -120.3773422241211,
      "KL/std": 83.78045654296875,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.0029892849270254374,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.079627990722656,
      "fcm_dpo/q_t": 0.4554681181907654,
      "grad_norm": 21.054372787475586,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": 1.2566373348236084,
      "logits/rejected": 1.314327359199524,
      "logps/chosen": -349.303466796875,
      "logps/ref_chosen": -290.00579833984375,
      "logps/ref_rejected": -266.8810119628906,
      "logps/rejected": -387.2583312988281,
      "loss": 4.9453,
      "margin_dpo/margin_mean": 61.079627990722656,
      "margin_dpo/margin_std": 100.90029907226562,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -47.4954948425293,
      "KL/mean": -84.02061462402344,
      "KL/rejected_KL_mean": -120.54574584960938,
      "KL/std": 81.5991439819336,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.0029917913489043713,
      "fcm_dpo/delta": 0.0016756762051954865,
      "fcm_dpo/margin": 73.05024719238281,
      "fcm_dpo/q_t": 0.44660890102386475,
      "grad_norm": 29.758264541625977,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": 1.3487331867218018,
      "logits/rejected": 1.445431113243103,
      "logps/chosen": -337.3065490722656,
      "logps/ref_chosen": -289.81103515625,
      "logps/ref_rejected": -278.2758483886719,
      "logps/rejected": -398.8216247558594,
      "loss": 4.8075,
      "margin_dpo/margin_mean": 73.05024719238281,
      "margin_dpo/margin_std": 97.64697265625,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -47.06409454345703,
      "KL/mean": -82.9272689819336,
      "KL/rejected_KL_mean": -118.79043579101562,
      "KL/std": 81.15219116210938,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.002983330050483346,
      "fcm_dpo/delta": -0.00448991684243083,
      "fcm_dpo/margin": 71.72635650634766,
      "fcm_dpo/q_t": 0.4476020336151123,
      "grad_norm": 36.681495666503906,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": 1.4974520206451416,
      "logits/rejected": 1.4911891222000122,
      "logps/chosen": -351.7129211425781,
      "logps/ref_chosen": -304.6488037109375,
      "logps/ref_rejected": -264.2703857421875,
      "logps/rejected": -383.06085205078125,
      "loss": 4.8205,
      "margin_dpo/margin_mean": 71.72635650634766,
      "margin_dpo/margin_std": 93.91630554199219,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -54.046382904052734,
      "KL/mean": -82.81436157226562,
      "KL/rejected_KL_mean": -111.58235168457031,
      "KL/std": 79.44721221923828,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.002982240868732333,
      "fcm_dpo/delta": -0.0010797420982271433,
      "fcm_dpo/margin": 57.535972595214844,
      "fcm_dpo/q_t": 0.45801296830177307,
      "grad_norm": 29.44574546813965,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": 1.6146728992462158,
      "logits/rejected": 1.7404768466949463,
      "logps/chosen": -355.935302734375,
      "logps/ref_chosen": -301.8889465332031,
      "logps/ref_rejected": -275.0347595214844,
      "logps/rejected": -386.6170959472656,
      "loss": 4.9804,
      "margin_dpo/margin_mean": 57.535972595214844,
      "margin_dpo/margin_std": 96.78091430664062,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -63.40422439575195,
      "KL/mean": -88.30333709716797,
      "KL/rejected_KL_mean": -113.20242309570312,
      "KL/std": 80.9029312133789,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.0029795095324516296,
      "fcm_dpo/delta": -0.0005354098393581808,
      "fcm_dpo/margin": 49.7982063293457,
      "fcm_dpo/q_t": 0.46364831924438477,
      "grad_norm": 30.729469299316406,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": 1.1461118459701538,
      "logits/rejected": 1.164353609085083,
      "logps/chosen": -332.65155029296875,
      "logps/ref_chosen": -269.2473449707031,
      "logps/ref_rejected": -275.5192565917969,
      "logps/rejected": -388.7216491699219,
      "loss": 5.0524,
      "margin_dpo/margin_mean": 49.7982063293457,
      "margin_dpo/margin_std": 86.78284454345703,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -62.56691360473633,
      "KL/mean": -96.71534729003906,
      "KL/rejected_KL_mean": -130.86375427246094,
      "KL/std": 86.99903869628906,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.0029724815394729376,
      "fcm_dpo/delta": -0.005732339341193438,
      "fcm_dpo/margin": 68.2968521118164,
      "fcm_dpo/q_t": 0.4504033923149109,
      "grad_norm": 20.659523010253906,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": 1.3438923358917236,
      "logits/rejected": 1.319571852684021,
      "logps/chosen": -353.7273254394531,
      "logps/ref_chosen": -291.1604309082031,
      "logps/ref_rejected": -245.18325805664062,
      "logps/rejected": -376.04705810546875,
      "loss": 4.8806,
      "margin_dpo/margin_mean": 68.29685974121094,
      "margin_dpo/margin_std": 105.344970703125,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -58.66317367553711,
      "KL/mean": -91.43772888183594,
      "KL/rejected_KL_mean": -124.2122802734375,
      "KL/std": 83.90119171142578,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.0029650831129401922,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 65.54910278320312,
      "fcm_dpo/q_t": 0.45241329073905945,
      "grad_norm": 33.202476501464844,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 1.1759965419769287,
      "logits/rejected": 1.1362457275390625,
      "logps/chosen": -317.13299560546875,
      "logps/ref_chosen": -258.4698181152344,
      "logps/ref_rejected": -242.916015625,
      "logps/rejected": -367.1282958984375,
      "loss": 4.8904,
      "margin_dpo/margin_mean": 65.5490951538086,
      "margin_dpo/margin_std": 97.03697967529297,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -43.604087829589844,
      "KL/mean": -80.7899169921875,
      "KL/rejected_KL_mean": -117.97574615478516,
      "KL/std": 80.2686538696289,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.0029609627090394497,
      "fcm_dpo/delta": -0.0019867790397256613,
      "fcm_dpo/margin": 74.37167358398438,
      "fcm_dpo/q_t": 0.4460716247558594,
      "grad_norm": 25.150007247924805,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.1784348487854004,
      "logits/rejected": 1.1704480648040771,
      "logps/chosen": -333.744873046875,
      "logps/ref_chosen": -290.1407775878906,
      "logps/ref_rejected": -239.65676879882812,
      "logps/rejected": -357.63250732421875,
      "loss": 4.7932,
      "margin_dpo/margin_mean": 74.37167358398438,
      "margin_dpo/margin_std": 93.87067413330078,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -58.75519561767578,
      "KL/mean": -83.53996276855469,
      "KL/rejected_KL_mean": -108.32472229003906,
      "KL/std": 82.04164123535156,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.0029603741131722927,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.56951141357422,
      "fcm_dpo/q_t": 0.46404701471328735,
      "grad_norm": 30.029403686523438,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": 1.292972207069397,
      "logits/rejected": 1.3811603784561157,
      "logps/chosen": -335.69122314453125,
      "logps/ref_chosen": -276.93603515625,
      "logps/ref_rejected": -250.398681640625,
      "logps/rejected": -358.7234191894531,
      "loss": 5.0738,
      "margin_dpo/margin_mean": 49.56951141357422,
      "margin_dpo/margin_std": 101.49687194824219,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -51.962852478027344,
      "KL/mean": -87.44894409179688,
      "KL/rejected_KL_mean": -122.93504333496094,
      "KL/std": 83.49008178710938,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.0029603741131722927,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.97218322753906,
      "fcm_dpo/q_t": 0.4486647844314575,
      "grad_norm": 25.58622169494629,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": 1.282218337059021,
      "logits/rejected": 1.3777168989181519,
      "logps/chosen": -337.3929443359375,
      "logps/ref_chosen": -285.43011474609375,
      "logps/ref_rejected": -258.0743408203125,
      "logps/rejected": -381.0093994140625,
      "loss": 4.8423,
      "margin_dpo/margin_mean": 70.97218322753906,
      "margin_dpo/margin_std": 101.25106811523438,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -47.1093635559082,
      "KL/mean": -81.98812866210938,
      "KL/rejected_KL_mean": -116.86688232421875,
      "KL/std": 79.94541931152344,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.0029596425592899323,
      "fcm_dpo/delta": -0.002472977852448821,
      "fcm_dpo/margin": 69.75753784179688,
      "fcm_dpo/q_t": 0.44938188791275024,
      "grad_norm": 25.160587310791016,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": 1.2656772136688232,
      "logits/rejected": 1.2671493291854858,
      "logps/chosen": -346.454345703125,
      "logps/ref_chosen": -299.3450012207031,
      "logps/ref_rejected": -244.60169982910156,
      "logps/rejected": -361.4685974121094,
      "loss": 4.833,
      "margin_dpo/margin_mean": 69.75753784179688,
      "margin_dpo/margin_std": 86.42337799072266,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -52.70196533203125,
      "KL/mean": -83.6665267944336,
      "KL/rejected_KL_mean": -114.63108825683594,
      "KL/std": 83.9303207397461,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.0029548085294663906,
      "fcm_dpo/delta": 0.00024179705360438675,
      "fcm_dpo/margin": 61.92912292480469,
      "fcm_dpo/q_t": 0.45527634024620056,
      "grad_norm": 36.73122787475586,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": 1.5209988355636597,
      "logits/rejected": 1.6224446296691895,
      "logps/chosen": -325.65167236328125,
      "logps/ref_chosen": -272.9497375488281,
      "logps/ref_rejected": -274.6335754394531,
      "logps/rejected": -389.2646484375,
      "loss": 4.9274,
      "margin_dpo/margin_mean": 61.92912292480469,
      "margin_dpo/margin_std": 90.35863494873047,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -46.836647033691406,
      "KL/mean": -79.29090118408203,
      "KL/rejected_KL_mean": -111.74514770507812,
      "KL/std": 80.94264221191406,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.002955094212666154,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.90850830078125,
      "fcm_dpo/q_t": 0.4529983103275299,
      "grad_norm": 29.958696365356445,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 1.2205544710159302,
      "logits/rejected": 1.2658956050872803,
      "logps/chosen": -328.7781982421875,
      "logps/ref_chosen": -281.9415588378906,
      "logps/ref_rejected": -253.20486450195312,
      "logps/rejected": -364.95001220703125,
      "loss": 4.883,
      "margin_dpo/margin_mean": 64.90850830078125,
      "margin_dpo/margin_std": 86.58686828613281,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -45.584571838378906,
      "KL/mean": -87.39900970458984,
      "KL/rejected_KL_mean": -129.2134552001953,
      "KL/std": 83.51866912841797,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.0029542187694460154,
      "fcm_dpo/delta": -0.004858981352299452,
      "fcm_dpo/margin": 83.6288833618164,
      "fcm_dpo/q_t": 0.43960875272750854,
      "grad_norm": 17.639759063720703,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": 1.4744099378585815,
      "logits/rejected": 1.4886142015457153,
      "logps/chosen": -326.201171875,
      "logps/ref_chosen": -280.6166076660156,
      "logps/ref_rejected": -263.7532958984375,
      "logps/rejected": -392.966796875,
      "loss": 4.6937,
      "margin_dpo/margin_mean": 83.6288833618164,
      "margin_dpo/margin_std": 89.7693099975586,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -55.330081939697266,
      "KL/mean": -95.19318389892578,
      "KL/rejected_KL_mean": -135.0562744140625,
      "KL/std": 84.32787322998047,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.0029360882472246885,
      "fcm_dpo/delta": -0.012875671498477459,
      "fcm_dpo/margin": 79.72618865966797,
      "fcm_dpo/q_t": 0.4427037835121155,
      "grad_norm": 37.12092590332031,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": 1.2398046255111694,
      "logits/rejected": 1.3619790077209473,
      "logps/chosen": -341.4923095703125,
      "logps/ref_chosen": -286.1622314453125,
      "logps/ref_rejected": -273.3803405761719,
      "logps/rejected": -408.43658447265625,
      "loss": 4.744,
      "margin_dpo/margin_mean": 79.72618865966797,
      "margin_dpo/margin_std": 91.71855163574219,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -54.90644073486328,
      "KL/mean": -85.97128295898438,
      "KL/rejected_KL_mean": -117.03612518310547,
      "KL/std": 81.06464385986328,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.0029083495028316975,
      "fcm_dpo/delta": -0.004732856526970863,
      "fcm_dpo/margin": 62.12968063354492,
      "fcm_dpo/q_t": 0.4558546841144562,
      "grad_norm": 23.292943954467773,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": 1.4240692853927612,
      "logits/rejected": 1.4549942016601562,
      "logps/chosen": -321.2430114746094,
      "logps/ref_chosen": -266.3365783691406,
      "logps/ref_rejected": -255.01956176757812,
      "logps/rejected": -372.0556640625,
      "loss": 4.9597,
      "margin_dpo/margin_mean": 62.12968063354492,
      "margin_dpo/margin_std": 107.12993621826172,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -58.210723876953125,
      "KL/mean": -85.08154296875,
      "KL/rejected_KL_mean": -111.95236206054688,
      "KL/std": 81.916748046875,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.002901084255427122,
      "fcm_dpo/delta": -0.004727974999696016,
      "fcm_dpo/margin": 53.741641998291016,
      "fcm_dpo/q_t": 0.46183308959007263,
      "grad_norm": 32.27956771850586,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 1.347362995147705,
      "logits/rejected": 1.3858599662780762,
      "logps/chosen": -340.3620910644531,
      "logps/ref_chosen": -282.1513977050781,
      "logps/ref_rejected": -263.4792175292969,
      "logps/rejected": -375.43157958984375,
      "loss": 5.0401,
      "margin_dpo/margin_mean": 53.74164581298828,
      "margin_dpo/margin_std": 100.47946166992188,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -60.45308303833008,
      "KL/mean": -85.091796875,
      "KL/rejected_KL_mean": -109.73051452636719,
      "KL/std": 81.77513122558594,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.002891496755182743,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.27744674682617,
      "fcm_dpo/q_t": 0.4651993215084076,
      "grad_norm": 42.159767150878906,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 1.4457708597183228,
      "logits/rejected": 1.6177581548690796,
      "logps/chosen": -329.1662292480469,
      "logps/ref_chosen": -268.713134765625,
      "logps/ref_rejected": -266.13446044921875,
      "logps/rejected": -375.864990234375,
      "loss": 5.0762,
      "margin_dpo/margin_mean": 49.27744674682617,
      "margin_dpo/margin_std": 96.19835662841797,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -45.9912223815918,
      "KL/mean": -80.69645690917969,
      "KL/rejected_KL_mean": -115.40168762207031,
      "KL/std": 82.09276580810547,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.0028947019018232822,
      "fcm_dpo/delta": 0.0015825885348021984,
      "fcm_dpo/margin": 69.41046142578125,
      "fcm_dpo/q_t": 0.4508829116821289,
      "grad_norm": 22.83631134033203,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": 1.5773351192474365,
      "logits/rejected": 1.6443573236465454,
      "logps/chosen": -329.9754333496094,
      "logps/ref_chosen": -283.9842224121094,
      "logps/ref_rejected": -253.9568634033203,
      "logps/rejected": -369.35858154296875,
      "loss": 4.8597,
      "margin_dpo/margin_mean": 69.41046142578125,
      "margin_dpo/margin_std": 94.52984619140625,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -66.14657592773438,
      "KL/mean": -93.89437103271484,
      "KL/rejected_KL_mean": -121.64218139648438,
      "KL/std": 81.39096069335938,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.0028951598796993494,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.495582580566406,
      "fcm_dpo/q_t": 0.46050265431404114,
      "grad_norm": 28.73857879638672,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": 1.0705053806304932,
      "logits/rejected": 1.096860408782959,
      "logps/chosen": -308.58978271484375,
      "logps/ref_chosen": -242.4431915283203,
      "logps/ref_rejected": -252.87979125976562,
      "logps/rejected": -374.5219421386719,
      "loss": 5.0085,
      "margin_dpo/margin_mean": 55.49559020996094,
      "margin_dpo/margin_std": 95.21666717529297,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -57.22355270385742,
      "KL/mean": -90.84718322753906,
      "KL/rejected_KL_mean": -124.47080993652344,
      "KL/std": 82.18671417236328,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.0028957778122276068,
      "fcm_dpo/delta": 0.0007112675812095404,
      "fcm_dpo/margin": 67.24724578857422,
      "fcm_dpo/q_t": 0.4523656368255615,
      "grad_norm": 22.60959243774414,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": 1.3310291767120361,
      "logits/rejected": 1.2694454193115234,
      "logps/chosen": -345.50775146484375,
      "logps/ref_chosen": -288.2841796875,
      "logps/ref_rejected": -252.9123992919922,
      "logps/rejected": -377.3832092285156,
      "loss": 4.8849,
      "margin_dpo/margin_mean": 67.24725341796875,
      "margin_dpo/margin_std": 95.45549774169922,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -49.694271087646484,
      "KL/mean": -79.8306884765625,
      "KL/rejected_KL_mean": -109.96711730957031,
      "KL/std": 81.31035614013672,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.0028968078549951315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.2728271484375,
      "fcm_dpo/q_t": 0.4571559429168701,
      "grad_norm": 34.47688293457031,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": 1.1597583293914795,
      "logits/rejected": 1.2501521110534668,
      "logps/chosen": -298.7091064453125,
      "logps/ref_chosen": -249.01483154296875,
      "logps/ref_rejected": -243.81192016601562,
      "logps/rejected": -353.7790222167969,
      "loss": 4.9508,
      "margin_dpo/margin_mean": 60.2728271484375,
      "margin_dpo/margin_std": 91.2297592163086,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -55.882102966308594,
      "KL/mean": -77.45880126953125,
      "KL/rejected_KL_mean": -99.03548431396484,
      "KL/std": 82.74370574951172,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.0028968078549951315,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.15338134765625,
      "fcm_dpo/q_t": 0.4693688452243805,
      "grad_norm": 27.51427459716797,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": 1.2233376502990723,
      "logits/rejected": 1.3262286186218262,
      "logps/chosen": -368.81219482421875,
      "logps/ref_chosen": -312.9300537109375,
      "logps/ref_rejected": -296.2721862792969,
      "logps/rejected": -395.30767822265625,
      "loss": 5.1334,
      "margin_dpo/margin_mean": 43.15338134765625,
      "margin_dpo/margin_std": 90.96741485595703,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -46.987117767333984,
      "KL/mean": -82.97389221191406,
      "KL/rejected_KL_mean": -118.96066284179688,
      "KL/std": 84.8348388671875,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.002893403172492981,
      "fcm_dpo/delta": -0.00392385246232152,
      "fcm_dpo/margin": 71.97354125976562,
      "fcm_dpo/q_t": 0.44898098707199097,
      "grad_norm": 16.229137420654297,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": 1.2665073871612549,
      "logits/rejected": 1.3057725429534912,
      "logps/chosen": -315.301025390625,
      "logps/ref_chosen": -268.31390380859375,
      "logps/ref_rejected": -268.7603759765625,
      "logps/rejected": -387.7210693359375,
      "loss": 4.8414,
      "margin_dpo/margin_mean": 71.97354125976562,
      "margin_dpo/margin_std": 97.26851654052734,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -70.13280487060547,
      "KL/mean": -85.00994110107422,
      "KL/rejected_KL_mean": -99.88707733154297,
      "KL/std": 80.36003112792969,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.002887728624045849,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.7542724609375,
      "fcm_dpo/q_t": 0.47897017002105713,
      "grad_norm": 42.626556396484375,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": 1.1785271167755127,
      "logits/rejected": 1.2548820972442627,
      "logps/chosen": -325.3647155761719,
      "logps/ref_chosen": -255.23190307617188,
      "logps/ref_rejected": -241.1186981201172,
      "logps/rejected": -341.00579833984375,
      "loss": 5.2926,
      "margin_dpo/margin_mean": 29.754268646240234,
      "margin_dpo/margin_std": 98.04901885986328,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -57.97383499145508,
      "KL/mean": -86.41525268554688,
      "KL/rejected_KL_mean": -114.85664367675781,
      "KL/std": 79.07952880859375,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.002887728624045849,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.882808685302734,
      "fcm_dpo/q_t": 0.4597659707069397,
      "grad_norm": 22.855236053466797,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 1.1580100059509277,
      "logits/rejected": 1.1792646646499634,
      "logps/chosen": -308.9541931152344,
      "logps/ref_chosen": -250.9803466796875,
      "logps/ref_rejected": -239.9314422607422,
      "logps/rejected": -354.7881164550781,
      "loss": 4.999,
      "margin_dpo/margin_mean": 56.882808685302734,
      "margin_dpo/margin_std": 98.267578125,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -66.53315734863281,
      "KL/mean": -90.01033020019531,
      "KL/rejected_KL_mean": -113.4875259399414,
      "KL/std": 79.06346130371094,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.0028836089186370373,
      "fcm_dpo/delta": -0.0035717112477868795,
      "fcm_dpo/margin": 46.95436096191406,
      "fcm_dpo/q_t": 0.46682170033454895,
      "grad_norm": 23.100778579711914,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 1.3069199323654175,
      "logits/rejected": 1.2460697889328003,
      "logps/chosen": -367.5359191894531,
      "logps/ref_chosen": -301.0027770996094,
      "logps/ref_rejected": -263.628662109375,
      "logps/rejected": -377.11614990234375,
      "loss": 5.1056,
      "margin_dpo/margin_mean": 46.95436096191406,
      "margin_dpo/margin_std": 92.00758361816406,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -52.10173034667969,
      "KL/mean": -86.34663391113281,
      "KL/rejected_KL_mean": -120.591552734375,
      "KL/std": 81.69588470458984,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.002872477052733302,
      "fcm_dpo/delta": -0.006102787796407938,
      "fcm_dpo/margin": 68.48980712890625,
      "fcm_dpo/q_t": 0.4517349600791931,
      "grad_norm": 22.147676467895508,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 1.2600282430648804,
      "logits/rejected": 1.3517117500305176,
      "logps/chosen": -352.95050048828125,
      "logps/ref_chosen": -300.8487854003906,
      "logps/ref_rejected": -296.7118835449219,
      "logps/rejected": -417.3034362792969,
      "loss": 4.867,
      "margin_dpo/margin_mean": 68.48980712890625,
      "margin_dpo/margin_std": 86.33146667480469,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -69.66346740722656,
      "KL/mean": -95.34979248046875,
      "KL/rejected_KL_mean": -121.03611755371094,
      "KL/std": 79.57841491699219,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.002865465125069022,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 51.37265396118164,
      "fcm_dpo/q_t": 0.46392667293548584,
      "grad_norm": 22.081876754760742,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": 1.200124979019165,
      "logits/rejected": 1.2482595443725586,
      "logps/chosen": -341.3118896484375,
      "logps/ref_chosen": -271.6484069824219,
      "logps/ref_rejected": -238.21197509765625,
      "logps/rejected": -359.2480773925781,
      "loss": 5.072,
      "margin_dpo/margin_mean": 51.37265396118164,
      "margin_dpo/margin_std": 105.71794128417969,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -60.01200866699219,
      "KL/mean": -87.1089859008789,
      "KL/rejected_KL_mean": -114.2059555053711,
      "KL/std": 80.2552719116211,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.002860015956684947,
      "fcm_dpo/delta": -0.009544446133077145,
      "fcm_dpo/margin": 54.193939208984375,
      "fcm_dpo/q_t": 0.4619584381580353,
      "grad_norm": 25.75157928466797,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": 1.277986764907837,
      "logits/rejected": 1.3753236532211304,
      "logps/chosen": -326.30853271484375,
      "logps/ref_chosen": -266.2965087890625,
      "logps/ref_rejected": -263.7326965332031,
      "logps/rejected": -377.93865966796875,
      "loss": 5.028,
      "margin_dpo/margin_mean": 54.193939208984375,
      "margin_dpo/margin_std": 92.48853302001953,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -55.54827880859375,
      "KL/mean": -84.5090103149414,
      "KL/rejected_KL_mean": -113.46974182128906,
      "KL/std": 83.19786834716797,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.002843668684363365,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.92145919799805,
      "fcm_dpo/q_t": 0.45976269245147705,
      "grad_norm": 23.576858520507812,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": 1.1388604640960693,
      "logits/rejected": 1.146153450012207,
      "logps/chosen": -347.9090576171875,
      "logps/ref_chosen": -292.36077880859375,
      "logps/ref_rejected": -264.35662841796875,
      "logps/rejected": -377.82635498046875,
      "loss": 4.9968,
      "margin_dpo/margin_mean": 57.92145919799805,
      "margin_dpo/margin_std": 100.44001007080078,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -55.74483871459961,
      "KL/mean": -79.62635040283203,
      "KL/rejected_KL_mean": -103.50787353515625,
      "KL/std": 81.81939697265625,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.00284277880564332,
      "fcm_dpo/delta": -0.00313313165679574,
      "fcm_dpo/margin": 47.76304626464844,
      "fcm_dpo/q_t": 0.4667210876941681,
      "grad_norm": 18.09466552734375,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": 1.300283670425415,
      "logits/rejected": 1.411697268486023,
      "logps/chosen": -302.4646301269531,
      "logps/ref_chosen": -246.71978759765625,
      "logps/ref_rejected": -261.339599609375,
      "logps/rejected": -364.84747314453125,
      "loss": 5.098,
      "margin_dpo/margin_mean": 47.7630500793457,
      "margin_dpo/margin_std": 91.22586059570312,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -47.80153274536133,
      "KL/mean": -81.09030151367188,
      "KL/rejected_KL_mean": -114.37908172607422,
      "KL/std": 80.94442749023438,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.0028339005075395107,
      "fcm_dpo/delta": -0.0031171212904155254,
      "fcm_dpo/margin": 66.57752990722656,
      "fcm_dpo/q_t": 0.45376816391944885,
      "grad_norm": 17.883445739746094,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": 1.356414556503296,
      "logits/rejected": 1.3712667226791382,
      "logps/chosen": -346.3602294921875,
      "logps/ref_chosen": -298.5587158203125,
      "logps/ref_rejected": -264.5877685546875,
      "logps/rejected": -378.96685791015625,
      "loss": 4.9079,
      "margin_dpo/margin_mean": 66.5775375366211,
      "margin_dpo/margin_std": 97.56668853759766,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -40.8297119140625,
      "KL/mean": -76.88773345947266,
      "KL/rejected_KL_mean": -112.94577026367188,
      "KL/std": 79.51182556152344,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.002830205485224724,
      "fcm_dpo/delta": 0.0005090997437946498,
      "fcm_dpo/margin": 72.11604309082031,
      "fcm_dpo/q_t": 0.4498889446258545,
      "grad_norm": 20.183141708374023,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": 1.0752991437911987,
      "logits/rejected": 1.0680744647979736,
      "logps/chosen": -290.6829528808594,
      "logps/ref_chosen": -249.8532257080078,
      "logps/ref_rejected": -233.2117156982422,
      "logps/rejected": -346.157470703125,
      "loss": 4.8373,
      "margin_dpo/margin_mean": 72.11604309082031,
      "margin_dpo/margin_std": 88.95883178710938,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -51.51407241821289,
      "KL/mean": -82.82537841796875,
      "KL/rejected_KL_mean": -114.13667297363281,
      "KL/std": 82.10943603515625,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.62259292602539,
      "fcm_dpo/q_t": 0.45651304721832275,
      "grad_norm": 27.930540084838867,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": 1.3815932273864746,
      "logits/rejected": 1.5573785305023193,
      "logps/chosen": -310.335693359375,
      "logps/ref_chosen": -258.8216247558594,
      "logps/ref_rejected": -260.4953308105469,
      "logps/rejected": -374.6319885253906,
      "loss": 4.943,
      "margin_dpo/margin_mean": 62.62259292602539,
      "margin_dpo/margin_std": 94.38391876220703,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -48.61699295043945,
      "KL/mean": -83.83662414550781,
      "KL/rejected_KL_mean": -119.0562515258789,
      "KL/std": 80.11707305908203,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.43926239013672,
      "fcm_dpo/q_t": 0.451080858707428,
      "grad_norm": 30.61359214782715,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": 1.5119996070861816,
      "logits/rejected": 1.4228246212005615,
      "logps/chosen": -372.8335266113281,
      "logps/ref_chosen": -324.2165222167969,
      "logps/ref_rejected": -237.05453491210938,
      "logps/rejected": -356.11077880859375,
      "loss": 4.8666,
      "margin_dpo/margin_mean": 70.43925476074219,
      "margin_dpo/margin_std": 96.86548614501953,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -65.09933471679688,
      "KL/mean": -90.59452819824219,
      "KL/rejected_KL_mean": -116.08970642089844,
      "KL/std": 82.98916625976562,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.99037551879883,
      "fcm_dpo/q_t": 0.46456727385520935,
      "grad_norm": 23.50550079345703,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 1.3652344942092896,
      "logits/rejected": 1.3705496788024902,
      "logps/chosen": -343.5005798339844,
      "logps/ref_chosen": -278.4012451171875,
      "logps/ref_rejected": -247.72671508789062,
      "logps/rejected": -363.8164367675781,
      "loss": 5.0628,
      "margin_dpo/margin_mean": 50.99037170410156,
      "margin_dpo/margin_std": 93.41893005371094,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -53.99995803833008,
      "KL/mean": -83.22591400146484,
      "KL/rejected_KL_mean": -112.45185852050781,
      "KL/std": 79.01634216308594,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.451904296875,
      "fcm_dpo/q_t": 0.4594079852104187,
      "grad_norm": 26.874584197998047,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.3355767726898193,
      "logits/rejected": 1.375266194343567,
      "logps/chosen": -346.0946044921875,
      "logps/ref_chosen": -292.0946350097656,
      "logps/ref_rejected": -252.24237060546875,
      "logps/rejected": -364.6942443847656,
      "loss": 4.9844,
      "margin_dpo/margin_mean": 58.451904296875,
      "margin_dpo/margin_std": 91.20552062988281,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -45.2663459777832,
      "KL/mean": -73.71417999267578,
      "KL/rejected_KL_mean": -102.16204071044922,
      "KL/std": 82.71453857421875,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.895687103271484,
      "fcm_dpo/q_t": 0.4605320990085602,
      "grad_norm": 11.732975959777832,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.237770676612854,
      "logits/rejected": 1.2993178367614746,
      "logps/chosen": -335.87713623046875,
      "logps/ref_chosen": -290.61077880859375,
      "logps/ref_rejected": -279.8963928222656,
      "logps/rejected": -382.0584411621094,
      "loss": 4.9993,
      "margin_dpo/margin_mean": 56.895687103271484,
      "margin_dpo/margin_std": 91.97207641601562,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -52.676918029785156,
      "KL/mean": -86.35785675048828,
      "KL/rejected_KL_mean": -120.03878784179688,
      "KL/std": 81.24234771728516,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.0028306376188993454,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.36186218261719,
      "fcm_dpo/q_t": 0.4531446099281311,
      "grad_norm": 24.140125274658203,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": 1.411354660987854,
      "logits/rejected": 1.3846681118011475,
      "logps/chosen": -368.9921875,
      "logps/ref_chosen": -316.3152770996094,
      "logps/ref_rejected": -300.5086975097656,
      "logps/rejected": -420.5474548339844,
      "loss": 4.8855,
      "margin_dpo/margin_mean": 67.36186218261719,
      "margin_dpo/margin_std": 90.47819519042969,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -52.64360809326172,
      "KL/mean": -83.51854705810547,
      "KL/rejected_KL_mean": -114.39349365234375,
      "KL/std": 84.81411743164062,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.002831387333571911,
      "fcm_dpo/delta": 0.0013233640929684043,
      "fcm_dpo/margin": 61.749874114990234,
      "fcm_dpo/q_t": 0.45713818073272705,
      "grad_norm": 21.921733856201172,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": 1.4237700700759888,
      "logits/rejected": 1.5310782194137573,
      "logps/chosen": -343.1888732910156,
      "logps/ref_chosen": -290.5452575683594,
      "logps/ref_rejected": -297.2102355957031,
      "logps/rejected": -411.6037292480469,
      "loss": 4.9493,
      "margin_dpo/margin_mean": 61.749874114990234,
      "margin_dpo/margin_std": 92.65682220458984,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -49.96514892578125,
      "KL/mean": -74.897705078125,
      "KL/rejected_KL_mean": -99.83026885986328,
      "KL/std": 79.30836486816406,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.0028336362447589636,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.86512756347656,
      "fcm_dpo/q_t": 0.4653032124042511,
      "grad_norm": 38.04045867919922,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": 1.142727255821228,
      "logits/rejected": 1.2456392049789429,
      "logps/chosen": -313.3854675292969,
      "logps/ref_chosen": -263.4203186035156,
      "logps/ref_rejected": -270.30291748046875,
      "logps/rejected": -370.1331787109375,
      "loss": 5.0723,
      "margin_dpo/margin_mean": 49.86512756347656,
      "margin_dpo/margin_std": 92.39405822753906,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -53.19122314453125,
      "KL/mean": -90.64097595214844,
      "KL/rejected_KL_mean": -128.09072875976562,
      "KL/std": 86.55854797363281,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.0028342760633677244,
      "fcm_dpo/delta": 0.000752477440983057,
      "fcm_dpo/margin": 74.89948272705078,
      "fcm_dpo/q_t": 0.4481929540634155,
      "grad_norm": 20.862154006958008,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": 1.1309120655059814,
      "logits/rejected": 1.29092538356781,
      "logps/chosen": -365.2286071777344,
      "logps/ref_chosen": -312.0373840332031,
      "logps/ref_rejected": -290.50946044921875,
      "logps/rejected": -418.6001892089844,
      "loss": 4.8327,
      "margin_dpo/margin_mean": 74.89949035644531,
      "margin_dpo/margin_std": 104.74353790283203,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -59.991397857666016,
      "KL/mean": -92.9652328491211,
      "KL/rejected_KL_mean": -125.93907165527344,
      "KL/std": 84.67816925048828,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.002835342427715659,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 65.94767761230469,
      "fcm_dpo/q_t": 0.4542052149772644,
      "grad_norm": 32.133750915527344,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": 1.1882331371307373,
      "logits/rejected": 1.2221328020095825,
      "logps/chosen": -357.1633605957031,
      "logps/ref_chosen": -297.1719970703125,
      "logps/ref_rejected": -269.15557861328125,
      "logps/rejected": -395.09466552734375,
      "loss": 4.9111,
      "margin_dpo/margin_mean": 65.94767761230469,
      "margin_dpo/margin_std": 98.27569580078125,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -52.401424407958984,
      "KL/mean": -81.9518051147461,
      "KL/rejected_KL_mean": -111.50218963623047,
      "KL/std": 84.52446746826172,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.002835342427715659,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.10076141357422,
      "fcm_dpo/q_t": 0.4589678943157196,
      "grad_norm": 22.236618041992188,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": 0.9942362308502197,
      "logits/rejected": 0.8979640007019043,
      "logps/chosen": -339.7663269042969,
      "logps/ref_chosen": -287.3648986816406,
      "logps/ref_rejected": -271.1472473144531,
      "logps/rejected": -382.6494140625,
      "loss": 4.9791,
      "margin_dpo/margin_mean": 59.100765228271484,
      "margin_dpo/margin_std": 93.59860229492188,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -62.456302642822266,
      "KL/mean": -84.23226928710938,
      "KL/rejected_KL_mean": -106.00822448730469,
      "KL/std": 81.38301086425781,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.002835342427715659,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.55192565917969,
      "fcm_dpo/q_t": 0.4697783887386322,
      "grad_norm": 26.54384422302246,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": 1.269493579864502,
      "logits/rejected": 1.3603447675704956,
      "logps/chosen": -351.06243896484375,
      "logps/ref_chosen": -288.6061096191406,
      "logps/ref_rejected": -285.6366271972656,
      "logps/rejected": -391.6448669433594,
      "loss": 5.151,
      "margin_dpo/margin_mean": 43.55192565917969,
      "margin_dpo/margin_std": 99.23912811279297,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -60.379207611083984,
      "KL/mean": -82.34085845947266,
      "KL/rejected_KL_mean": -104.30252075195312,
      "KL/std": 79.87520599365234,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.002835342427715659,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.92329788208008,
      "fcm_dpo/q_t": 0.4695143401622772,
      "grad_norm": 41.444244384765625,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 1.1981046199798584,
      "logits/rejected": 1.1511203050613403,
      "logps/chosen": -370.63116455078125,
      "logps/ref_chosen": -310.251953125,
      "logps/ref_rejected": -252.962158203125,
      "logps/rejected": -357.2646789550781,
      "loss": 5.1535,
      "margin_dpo/margin_mean": 43.92329788208008,
      "margin_dpo/margin_std": 103.72200775146484,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -56.84168243408203,
      "KL/mean": -85.50784301757812,
      "KL/rejected_KL_mean": -114.17401123046875,
      "KL/std": 83.11986541748047,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.002835342427715659,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.33232498168945,
      "fcm_dpo/q_t": 0.46014267206192017,
      "grad_norm": 25.801481246948242,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 1.423187017440796,
      "logits/rejected": 1.5700111389160156,
      "logps/chosen": -293.6221008300781,
      "logps/ref_chosen": -236.7804412841797,
      "logps/ref_rejected": -229.67550659179688,
      "logps/rejected": -343.8495178222656,
      "loss": 5.0121,
      "margin_dpo/margin_mean": 57.33232498168945,
      "margin_dpo/margin_std": 103.93323516845703,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -48.929378509521484,
      "KL/mean": -86.91568756103516,
      "KL/rejected_KL_mean": -124.9020004272461,
      "KL/std": 84.10964965820312,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.002835442079231143,
      "fcm_dpo/delta": 5.0358648877590895e-05,
      "fcm_dpo/margin": 75.97261810302734,
      "fcm_dpo/q_t": 0.44730231165885925,
      "grad_norm": 20.091711044311523,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": 1.2399934530258179,
      "logits/rejected": 1.1392191648483276,
      "logps/chosen": -384.23291015625,
      "logps/ref_chosen": -335.30352783203125,
      "logps/ref_rejected": -250.6796875,
      "logps/rejected": -375.5816650390625,
      "loss": 4.8202,
      "margin_dpo/margin_mean": 75.97261810302734,
      "margin_dpo/margin_std": 104.06936645507812,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -59.71469497680664,
      "KL/mean": -84.98561096191406,
      "KL/rejected_KL_mean": -110.25652313232422,
      "KL/std": 81.80272674560547,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.0028354565147310495,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.541839599609375,
      "fcm_dpo/q_t": 0.4648284912109375,
      "grad_norm": 20.047609329223633,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": 1.3954814672470093,
      "logits/rejected": 1.2403744459152222,
      "logps/chosen": -347.9349365234375,
      "logps/ref_chosen": -288.22021484375,
      "logps/ref_rejected": -227.96673583984375,
      "logps/rejected": -338.2232360839844,
      "loss": 5.063,
      "margin_dpo/margin_mean": 50.541839599609375,
      "margin_dpo/margin_std": 91.96941375732422,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -60.3553466796875,
      "KL/mean": -88.67678833007812,
      "KL/rejected_KL_mean": -116.99821472167969,
      "KL/std": 81.18622589111328,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.0028354565147310495,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.64287185668945,
      "fcm_dpo/q_t": 0.4605252146720886,
      "grad_norm": 17.572399139404297,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": 1.4314088821411133,
      "logits/rejected": 1.4715723991394043,
      "logps/chosen": -320.6922607421875,
      "logps/ref_chosen": -260.3369140625,
      "logps/ref_rejected": -254.2083740234375,
      "logps/rejected": -371.20660400390625,
      "loss": 5.0041,
      "margin_dpo/margin_mean": 56.64287185668945,
      "margin_dpo/margin_std": 93.06903076171875,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -42.13630294799805,
      "KL/mean": -78.58474731445312,
      "KL/rejected_KL_mean": -115.03319549560547,
      "KL/std": 81.96719360351562,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.0028167786076664925,
      "fcm_dpo/delta": -0.012979063205420971,
      "fcm_dpo/margin": 72.89688873291016,
      "fcm_dpo/q_t": 0.44967401027679443,
      "grad_norm": 19.711421966552734,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.6216853857040405,
      "logits/rejected": 1.6912635564804077,
      "logps/chosen": -325.2057800292969,
      "logps/ref_chosen": -283.0694885253906,
      "logps/ref_rejected": -289.1370544433594,
      "logps/rejected": -404.1702575683594,
      "loss": 4.8486,
      "margin_dpo/margin_mean": 72.89688110351562,
      "margin_dpo/margin_std": 92.64458465576172,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -42.541412353515625,
      "KL/mean": -81.42070007324219,
      "KL/rejected_KL_mean": -120.29998779296875,
      "KL/std": 82.61752319335938,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.0028043976053595543,
      "fcm_dpo/delta": -0.0012621973874047399,
      "fcm_dpo/margin": 77.75856018066406,
      "fcm_dpo/q_t": 0.4464620351791382,
      "grad_norm": 37.86336135864258,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": 1.0079892873764038,
      "logits/rejected": 1.0737838745117188,
      "logps/chosen": -299.42645263671875,
      "logps/ref_chosen": -256.8850402832031,
      "logps/ref_rejected": -247.0802459716797,
      "logps/rejected": -367.3802185058594,
      "loss": 4.7877,
      "margin_dpo/margin_mean": 77.75856018066406,
      "margin_dpo/margin_std": 90.67868041992188,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -56.05849838256836,
      "KL/mean": -84.90553283691406,
      "KL/rejected_KL_mean": -113.75257873535156,
      "KL/std": 82.11669921875,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.0028033354319632053,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.694061279296875,
      "fcm_dpo/q_t": 0.46038222312927246,
      "grad_norm": 21.033924102783203,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": 1.1114262342453003,
      "logits/rejected": 1.272428035736084,
      "logps/chosen": -313.58258056640625,
      "logps/ref_chosen": -257.5240783691406,
      "logps/ref_rejected": -274.82122802734375,
      "logps/rejected": -388.57379150390625,
      "loss": 5.0026,
      "margin_dpo/margin_mean": 57.694061279296875,
      "margin_dpo/margin_std": 96.08705139160156,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -47.38501739501953,
      "KL/mean": -72.31178283691406,
      "KL/rejected_KL_mean": -97.23856353759766,
      "KL/std": 80.07518005371094,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.002803068608045578,
      "fcm_dpo/delta": -0.0004765581397805363,
      "fcm_dpo/margin": 49.85354232788086,
      "fcm_dpo/q_t": 0.4657519459724426,
      "grad_norm": 21.668886184692383,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": 1.181845784187317,
      "logits/rejected": 1.0649257898330688,
      "logps/chosen": -359.6447448730469,
      "logps/ref_chosen": -312.2597351074219,
      "logps/ref_rejected": -259.74383544921875,
      "logps/rejected": -356.98236083984375,
      "loss": 5.0727,
      "margin_dpo/margin_mean": 49.85354232788086,
      "margin_dpo/margin_std": 87.8105697631836,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -42.68141174316406,
      "KL/mean": -75.74369812011719,
      "KL/rejected_KL_mean": -108.80599212646484,
      "KL/std": 84.55339813232422,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.0028038646560162306,
      "fcm_dpo/delta": 0.0014241663739085197,
      "fcm_dpo/margin": 66.12458801269531,
      "fcm_dpo/q_t": 0.4545362889766693,
      "grad_norm": 17.530189514160156,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": 1.0863707065582275,
      "logits/rejected": 0.996247410774231,
      "logps/chosen": -359.7211608886719,
      "logps/ref_chosen": -317.03973388671875,
      "logps/ref_rejected": -260.12628173828125,
      "logps/rejected": -368.9322509765625,
      "loss": 4.922,
      "margin_dpo/margin_mean": 66.12458801269531,
      "margin_dpo/margin_std": 101.11981964111328,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -43.86881637573242,
      "KL/mean": -80.62144470214844,
      "KL/rejected_KL_mean": -117.37406921386719,
      "KL/std": 83.86090087890625,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.0028030124958604574,
      "fcm_dpo/delta": -0.007562968414276838,
      "fcm_dpo/margin": 73.5052490234375,
      "fcm_dpo/q_t": 0.44953328371047974,
      "grad_norm": 19.42373275756836,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 1.1772277355194092,
      "logits/rejected": 1.1053433418273926,
      "logps/chosen": -348.26275634765625,
      "logps/ref_chosen": -304.3939514160156,
      "logps/ref_rejected": -257.6587219238281,
      "logps/rejected": -375.0327453613281,
      "loss": 4.8482,
      "margin_dpo/margin_mean": 73.5052490234375,
      "margin_dpo/margin_std": 97.07100677490234,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -54.481101989746094,
      "KL/mean": -82.42228698730469,
      "KL/rejected_KL_mean": -110.36346435546875,
      "KL/std": 83.83647155761719,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.0027885385788977146,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.88237380981445,
      "fcm_dpo/q_t": 0.46181342005729675,
      "grad_norm": 45.44668197631836,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 0.995488166809082,
      "logits/rejected": 1.0806761980056763,
      "logps/chosen": -339.4249267578125,
      "logps/ref_chosen": -284.94384765625,
      "logps/ref_rejected": -297.4357604980469,
      "logps/rejected": -407.7992248535156,
      "loss": 5.0206,
      "margin_dpo/margin_mean": 55.88237380981445,
      "margin_dpo/margin_std": 95.30608367919922,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -43.22677993774414,
      "KL/mean": -78.55381774902344,
      "KL/rejected_KL_mean": -113.880859375,
      "KL/std": 80.9053955078125,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.0027885385788977146,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.65408325195312,
      "fcm_dpo/q_t": 0.45181718468666077,
      "grad_norm": 33.319461822509766,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": 1.2500348091125488,
      "logits/rejected": 1.3851152658462524,
      "logps/chosen": -275.5810546875,
      "logps/ref_chosen": -232.35427856445312,
      "logps/ref_rejected": -245.6057891845703,
      "logps/rejected": -359.4866027832031,
      "loss": 4.8725,
      "margin_dpo/margin_mean": 70.65408325195312,
      "margin_dpo/margin_std": 97.09038543701172,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -51.096282958984375,
      "KL/mean": -86.0384750366211,
      "KL/rejected_KL_mean": -120.98068237304688,
      "KL/std": 82.8897476196289,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.0027885385788977146,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 69.88441467285156,
      "fcm_dpo/q_t": 0.4522125720977783,
      "grad_norm": 18.36236572265625,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": 1.0728105306625366,
      "logits/rejected": 1.1157150268554688,
      "logps/chosen": -329.1031494140625,
      "logps/ref_chosen": -278.0068664550781,
      "logps/ref_rejected": -257.19256591796875,
      "logps/rejected": -378.1732482910156,
      "loss": 4.8815,
      "margin_dpo/margin_mean": 69.88440704345703,
      "margin_dpo/margin_std": 98.59559631347656,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -64.66651153564453,
      "KL/mean": -89.14273834228516,
      "KL/rejected_KL_mean": -113.61898040771484,
      "KL/std": 81.29632568359375,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.0027885385788977146,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.95246887207031,
      "fcm_dpo/q_t": 0.4664877951145172,
      "grad_norm": 34.43251037597656,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": 1.0841894149780273,
      "logits/rejected": 1.134438395500183,
      "logps/chosen": -345.42401123046875,
      "logps/ref_chosen": -280.75750732421875,
      "logps/ref_rejected": -264.34295654296875,
      "logps/rejected": -377.9619140625,
      "loss": 5.0911,
      "margin_dpo/margin_mean": 48.95246887207031,
      "margin_dpo/margin_std": 90.29544067382812,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -49.36830139160156,
      "KL/mean": -76.94461059570312,
      "KL/rejected_KL_mean": -104.52091979980469,
      "KL/std": 78.10546112060547,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.0027885385788977146,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.15261459350586,
      "fcm_dpo/q_t": 0.46227532625198364,
      "grad_norm": 25.71925163269043,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": 1.0371792316436768,
      "logits/rejected": 1.1587921380996704,
      "logps/chosen": -290.83966064453125,
      "logps/ref_chosen": -241.4713592529297,
      "logps/ref_rejected": -244.0782928466797,
      "logps/rejected": -348.59918212890625,
      "loss": 5.0265,
      "margin_dpo/margin_mean": 55.15261459350586,
      "margin_dpo/margin_std": 92.39383697509766,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -36.07765197753906,
      "KL/mean": -69.0077133178711,
      "KL/rejected_KL_mean": -101.93775939941406,
      "KL/std": 79.12623596191406,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.0027798654045909643,
      "fcm_dpo/delta": -0.006236155517399311,
      "fcm_dpo/margin": 65.86012268066406,
      "fcm_dpo/q_t": 0.45506489276885986,
      "grad_norm": 16.269027709960938,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": 1.2806975841522217,
      "logits/rejected": 1.2856770753860474,
      "logps/chosen": -354.85198974609375,
      "logps/ref_chosen": -318.7742919921875,
      "logps/ref_rejected": -293.66595458984375,
      "logps/rejected": -395.6037292480469,
      "loss": 4.9215,
      "margin_dpo/margin_mean": 65.86012268066406,
      "margin_dpo/margin_std": 94.6804428100586,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -51.50764083862305,
      "KL/mean": -85.54444122314453,
      "KL/rejected_KL_mean": -119.58123016357422,
      "KL/std": 83.2199935913086,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.0027746616397053003,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.0735855102539,
      "fcm_dpo/q_t": 0.4537642300128937,
      "grad_norm": 30.475337982177734,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": 1.1403731107711792,
      "logits/rejected": 1.1754294633865356,
      "logps/chosen": -317.4433288574219,
      "logps/ref_chosen": -265.9356994628906,
      "logps/ref_rejected": -243.6986083984375,
      "logps/rejected": -363.27984619140625,
      "loss": 4.9104,
      "margin_dpo/margin_mean": 68.07359313964844,
      "margin_dpo/margin_std": 102.31426239013672,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 5.04975961889111,
      "train_runtime": 7451.4996,
      "train_samples_per_second": 8.204,
      "train_steps_per_second": 0.064
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}