{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.04536247253417969,
      "KL/mean": -0.05389120429754257,
      "KL/rejected_KL_mean": -0.06241607666015625,
      "KL/std": 0.5513200759887695,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.01704825460910797,
      "fcm_dpo/q_t": 0.499957412481308,
      "grad_norm": 14.617972373962402,
      "learning_rate": 0.0,
      "logits/chosen": 1.9854813814163208,
      "logits/rejected": 1.7737188339233398,
      "logps/chosen": -267.0936584472656,
      "logps/ref_chosen": -267.04827880859375,
      "logps/ref_rejected": -219.76657104492188,
      "logps/rejected": -219.82899475097656,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.01704716682434082,
      "margin_dpo/margin_std": 0.7467707991600037,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.07550811767578125,
      "KL/mean": 0.03060000389814377,
      "KL/rejected_KL_mean": -0.014301300048828125,
      "KL/std": 0.6403332352638245,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08979703485965729,
      "fcm_dpo/q_t": 0.4997755289077759,
      "grad_norm": 15.140359878540039,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 2.1993117332458496,
      "logits/rejected": 2.1368322372436523,
      "logps/chosen": -256.5101623535156,
      "logps/ref_chosen": -256.585693359375,
      "logps/ref_rejected": -239.92742919921875,
      "logps/rejected": -239.94171142578125,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08979731798171997,
      "margin_dpo/margin_std": 0.9745746850967407,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": -0.00539398193359375,
      "KL/mean": -0.049824588000774384,
      "KL/rejected_KL_mean": -0.09424400329589844,
      "KL/std": 0.5856199860572815,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08885060250759125,
      "fcm_dpo/q_t": 0.4997778832912445,
      "grad_norm": 14.618629455566406,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": 2.2567059993743896,
      "logits/rejected": 2.296304702758789,
      "logps/chosen": -265.67425537109375,
      "logps/ref_chosen": -265.66888427734375,
      "logps/ref_rejected": -284.3778991699219,
      "logps/rejected": -284.47216796875,
      "loss": 5.5417,
      "margin_dpo/margin_mean": 0.08885043859481812,
      "margin_dpo/margin_std": 0.8570029735565186,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.023189544677734375,
      "KL/mean": -0.006526872515678406,
      "KL/rejected_KL_mean": -0.03623199462890625,
      "KL/std": 0.7063453197479248,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05941268801689148,
      "fcm_dpo/q_t": 0.49985143542289734,
      "grad_norm": 15.753974914550781,
      "learning_rate": 3.125e-08,
      "logits/chosen": 1.936367392539978,
      "logits/rejected": 1.8752315044403076,
      "logps/chosen": -319.4842834472656,
      "logps/ref_chosen": -319.50750732421875,
      "logps/ref_rejected": -296.22314453125,
      "logps/rejected": -296.2593688964844,
      "loss": 5.5429,
      "margin_dpo/margin_mean": 0.05941309034824371,
      "margin_dpo/margin_std": 0.9982026815414429,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.016588211059570312,
      "KL/mean": 0.03492110222578049,
      "KL/rejected_KL_mean": 0.05326080322265625,
      "KL/std": 0.6035839319229126,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03666464984416962,
      "fcm_dpo/q_t": 0.5000916719436646,
      "grad_norm": 15.684805870056152,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": 1.9481229782104492,
      "logits/rejected": 1.964356780052185,
      "logps/chosen": -292.8564147949219,
      "logps/ref_chosen": -292.87298583984375,
      "logps/ref_rejected": -270.3802795410156,
      "logps/rejected": -270.3269958496094,
      "loss": 5.5467,
      "margin_dpo/margin_mean": -0.036664366722106934,
      "margin_dpo/margin_std": 0.7655607461929321,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": 0.04851722717285156,
      "KL/mean": 0.04088638722896576,
      "KL/rejected_KL_mean": 0.03325462341308594,
      "KL/std": 0.5895508527755737,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.015266358852386475,
      "fcm_dpo/q_t": 0.49996185302734375,
      "grad_norm": 15.476506233215332,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 2.103102207183838,
      "logits/rejected": 2.037451982498169,
      "logps/chosen": -280.2369384765625,
      "logps/ref_chosen": -280.28546142578125,
      "logps/ref_rejected": -242.7611541748047,
      "logps/rejected": -242.72789001464844,
      "loss": 5.5447,
      "margin_dpo/margin_mean": 0.015265822410583496,
      "margin_dpo/margin_std": 0.8650776147842407,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": -0.0046672821044921875,
      "KL/mean": 0.012224093079566956,
      "KL/rejected_KL_mean": 0.029111862182617188,
      "KL/std": 0.6667926907539368,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.033790141344070435,
      "fcm_dpo/q_t": 0.5000845193862915,
      "grad_norm": 15.583406448364258,
      "learning_rate": 6.25e-08,
      "logits/chosen": 1.862593650817871,
      "logits/rejected": 1.7612130641937256,
      "logps/chosen": -255.11036682128906,
      "logps/ref_chosen": -255.10572814941406,
      "logps/ref_rejected": -251.15402221679688,
      "logps/rejected": -251.1249237060547,
      "loss": 5.5466,
      "margin_dpo/margin_mean": -0.03379000723361969,
      "margin_dpo/margin_std": 0.9376596212387085,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.0808868408203125,
      "KL/mean": 0.015082728117704391,
      "KL/rejected_KL_mean": 0.11106109619140625,
      "KL/std": 0.7702391743659973,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.19195644557476044,
      "fcm_dpo/q_t": 0.5004799365997314,
      "grad_norm": 15.945377349853516,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": 2.199270248413086,
      "logits/rejected": 1.9912503957748413,
      "logps/chosen": -345.262451171875,
      "logps/ref_chosen": -345.18157958984375,
      "logps/ref_rejected": -277.3522644042969,
      "logps/rejected": -277.2412109375,
      "loss": 5.553,
      "margin_dpo/margin_mean": -0.1919558048248291,
      "margin_dpo/margin_std": 1.0858811140060425,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": 0.15078353881835938,
      "KL/mean": 0.09128256887197495,
      "KL/rejected_KL_mean": 0.03178596496582031,
      "KL/std": 0.6456024646759033,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11899183690547943,
      "fcm_dpo/q_t": 0.499702513217926,
      "grad_norm": 14.088343620300293,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": 1.901161551475525,
      "logits/rejected": 1.868157148361206,
      "logps/chosen": -276.57958984375,
      "logps/ref_chosen": -276.7303161621094,
      "logps/ref_rejected": -266.19952392578125,
      "logps/rejected": -266.167724609375,
      "loss": 5.5405,
      "margin_dpo/margin_mean": 0.11899170279502869,
      "margin_dpo/margin_std": 0.9147509336471558,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": -0.055660247802734375,
      "KL/mean": -0.0029932335019111633,
      "KL/rejected_KL_mean": 0.04967498779296875,
      "KL/std": 0.5684869289398193,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.10534195601940155,
      "fcm_dpo/q_t": 0.5002633333206177,
      "grad_norm": 13.85465145111084,
      "learning_rate": 9.375e-08,
      "logits/chosen": 1.9780428409576416,
      "logits/rejected": 2.0317459106445312,
      "logps/chosen": -246.617919921875,
      "logps/ref_chosen": -246.562255859375,
      "logps/ref_rejected": -224.70401000976562,
      "logps/rejected": -224.6543426513672,
      "loss": 5.5495,
      "margin_dpo/margin_mean": -0.10534200072288513,
      "margin_dpo/margin_std": 0.8190886974334717,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": 0.07959938049316406,
      "KL/mean": 0.04428281635046005,
      "KL/rejected_KL_mean": 0.008962631225585938,
      "KL/std": 0.6608096361160278,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07062841951847076,
      "fcm_dpo/q_t": 0.4998234212398529,
      "grad_norm": 15.348572731018066,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 2.0220999717712402,
      "logits/rejected": 1.9633264541625977,
      "logps/chosen": -290.4215087890625,
      "logps/ref_chosen": -290.5010986328125,
      "logps/ref_rejected": -253.9559326171875,
      "logps/rejected": -253.94696044921875,
      "loss": 5.5425,
      "margin_dpo/margin_mean": 0.07062935829162598,
      "margin_dpo/margin_std": 0.941288411617279,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": 0.050846099853515625,
      "KL/mean": 0.07712649554014206,
      "KL/rejected_KL_mean": 0.10341072082519531,
      "KL/std": 0.6643091440200806,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.05256269872188568,
      "fcm_dpo/q_t": 0.5001314282417297,
      "grad_norm": 14.921819686889648,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": 1.5968624353408813,
      "logits/rejected": 1.5814290046691895,
      "logps/chosen": -278.4676513671875,
      "logps/ref_chosen": -278.5185241699219,
      "logps/ref_rejected": -273.56207275390625,
      "logps/rejected": -273.45867919921875,
      "loss": 5.5474,
      "margin_dpo/margin_mean": -0.05256234109401703,
      "margin_dpo/margin_std": 0.8758862018585205,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.07026100158691406,
      "KL/mean": -0.05152081698179245,
      "KL/rejected_KL_mean": -0.03277015686035156,
      "KL/std": 0.6123308539390564,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.037490978837013245,
      "fcm_dpo/q_t": 0.5000936985015869,
      "grad_norm": 14.892748832702637,
      "learning_rate": 1.25e-07,
      "logits/chosen": 1.7829504013061523,
      "logits/rejected": 1.9086259603500366,
      "logps/chosen": -264.7274169921875,
      "logps/ref_chosen": -264.6571350097656,
      "logps/ref_rejected": -274.3492736816406,
      "logps/rejected": -274.382080078125,
      "loss": 5.5468,
      "margin_dpo/margin_mean": -0.03749191761016846,
      "margin_dpo/margin_std": 0.8440827131271362,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": -0.008312225341796875,
      "KL/mean": -0.005231525748968124,
      "KL/rejected_KL_mean": -0.00215911865234375,
      "KL/std": 0.5663936734199524,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.006156213581562042,
      "fcm_dpo/q_t": 0.500015377998352,
      "grad_norm": 15.241767883300781,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": 1.9183093309402466,
      "logits/rejected": 1.9386394023895264,
      "logps/chosen": -274.1228942871094,
      "logps/ref_chosen": -274.1145935058594,
      "logps/ref_rejected": -261.0951843261719,
      "logps/rejected": -261.09735107421875,
      "loss": 5.5455,
      "margin_dpo/margin_mean": -0.006156221032142639,
      "margin_dpo/margin_std": 0.772310733795166,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": 0.028955459594726562,
      "KL/mean": 0.047754719853401184,
      "KL/rejected_KL_mean": 0.06655311584472656,
      "KL/std": 0.5765488147735596,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.037596166133880615,
      "fcm_dpo/q_t": 0.500093936920166,
      "grad_norm": 16.109569549560547,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": 2.0001466274261475,
      "logits/rejected": 1.8582792282104492,
      "logps/chosen": -288.55572509765625,
      "logps/ref_chosen": -288.584716796875,
      "logps/ref_rejected": -224.8094482421875,
      "logps/rejected": -224.74288940429688,
      "loss": 5.5467,
      "margin_dpo/margin_mean": -0.037596121430397034,
      "margin_dpo/margin_std": 0.7449219226837158,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": -0.052204132080078125,
      "KL/mean": -0.05075667053461075,
      "KL/rejected_KL_mean": -0.04930686950683594,
      "KL/std": 0.6129648685455322,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.002895936369895935,
      "fcm_dpo/q_t": 0.5000072121620178,
      "grad_norm": 16.592559814453125,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": 1.8588883876800537,
      "logits/rejected": 1.705232858657837,
      "logps/chosen": -283.8953552246094,
      "logps/ref_chosen": -283.8431091308594,
      "logps/ref_rejected": -248.63645935058594,
      "logps/rejected": -248.68576049804688,
      "loss": 5.5454,
      "margin_dpo/margin_mean": -0.002897113561630249,
      "margin_dpo/margin_std": 0.8401002287864685,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.05851936340332031,
      "KL/mean": 0.013566896319389343,
      "KL/rejected_KL_mean": -0.031375885009765625,
      "KL/std": 0.5930002927780151,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08988568186759949,
      "fcm_dpo/q_t": 0.4997752904891968,
      "grad_norm": 15.167532920837402,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 1.8319032192230225,
      "logits/rejected": 1.7843517065048218,
      "logps/chosen": -254.5750274658203,
      "logps/ref_chosen": -254.6335906982422,
      "logps/ref_rejected": -245.3981170654297,
      "logps/rejected": -245.4294891357422,
      "loss": 5.5416,
      "margin_dpo/margin_mean": 0.08988560736179352,
      "margin_dpo/margin_std": 0.7138881087303162,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.08774185180664062,
      "KL/mean": -0.03821343183517456,
      "KL/rejected_KL_mean": 0.0113067626953125,
      "KL/std": 0.5794514417648315,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.09905266761779785,
      "fcm_dpo/q_t": 0.500247597694397,
      "grad_norm": 15.121091842651367,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": 1.7991856336593628,
      "logits/rejected": 1.7099615335464478,
      "logps/chosen": -287.8372802734375,
      "logps/ref_chosen": -287.7495422363281,
      "logps/ref_rejected": -273.03363037109375,
      "logps/rejected": -273.0223083496094,
      "loss": 5.5492,
      "margin_dpo/margin_mean": -0.09905266761779785,
      "margin_dpo/margin_std": 0.844775915145874,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.029865264892578125,
      "KL/mean": 0.040704671293497086,
      "KL/rejected_KL_mean": 0.11126708984375,
      "KL/std": 0.6467154622077942,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.14113231003284454,
      "fcm_dpo/q_t": 0.5003528594970703,
      "grad_norm": 14.851866722106934,
      "learning_rate": 1.875e-07,
      "logits/chosen": 2.1018497943878174,
      "logits/rejected": 2.0734498500823975,
      "logps/chosen": -273.4337463378906,
      "logps/ref_chosen": -273.4039001464844,
      "logps/ref_rejected": -260.09722900390625,
      "logps/rejected": -259.9859619140625,
      "loss": 5.5509,
      "margin_dpo/margin_mean": -0.14113177359104156,
      "margin_dpo/margin_std": 0.8985235691070557,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": -0.0021514892578125,
      "KL/mean": 0.016249872744083405,
      "KL/rejected_KL_mean": 0.034656524658203125,
      "KL/std": 0.6506487131118774,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03680087625980377,
      "fcm_dpo/q_t": 0.5000919699668884,
      "grad_norm": 15.49797248840332,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.788353681564331,
      "logits/rejected": 1.7263466119766235,
      "logps/chosen": -292.9642639160156,
      "logps/ref_chosen": -292.96209716796875,
      "logps/ref_rejected": -256.602783203125,
      "logps/rejected": -256.568115234375,
      "loss": 5.5467,
      "margin_dpo/margin_mean": -0.0368005633354187,
      "margin_dpo/margin_std": 0.8343037366867065,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.040279388427734375,
      "KL/mean": -0.02052316814661026,
      "KL/rejected_KL_mean": -0.08134651184082031,
      "KL/std": 0.6289079785346985,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12164483964443207,
      "fcm_dpo/q_t": 0.49969586730003357,
      "grad_norm": 15.307790756225586,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 2.1811044216156006,
      "logits/rejected": 2.1350505352020264,
      "logps/chosen": -292.28692626953125,
      "logps/ref_chosen": -292.3271789550781,
      "logps/ref_rejected": -274.20635986328125,
      "logps/rejected": -274.2877197265625,
      "loss": 5.5404,
      "margin_dpo/margin_mean": 0.12164326012134552,
      "margin_dpo/margin_std": 0.857761025428772,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.1517505645751953,
      "KL/mean": 0.044946957379579544,
      "KL/rejected_KL_mean": -0.06185722351074219,
      "KL/std": 0.6266728043556213,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.21360813081264496,
      "fcm_dpo/q_t": 0.49946603178977966,
      "grad_norm": 13.711256980895996,
      "learning_rate": 2.1875e-07,
      "logits/chosen": 2.0483126640319824,
      "logits/rejected": 2.0516109466552734,
      "logps/chosen": -253.11447143554688,
      "logps/ref_chosen": -253.2662353515625,
      "logps/ref_rejected": -241.4681396484375,
      "logps/rejected": -241.53001403808594,
      "loss": 5.5367,
      "margin_dpo/margin_mean": 0.21360869705677032,
      "margin_dpo/margin_std": 0.8758575916290283,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.027950286865234375,
      "KL/mean": 0.05429338663816452,
      "KL/rejected_KL_mean": 0.08062171936035156,
      "KL/std": 0.6153542399406433,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.052675649523735046,
      "fcm_dpo/q_t": 0.5001316666603088,
      "grad_norm": 15.32956600189209,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": 1.9792588949203491,
      "logits/rejected": 1.9631540775299072,
      "logps/chosen": -275.39361572265625,
      "logps/ref_chosen": -275.4216003417969,
      "logps/ref_rejected": -264.44024658203125,
      "logps/rejected": -264.359619140625,
      "loss": 5.5474,
      "margin_dpo/margin_mean": -0.05267469584941864,
      "margin_dpo/margin_std": 0.8457110524177551,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": -0.06866836547851562,
      "KL/mean": -0.03027290105819702,
      "KL/rejected_KL_mean": 0.008121490478515625,
      "KL/std": 0.6753708720207214,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07679641246795654,
      "fcm_dpo/q_t": 0.5001919865608215,
      "grad_norm": 16.8044490814209,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": 2.1006295680999756,
      "logits/rejected": 2.1262643337249756,
      "logps/chosen": -293.4462585449219,
      "logps/ref_chosen": -293.3775634765625,
      "logps/ref_rejected": -301.59014892578125,
      "logps/rejected": -301.5820007324219,
      "loss": 5.5483,
      "margin_dpo/margin_mean": -0.07679517567157745,
      "margin_dpo/margin_std": 0.9318934082984924,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.027276992797851562,
      "KL/mean": 0.012380778789520264,
      "KL/rejected_KL_mean": -0.0025081634521484375,
      "KL/std": 0.5253986716270447,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.029784828424453735,
      "fcm_dpo/q_t": 0.49992549419403076,
      "grad_norm": 14.85913372039795,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.6880950927734375,
      "logits/rejected": 1.7217001914978027,
      "logps/chosen": -240.90640258789062,
      "logps/ref_chosen": -240.93368530273438,
      "logps/ref_rejected": -235.3230743408203,
      "logps/rejected": -235.32559204101562,
      "loss": 5.544,
      "margin_dpo/margin_mean": 0.029784739017486572,
      "margin_dpo/margin_std": 0.7291895151138306,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": 0.07761001586914062,
      "KL/mean": 0.03460410237312317,
      "KL/rejected_KL_mean": -0.008405685424804688,
      "KL/std": 0.5668712258338928,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08601255714893341,
      "fcm_dpo/q_t": 0.4997849464416504,
      "grad_norm": 15.568114280700684,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": 1.7278506755828857,
      "logits/rejected": 1.7104250192642212,
      "logps/chosen": -273.5177001953125,
      "logps/ref_chosen": -273.5953369140625,
      "logps/ref_rejected": -234.49148559570312,
      "logps/rejected": -234.4998779296875,
      "loss": 5.5418,
      "margin_dpo/margin_mean": 0.0860128402709961,
      "margin_dpo/margin_std": 0.8191437721252441,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.09130287170410156,
      "KL/mean": 0.05760665237903595,
      "KL/rejected_KL_mean": 0.023929595947265625,
      "KL/std": 0.6006314158439636,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06736423075199127,
      "fcm_dpo/q_t": 0.4998316168785095,
      "grad_norm": 15.164607048034668,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": 2.239915609359741,
      "logits/rejected": 2.012601375579834,
      "logps/chosen": -314.1725769042969,
      "logps/ref_chosen": -314.26385498046875,
      "logps/ref_rejected": -242.06735229492188,
      "logps/rejected": -242.04339599609375,
      "loss": 5.5426,
      "margin_dpo/margin_mean": 0.06736427545547485,
      "margin_dpo/margin_std": 0.8173242807388306,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.14200973510742188,
      "KL/mean": 0.10720671713352203,
      "KL/rejected_KL_mean": 0.07239913940429688,
      "KL/std": 0.529901921749115,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06961091607809067,
      "fcm_dpo/q_t": 0.49982595443725586,
      "grad_norm": 15.404664039611816,
      "learning_rate": 2.8125e-07,
      "logits/chosen": 1.9246848821640015,
      "logits/rejected": 1.928609848022461,
      "logps/chosen": -240.59681701660156,
      "logps/ref_chosen": -240.7388153076172,
      "logps/ref_rejected": -237.8331298828125,
      "logps/rejected": -237.7607421875,
      "loss": 5.5424,
      "margin_dpo/margin_mean": 0.06961117684841156,
      "margin_dpo/margin_std": 0.6830191612243652,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.10881614685058594,
      "KL/mean": 0.04860023409128189,
      "KL/rejected_KL_mean": -0.011608123779296875,
      "KL/std": 0.6711025238037109,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12042583525180817,
      "fcm_dpo/q_t": 0.4996989369392395,
      "grad_norm": 14.712845802307129,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": 1.9957093000411987,
      "logits/rejected": 1.8888139724731445,
      "logps/chosen": -307.7205810546875,
      "logps/ref_chosen": -307.82940673828125,
      "logps/ref_rejected": -281.8109130859375,
      "logps/rejected": -281.822509765625,
      "loss": 5.5405,
      "margin_dpo/margin_mean": 0.1204247772693634,
      "margin_dpo/margin_std": 0.9529728889465332,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.05753135681152344,
      "KL/mean": 0.04166576266288757,
      "KL/rejected_KL_mean": 0.025800704956054688,
      "KL/std": 0.5961987376213074,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03173324465751648,
      "fcm_dpo/q_t": 0.49992066621780396,
      "grad_norm": 14.923717498779297,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.6469576358795166,
      "logits/rejected": 1.6370129585266113,
      "logps/chosen": -247.95623779296875,
      "logps/ref_chosen": -248.01376342773438,
      "logps/ref_rejected": -259.0014953613281,
      "logps/rejected": -258.9756774902344,
      "loss": 5.544,
      "margin_dpo/margin_mean": 0.03173212707042694,
      "margin_dpo/margin_std": 0.8410557508468628,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.11629486083984375,
      "KL/mean": 0.07024562358856201,
      "KL/rejected_KL_mean": 0.0241851806640625,
      "KL/std": 0.6172938942909241,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09210622310638428,
      "fcm_dpo/q_t": 0.4997697174549103,
      "grad_norm": 13.699516296386719,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 1.8209006786346436,
      "logits/rejected": 1.7459312677383423,
      "logps/chosen": -239.94651794433594,
      "logps/ref_chosen": -240.0628204345703,
      "logps/ref_rejected": -201.91021728515625,
      "logps/rejected": -201.88601684570312,
      "loss": 5.5416,
      "margin_dpo/margin_mean": 0.09210464358329773,
      "margin_dpo/margin_std": 0.9005764722824097,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.11916351318359375,
      "KL/mean": 0.09504290670156479,
      "KL/rejected_KL_mean": 0.0709228515625,
      "KL/std": 0.6422213912010193,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04823760688304901,
      "fcm_dpo/q_t": 0.499879390001297,
      "grad_norm": 15.587300300598145,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": 1.909905195236206,
      "logits/rejected": 1.8323357105255127,
      "logps/chosen": -294.0130615234375,
      "logps/ref_chosen": -294.1322326660156,
      "logps/ref_rejected": -265.8150634765625,
      "logps/rejected": -265.744140625,
      "loss": 5.5433,
      "margin_dpo/margin_mean": 0.04823726415634155,
      "margin_dpo/margin_std": 0.8094491362571716,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.2897014617919922,
      "KL/mean": 0.20227915048599243,
      "KL/rejected_KL_mean": 0.11484718322753906,
      "KL/std": 0.6836931109428406,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.17485807836055756,
      "fcm_dpo/q_t": 0.4995628595352173,
      "grad_norm": 14.935464859008789,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 1.9384809732437134,
      "logits/rejected": 1.8985525369644165,
      "logps/chosen": -275.36541748046875,
      "logps/ref_chosen": -275.6551208496094,
      "logps/ref_rejected": -253.5972137451172,
      "logps/rejected": -253.4823760986328,
      "loss": 5.5383,
      "margin_dpo/margin_mean": 0.17485833168029785,
      "margin_dpo/margin_std": 0.985473096370697,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.1765918731689453,
      "KL/mean": 0.1263720840215683,
      "KL/rejected_KL_mean": 0.07616043090820312,
      "KL/std": 0.7270158529281616,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10043685883283615,
      "fcm_dpo/q_t": 0.49974894523620605,
      "grad_norm": 17.664344787597656,
      "learning_rate": 3.4375e-07,
      "logits/chosen": 1.986114501953125,
      "logits/rejected": 1.8687976598739624,
      "logps/chosen": -332.79351806640625,
      "logps/ref_chosen": -332.9701232910156,
      "logps/ref_rejected": -278.2213439941406,
      "logps/rejected": -278.1451721191406,
      "loss": 5.5413,
      "margin_dpo/margin_mean": 0.10043707489967346,
      "margin_dpo/margin_std": 1.028347134590149,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.2259082794189453,
      "KL/mean": 0.20131874084472656,
      "KL/rejected_KL_mean": 0.17673301696777344,
      "KL/std": 0.6464886665344238,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04918156564235687,
      "fcm_dpo/q_t": 0.49987703561782837,
      "grad_norm": 13.827043533325195,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": 1.792829990386963,
      "logits/rejected": 1.8540301322937012,
      "logps/chosen": -258.2029724121094,
      "logps/ref_chosen": -258.42889404296875,
      "logps/ref_rejected": -259.5119934082031,
      "logps/rejected": -259.33526611328125,
      "loss": 5.5433,
      "margin_dpo/margin_mean": 0.04918116331100464,
      "margin_dpo/margin_std": 0.8877348899841309,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.3982410430908203,
      "KL/mean": 0.25082436203956604,
      "KL/rejected_KL_mean": 0.10341072082519531,
      "KL/std": 0.7977854013442993,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2948343753814697,
      "fcm_dpo/q_t": 0.4992629289627075,
      "grad_norm": 17.55563735961914,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": 1.8976614475250244,
      "logits/rejected": 1.8439788818359375,
      "logps/chosen": -290.2210998535156,
      "logps/ref_chosen": -290.6193542480469,
      "logps/ref_rejected": -280.2503967285156,
      "logps/rejected": -280.14697265625,
      "loss": 5.5335,
      "margin_dpo/margin_mean": 0.2948346436023712,
      "margin_dpo/margin_std": 1.108701467514038,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.29657554626464844,
      "KL/mean": 0.17259304225444794,
      "KL/rejected_KL_mean": 0.04860687255859375,
      "KL/std": 0.7573060989379883,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.24796709418296814,
      "fcm_dpo/q_t": 0.49938011169433594,
      "grad_norm": 15.190077781677246,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.9903556108474731,
      "logits/rejected": 1.9907883405685425,
      "logps/chosen": -303.2762145996094,
      "logps/ref_chosen": -303.57281494140625,
      "logps/ref_rejected": -294.56878662109375,
      "logps/rejected": -294.5201721191406,
      "loss": 5.5354,
      "margin_dpo/margin_mean": 0.24796608090400696,
      "margin_dpo/margin_std": 1.0593851804733276,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.3377208709716797,
      "KL/mean": 0.24843284487724304,
      "KL/rejected_KL_mean": 0.15913963317871094,
      "KL/std": 0.7184921503067017,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.17857247591018677,
      "fcm_dpo/q_t": 0.4995535910129547,
      "grad_norm": 15.264507293701172,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": 1.9910590648651123,
      "logits/rejected": 2.0508832931518555,
      "logps/chosen": -272.21575927734375,
      "logps/ref_chosen": -272.5534973144531,
      "logps/ref_rejected": -266.9136047363281,
      "logps/rejected": -266.7544860839844,
      "loss": 5.5381,
      "margin_dpo/margin_mean": 0.1785726696252823,
      "margin_dpo/margin_std": 0.8639269471168518,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 0.5904388427734375,
      "KL/mean": 0.35096678137779236,
      "KL/rejected_KL_mean": 0.11149406433105469,
      "KL/std": 0.8202947974205017,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.47893744707107544,
      "fcm_dpo/q_t": 0.49880272150039673,
      "grad_norm": 15.748991012573242,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": 1.6965254545211792,
      "logits/rejected": 1.625367522239685,
      "logps/chosen": -264.7961120605469,
      "logps/ref_chosen": -265.38653564453125,
      "logps/ref_rejected": -244.34774780273438,
      "logps/rejected": -244.23626708984375,
      "loss": 5.5262,
      "margin_dpo/margin_mean": 0.4789374768733978,
      "margin_dpo/margin_std": 1.0037331581115723,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 0.5296916961669922,
      "KL/mean": 0.32585608959198,
      "KL/rejected_KL_mean": 0.12202072143554688,
      "KL/std": 0.8493956923484802,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4076656699180603,
      "fcm_dpo/q_t": 0.49898087978363037,
      "grad_norm": 14.96629810333252,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.7624915838241577,
      "logits/rejected": 1.7462505102157593,
      "logps/chosen": -274.3641357421875,
      "logps/ref_chosen": -274.893798828125,
      "logps/ref_rejected": -244.3120880126953,
      "logps/rejected": -244.19007873535156,
      "loss": 5.529,
      "margin_dpo/margin_mean": 0.4076656103134155,
      "margin_dpo/margin_std": 1.056751012802124,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 0.3178386688232422,
      "KL/mean": 0.2065788209438324,
      "KL/rejected_KL_mean": 0.09532356262207031,
      "KL/std": 0.7507030963897705,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.22250071167945862,
      "fcm_dpo/q_t": 0.49944376945495605,
      "grad_norm": 15.45833683013916,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 2.059506893157959,
      "logits/rejected": 1.997917652130127,
      "logps/chosen": -284.4812927246094,
      "logps/ref_chosen": -284.7991027832031,
      "logps/ref_rejected": -248.8882598876953,
      "logps/rejected": -248.7929229736328,
      "loss": 5.5364,
      "margin_dpo/margin_mean": 0.22250135242938995,
      "margin_dpo/margin_std": 1.0247411727905273,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.4251117706298828,
      "KL/mean": 0.2930000126361847,
      "KL/rejected_KL_mean": 0.160888671875,
      "KL/std": 0.7375829219818115,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2642222046852112,
      "fcm_dpo/q_t": 0.49933943152427673,
      "grad_norm": 14.78955078125,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": 2.01818585395813,
      "logits/rejected": 2.0775935649871826,
      "logps/chosen": -245.41455078125,
      "logps/ref_chosen": -245.83966064453125,
      "logps/ref_rejected": -259.2870788574219,
      "logps/rejected": -259.126220703125,
      "loss": 5.5347,
      "margin_dpo/margin_mean": 0.26422324776649475,
      "margin_dpo/margin_std": 0.9873778820037842,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.4855918884277344,
      "KL/mean": 0.3133164346218109,
      "KL/rejected_KL_mean": 0.14104270935058594,
      "KL/std": 0.8921718001365662,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3445456027984619,
      "fcm_dpo/q_t": 0.49913865327835083,
      "grad_norm": 15.625173568725586,
      "learning_rate": 4.375e-07,
      "logits/chosen": 1.832188367843628,
      "logits/rejected": 1.8114113807678223,
      "logps/chosen": -307.9119567871094,
      "logps/ref_chosen": -308.39752197265625,
      "logps/ref_rejected": -270.3797912597656,
      "logps/rejected": -270.23876953125,
      "loss": 5.5315,
      "margin_dpo/margin_mean": 0.3445459008216858,
      "margin_dpo/margin_std": 1.1119972467422485,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 0.6565837860107422,
      "KL/mean": 0.40236926078796387,
      "KL/rejected_KL_mean": 0.1481609344482422,
      "KL/std": 0.8770255446434021,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5084190368652344,
      "fcm_dpo/q_t": 0.49872899055480957,
      "grad_norm": 15.809155464172363,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": 1.863312005996704,
      "logits/rejected": 1.862404227256775,
      "logps/chosen": -274.70513916015625,
      "logps/ref_chosen": -275.36175537109375,
      "logps/ref_rejected": -279.7540588378906,
      "logps/rejected": -279.6059265136719,
      "loss": 5.525,
      "margin_dpo/margin_mean": 0.5084197521209717,
      "margin_dpo/margin_std": 1.115541934967041,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 0.8092670440673828,
      "KL/mean": 0.5417796969413757,
      "KL/rejected_KL_mean": 0.27428245544433594,
      "KL/std": 0.9826558828353882,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5349997282028198,
      "fcm_dpo/q_t": 0.4986625909805298,
      "grad_norm": 16.84693717956543,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": 1.8046939373016357,
      "logits/rejected": 1.7692142724990845,
      "logps/chosen": -314.3109436035156,
      "logps/ref_chosen": -315.1202087402344,
      "logps/ref_rejected": -290.34210205078125,
      "logps/rejected": -290.06781005859375,
      "loss": 5.524,
      "margin_dpo/margin_mean": 0.534998893737793,
      "margin_dpo/margin_std": 1.2926781177520752,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 0.6850337982177734,
      "KL/mean": 0.3690078854560852,
      "KL/rejected_KL_mean": 0.052982330322265625,
      "KL/std": 0.963544487953186,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6320487260818481,
      "fcm_dpo/q_t": 0.49842002987861633,
      "grad_norm": 16.419578552246094,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": 1.7293682098388672,
      "logits/rejected": 1.7362051010131836,
      "logps/chosen": -292.2555847167969,
      "logps/ref_chosen": -292.94061279296875,
      "logps/ref_rejected": -264.0979309082031,
      "logps/rejected": -264.0449523925781,
      "loss": 5.5201,
      "margin_dpo/margin_mean": 0.6320484280586243,
      "margin_dpo/margin_std": 1.1884610652923584,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 0.6624011993408203,
      "KL/mean": 0.48483121395111084,
      "KL/rejected_KL_mean": 0.3072624206542969,
      "KL/std": 0.9581394195556641,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3551379442214966,
      "fcm_dpo/q_t": 0.49911218881607056,
      "grad_norm": 13.575396537780762,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": 1.710496187210083,
      "logits/rejected": 1.8097968101501465,
      "logps/chosen": -254.19027709960938,
      "logps/ref_chosen": -254.85269165039062,
      "logps/ref_rejected": -255.88380432128906,
      "logps/rejected": -255.5765380859375,
      "loss": 5.5312,
      "margin_dpo/margin_mean": 0.3551369607448578,
      "margin_dpo/margin_std": 1.3625694513320923,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 0.7664413452148438,
      "KL/mean": 0.4564923644065857,
      "KL/rejected_KL_mean": 0.14653396606445312,
      "KL/std": 1.0759146213531494,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6199021339416504,
      "fcm_dpo/q_t": 0.4984503984451294,
      "grad_norm": 14.850208282470703,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": 1.8286491632461548,
      "logits/rejected": 1.8065013885498047,
      "logps/chosen": -261.7413024902344,
      "logps/ref_chosen": -262.5077209472656,
      "logps/ref_rejected": -239.25320434570312,
      "logps/rejected": -239.106689453125,
      "loss": 5.5206,
      "margin_dpo/margin_mean": 0.6199018955230713,
      "margin_dpo/margin_std": 1.3536527156829834,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.8738307952880859,
      "KL/mean": 0.624643862247467,
      "KL/rejected_KL_mean": 0.37546539306640625,
      "KL/std": 1.174159049987793,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4983612298965454,
      "fcm_dpo/q_t": 0.4987541437149048,
      "grad_norm": 15.598930358886719,
      "learning_rate": 5e-07,
      "logits/chosen": 1.860569953918457,
      "logits/rejected": 1.9766619205474854,
      "logps/chosen": -248.654052734375,
      "logps/ref_chosen": -249.5278778076172,
      "logps/ref_rejected": -254.6465301513672,
      "logps/rejected": -254.2710723876953,
      "loss": 5.5255,
      "margin_dpo/margin_mean": 0.4983615279197693,
      "margin_dpo/margin_std": 1.5766419172286987,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 0.9579315185546875,
      "KL/mean": 0.7092702388763428,
      "KL/rejected_KL_mean": 0.4605979919433594,
      "KL/std": 1.1542375087738037,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4973296523094177,
      "fcm_dpo/q_t": 0.4987567365169525,
      "grad_norm": 14.601441383361816,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 2.0535237789154053,
      "logits/rejected": 2.0711452960968018,
      "logps/chosen": -291.54376220703125,
      "logps/ref_chosen": -292.501708984375,
      "logps/ref_rejected": -294.6496887207031,
      "logps/rejected": -294.1890869140625,
      "loss": 5.5256,
      "margin_dpo/margin_mean": 0.49733057618141174,
      "margin_dpo/margin_std": 1.5240452289581299,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 0.9704437255859375,
      "KL/mean": 0.6416925191879272,
      "KL/rejected_KL_mean": 0.31293487548828125,
      "KL/std": 1.3135533332824707,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6575091481208801,
      "fcm_dpo/q_t": 0.49835634231567383,
      "grad_norm": 14.740985870361328,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 1.6504236459732056,
      "logits/rejected": 1.7731714248657227,
      "logps/chosen": -258.8914489746094,
      "logps/ref_chosen": -259.8619079589844,
      "logps/ref_rejected": -293.1994934082031,
      "logps/rejected": -292.8865661621094,
      "loss": 5.5193,
      "margin_dpo/margin_mean": 0.657508909702301,
      "margin_dpo/margin_std": 1.8060803413391113,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 1.2036418914794922,
      "KL/mean": 0.7828872799873352,
      "KL/rejected_KL_mean": 0.36214447021484375,
      "KL/std": 1.4433791637420654,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8414902687072754,
      "fcm_dpo/q_t": 0.49789655208587646,
      "grad_norm": 17.082773208618164,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": 2.0648694038391113,
      "logits/rejected": 1.9914741516113281,
      "logps/chosen": -297.2216491699219,
      "logps/ref_chosen": -298.42529296875,
      "logps/ref_rejected": -278.5128479003906,
      "logps/rejected": -278.1506652832031,
      "loss": 5.512,
      "margin_dpo/margin_mean": 0.8414908647537231,
      "margin_dpo/margin_std": 1.8258335590362549,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 1.2386970520019531,
      "KL/mean": 0.8474341630935669,
      "KL/rejected_KL_mean": 0.45616912841796875,
      "KL/std": 1.4484680891036987,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7825204730033875,
      "fcm_dpo/q_t": 0.498043954372406,
      "grad_norm": 15.948686599731445,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": 2.0358541011810303,
      "logits/rejected": 2.1113274097442627,
      "logps/chosen": -274.06982421875,
      "logps/ref_chosen": -275.3085021972656,
      "logps/ref_rejected": -267.97381591796875,
      "logps/rejected": -267.5176696777344,
      "loss": 5.5144,
      "margin_dpo/margin_mean": 0.78252112865448,
      "margin_dpo/margin_std": 1.8762080669403076,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 1.3127479553222656,
      "KL/mean": 0.8966032862663269,
      "KL/rejected_KL_mean": 0.48046112060546875,
      "KL/std": 1.5586217641830444,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8322992324829102,
      "fcm_dpo/q_t": 0.49791961908340454,
      "grad_norm": 14.533020973205566,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": 1.9465746879577637,
      "logits/rejected": 2.00445294380188,
      "logps/chosen": -296.8345947265625,
      "logps/ref_chosen": -298.1473388671875,
      "logps/ref_rejected": -269.7539367675781,
      "logps/rejected": -269.2734680175781,
      "loss": 5.5125,
      "margin_dpo/margin_mean": 0.8322997689247131,
      "margin_dpo/margin_std": 2.0763347148895264,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.0745601654052734,
      "KL/mean": 0.8155922889709473,
      "KL/rejected_KL_mean": 0.5566215515136719,
      "KL/std": 1.2993042469024658,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5179425477981567,
      "fcm_dpo/q_t": 0.49870526790618896,
      "grad_norm": 12.894977569580078,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": 1.7002758979797363,
      "logits/rejected": 1.7541223764419556,
      "logps/chosen": -215.6754150390625,
      "logps/ref_chosen": -216.74998474121094,
      "logps/ref_rejected": -202.7234649658203,
      "logps/rejected": -202.16683959960938,
      "loss": 5.5248,
      "margin_dpo/margin_mean": 0.5179433822631836,
      "margin_dpo/margin_std": 1.6850924491882324,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 1.2860260009765625,
      "KL/mean": 0.7953134775161743,
      "KL/rejected_KL_mean": 0.30460166931152344,
      "KL/std": 1.789381742477417,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9814178347587585,
      "fcm_dpo/q_t": 0.49754685163497925,
      "grad_norm": 14.379996299743652,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": 1.4942225217819214,
      "logits/rejected": 1.5768795013427734,
      "logps/chosen": -242.4053955078125,
      "logps/ref_chosen": -243.69140625,
      "logps/ref_rejected": -234.970947265625,
      "logps/rejected": -234.66632080078125,
      "loss": 5.5066,
      "margin_dpo/margin_mean": 0.9814174771308899,
      "margin_dpo/margin_std": 2.165114164352417,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 1.7764663696289062,
      "KL/mean": 1.0924129486083984,
      "KL/rejected_KL_mean": 0.4083709716796875,
      "KL/std": 1.7504782676696777,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3680773973464966,
      "fcm_dpo/q_t": 0.4965808391571045,
      "grad_norm": 17.579410552978516,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": 2.083345413208008,
      "logits/rejected": 1.9848374128341675,
      "logps/chosen": -308.9698486328125,
      "logps/ref_chosen": -310.746337890625,
      "logps/ref_rejected": -270.6407165527344,
      "logps/rejected": -270.2323303222656,
      "loss": 5.4912,
      "margin_dpo/margin_mean": 1.3680768013000488,
      "margin_dpo/margin_std": 2.1828365325927734,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 1.287679672241211,
      "KL/mean": 0.8008712530136108,
      "KL/rejected_KL_mean": 0.3140678405761719,
      "KL/std": 1.5249439477920532,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.973612904548645,
      "fcm_dpo/q_t": 0.4975661039352417,
      "grad_norm": 14.852359771728516,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": 1.6281938552856445,
      "logits/rejected": 1.6098155975341797,
      "logps/chosen": -257.52532958984375,
      "logps/ref_chosen": -258.8130187988281,
      "logps/ref_rejected": -268.2101745605469,
      "logps/rejected": -267.8961181640625,
      "loss": 5.5068,
      "margin_dpo/margin_mean": 0.973612904548645,
      "margin_dpo/margin_std": 1.8885267972946167,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 1.3679084777832031,
      "KL/mean": 0.9293200373649597,
      "KL/rejected_KL_mean": 0.49073219299316406,
      "KL/std": 1.9969879388809204,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8771830201148987,
      "fcm_dpo/q_t": 0.49780765175819397,
      "grad_norm": 15.218059539794922,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": 1.8456453084945679,
      "logits/rejected": 1.7928926944732666,
      "logps/chosen": -275.87353515625,
      "logps/ref_chosen": -277.2414245605469,
      "logps/ref_rejected": -259.13519287109375,
      "logps/rejected": -258.6444396972656,
      "loss": 5.5109,
      "margin_dpo/margin_mean": 0.877183198928833,
      "margin_dpo/margin_std": 2.609489917755127,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 1.5025253295898438,
      "KL/mean": 0.9047397375106812,
      "KL/rejected_KL_mean": 0.3069572448730469,
      "KL/std": 1.6809602975845337,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1955626010894775,
      "fcm_dpo/q_t": 0.49701154232025146,
      "grad_norm": 14.69546127319336,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.6742585897445679,
      "logits/rejected": 1.7060997486114502,
      "logps/chosen": -249.8433074951172,
      "logps/ref_chosen": -251.34585571289062,
      "logps/ref_rejected": -251.4120635986328,
      "logps/rejected": -251.1051025390625,
      "loss": 5.4981,
      "margin_dpo/margin_mean": 1.1955628395080566,
      "margin_dpo/margin_std": 2.276646375656128,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 1.7299690246582031,
      "KL/mean": 1.1431163549423218,
      "KL/rejected_KL_mean": 0.5562629699707031,
      "KL/std": 1.8086351156234741,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1737115383148193,
      "fcm_dpo/q_t": 0.4970662593841553,
      "grad_norm": 15.030417442321777,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 1.9662216901779175,
      "logits/rejected": 1.9363595247268677,
      "logps/chosen": -267.25262451171875,
      "logps/ref_chosen": -268.98260498046875,
      "logps/ref_rejected": -249.34474182128906,
      "logps/rejected": -248.7884979248047,
      "loss": 5.499,
      "margin_dpo/margin_mean": 1.1737103462219238,
      "margin_dpo/margin_std": 2.3234198093414307,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 1.9705009460449219,
      "KL/mean": 1.4003628492355347,
      "KL/rejected_KL_mean": 0.8302192687988281,
      "KL/std": 1.9880305528640747,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.140271782875061,
      "fcm_dpo/q_t": 0.4971500635147095,
      "grad_norm": 14.689555168151855,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 1.8667433261871338,
      "logits/rejected": 1.8346644639968872,
      "logps/chosen": -262.2330017089844,
      "logps/ref_chosen": -264.2034912109375,
      "logps/ref_rejected": -260.2911376953125,
      "logps/rejected": -259.4609375,
      "loss": 5.5004,
      "margin_dpo/margin_mean": 1.1402709484100342,
      "margin_dpo/margin_std": 2.6029555797576904,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 1.9411506652832031,
      "KL/mean": 1.2579898834228516,
      "KL/rejected_KL_mean": 0.5748271942138672,
      "KL/std": 2.1325440406799316,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3663285970687866,
      "fcm_dpo/q_t": 0.49658501148223877,
      "grad_norm": 14.777442932128906,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": 1.8309917449951172,
      "logits/rejected": 1.8587820529937744,
      "logps/chosen": -269.5271301269531,
      "logps/ref_chosen": -271.4682922363281,
      "logps/ref_rejected": -273.0804748535156,
      "logps/rejected": -272.5056457519531,
      "loss": 5.4916,
      "margin_dpo/margin_mean": 1.3663294315338135,
      "margin_dpo/margin_std": 2.811391830444336,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 2.314973831176758,
      "KL/mean": 1.5736117362976074,
      "KL/rejected_KL_mean": 0.8322505950927734,
      "KL/std": 2.369312286376953,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4827196598052979,
      "fcm_dpo/q_t": 0.49629443883895874,
      "grad_norm": 15.67747688293457,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": 1.877082347869873,
      "logits/rejected": 1.887677788734436,
      "logps/chosen": -271.2884216308594,
      "logps/ref_chosen": -273.6033935546875,
      "logps/ref_rejected": -259.54107666015625,
      "logps/rejected": -258.7088317871094,
      "loss": 5.4871,
      "margin_dpo/margin_mean": 1.4827189445495605,
      "margin_dpo/margin_std": 3.1460988521575928,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 2.2960662841796875,
      "KL/mean": 1.734660029411316,
      "KL/rejected_KL_mean": 1.1732597351074219,
      "KL/std": 2.087191581726074,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1228063106536865,
      "fcm_dpo/q_t": 0.4971935451030731,
      "grad_norm": 14.487714767456055,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": 1.6395456790924072,
      "logits/rejected": 1.7425241470336914,
      "logps/chosen": -256.2215881347656,
      "logps/ref_chosen": -258.5176696777344,
      "logps/ref_rejected": -264.9944763183594,
      "logps/rejected": -263.82122802734375,
      "loss": 5.5012,
      "margin_dpo/margin_mean": 1.1228067874908447,
      "margin_dpo/margin_std": 2.7316336631774902,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 2.2542800903320312,
      "KL/mean": 1.502932071685791,
      "KL/rejected_KL_mean": 0.7515830993652344,
      "KL/std": 2.722247362136841,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5026881694793701,
      "fcm_dpo/q_t": 0.4962446987628937,
      "grad_norm": 14.81159782409668,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": 1.8942333459854126,
      "logits/rejected": 1.9933359622955322,
      "logps/chosen": -265.255126953125,
      "logps/ref_chosen": -267.5094299316406,
      "logps/ref_rejected": -277.03228759765625,
      "logps/rejected": -276.28070068359375,
      "loss": 5.4866,
      "margin_dpo/margin_mean": 1.502687931060791,
      "margin_dpo/margin_std": 3.5797061920166016,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 2.638578414916992,
      "KL/mean": 2.002694845199585,
      "KL/rejected_KL_mean": 1.3668155670166016,
      "KL/std": 2.574723243713379,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2717528343200684,
      "fcm_dpo/q_t": 0.49682140350341797,
      "grad_norm": 14.236442565917969,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": 1.9177266359329224,
      "logits/rejected": 1.9541107416152954,
      "logps/chosen": -254.82635498046875,
      "logps/ref_chosen": -257.4649353027344,
      "logps/ref_rejected": -247.29161071777344,
      "logps/rejected": -245.92481994628906,
      "loss": 5.4956,
      "margin_dpo/margin_mean": 1.2717523574829102,
      "margin_dpo/margin_std": 3.168820381164551,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 2.471128463745117,
      "KL/mean": 1.6251890659332275,
      "KL/rejected_KL_mean": 0.7792491912841797,
      "KL/std": 2.897090196609497,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6918779611587524,
      "fcm_dpo/q_t": 0.4957726001739502,
      "grad_norm": 14.758001327514648,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": 1.6594077348709106,
      "logits/rejected": 1.6014090776443481,
      "logps/chosen": -288.0130615234375,
      "logps/ref_chosen": -290.4841613769531,
      "logps/ref_rejected": -260.8398132324219,
      "logps/rejected": -260.060546875,
      "loss": 5.4793,
      "margin_dpo/margin_mean": 1.6918784379959106,
      "margin_dpo/margin_std": 3.766761541366577,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 2.7915287017822266,
      "KL/mean": 1.713508129119873,
      "KL/rejected_KL_mean": 0.6354866027832031,
      "KL/std": 2.9748852252960205,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.1560559272766113,
      "fcm_dpo/q_t": 0.4946123957633972,
      "grad_norm": 15.082676887512207,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": 1.736380696296692,
      "logits/rejected": 1.7600477933883667,
      "logps/chosen": -254.8460693359375,
      "logps/ref_chosen": -257.6375732421875,
      "logps/ref_rejected": -235.6666717529297,
      "logps/rejected": -235.03118896484375,
      "loss": 5.461,
      "margin_dpo/margin_mean": 2.1560559272766113,
      "margin_dpo/margin_std": 3.8814003467559814,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 3.4765377044677734,
      "KL/mean": 2.389936923980713,
      "KL/rejected_KL_mean": 1.3033294677734375,
      "KL/std": 3.0012106895446777,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.173213243484497,
      "fcm_dpo/q_t": 0.49456945061683655,
      "grad_norm": 15.587392807006836,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.7783091068267822,
      "logits/rejected": 1.7419722080230713,
      "logps/chosen": -283.9886779785156,
      "logps/ref_chosen": -287.4651794433594,
      "logps/ref_rejected": -253.4770965576172,
      "logps/rejected": -252.1737518310547,
      "loss": 5.4601,
      "margin_dpo/margin_mean": 2.1732125282287598,
      "margin_dpo/margin_std": 3.59454345703125,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 3.625062942504883,
      "KL/mean": 2.61733341217041,
      "KL/rejected_KL_mean": 1.6096076965332031,
      "KL/std": 3.2491953372955322,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0154664516448975,
      "fcm_dpo/q_t": 0.49496328830718994,
      "grad_norm": 14.981285095214844,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 1.926743745803833,
      "logits/rejected": 1.9310632944107056,
      "logps/chosen": -277.3103332519531,
      "logps/ref_chosen": -280.9353942871094,
      "logps/ref_rejected": -275.998779296875,
      "logps/rejected": -274.3891906738281,
      "loss": 5.467,
      "margin_dpo/margin_mean": 2.0154664516448975,
      "margin_dpo/margin_std": 4.222230434417725,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 3.0893211364746094,
      "KL/mean": 2.062171459197998,
      "KL/rejected_KL_mean": 1.0350170135498047,
      "KL/std": 3.059035539627075,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.054305076599121,
      "fcm_dpo/q_t": 0.4948667585849762,
      "grad_norm": 14.565410614013672,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": 1.9909703731536865,
      "logits/rejected": 2.150257110595703,
      "logps/chosen": -227.05618286132812,
      "logps/ref_chosen": -230.14553833007812,
      "logps/ref_rejected": -253.37661743164062,
      "logps/rejected": -252.3415985107422,
      "loss": 5.4652,
      "margin_dpo/margin_mean": 2.0543031692504883,
      "margin_dpo/margin_std": 3.973010301589966,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 3.846883773803711,
      "KL/mean": 2.855379343032837,
      "KL/rejected_KL_mean": 1.8638763427734375,
      "KL/std": 3.3217928409576416,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9830095767974854,
      "fcm_dpo/q_t": 0.49504488706588745,
      "grad_norm": 14.660309791564941,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": 1.7757024765014648,
      "logits/rejected": 1.801865816116333,
      "logps/chosen": -261.0193786621094,
      "logps/ref_chosen": -264.86627197265625,
      "logps/ref_rejected": -245.21304321289062,
      "logps/rejected": -243.34915161132812,
      "loss": 5.4685,
      "margin_dpo/margin_mean": 1.9830081462860107,
      "margin_dpo/margin_std": 4.586266994476318,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.7414321899414062,
      "KL/mean": 2.815011501312256,
      "KL/rejected_KL_mean": 1.8886127471923828,
      "KL/std": 3.515139579772949,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8528265953063965,
      "fcm_dpo/q_t": 0.4953702390193939,
      "grad_norm": 16.46287727355957,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": 1.9288008213043213,
      "logits/rejected": 1.9863770008087158,
      "logps/chosen": -330.8121032714844,
      "logps/ref_chosen": -334.55352783203125,
      "logps/ref_rejected": -325.8403015136719,
      "logps/rejected": -323.95166015625,
      "loss": 5.4736,
      "margin_dpo/margin_mean": 1.8528268337249756,
      "margin_dpo/margin_std": 4.464634418487549,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.68433952331543,
      "KL/mean": 3.2860584259033203,
      "KL/rejected_KL_mean": 1.8877716064453125,
      "KL/std": 3.6699204444885254,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.7965736389160156,
      "fcm_dpo/q_t": 0.49301251769065857,
      "grad_norm": 15.343767166137695,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 1.7219873666763306,
      "logits/rejected": 1.8002417087554932,
      "logps/chosen": -255.737548828125,
      "logps/ref_chosen": -260.421875,
      "logps/ref_rejected": -273.0882263183594,
      "logps/rejected": -271.200439453125,
      "loss": 5.4361,
      "margin_dpo/margin_mean": 2.7965729236602783,
      "margin_dpo/margin_std": 4.23710298538208,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.558460235595703,
      "KL/mean": 3.0708131790161133,
      "KL/rejected_KL_mean": 1.5831737518310547,
      "KL/std": 3.819056987762451,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9753007888793945,
      "fcm_dpo/q_t": 0.49256908893585205,
      "grad_norm": 16.729440689086914,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": 1.8601655960083008,
      "logits/rejected": 1.8498300313949585,
      "logps/chosen": -267.1769714355469,
      "logps/ref_chosen": -271.7354431152344,
      "logps/ref_rejected": -262.64111328125,
      "logps/rejected": -261.0579833984375,
      "loss": 5.4295,
      "margin_dpo/margin_mean": 2.9753010272979736,
      "margin_dpo/margin_std": 4.757770538330078,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 4.880439758300781,
      "KL/mean": 3.665940284729004,
      "KL/rejected_KL_mean": 2.451446533203125,
      "KL/std": 4.049179553985596,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.4289913177490234,
      "fcm_dpo/q_t": 0.49393343925476074,
      "grad_norm": 14.857162475585938,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": 1.7421700954437256,
      "logits/rejected": 1.5841397047042847,
      "logps/chosen": -289.7901916503906,
      "logps/ref_chosen": -294.6706237792969,
      "logps/ref_rejected": -247.6900177001953,
      "logps/rejected": -245.23858642578125,
      "loss": 5.4514,
      "margin_dpo/margin_mean": 2.4289917945861816,
      "margin_dpo/margin_std": 5.023187637329102,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 5.149021148681641,
      "KL/mean": 3.6707019805908203,
      "KL/rejected_KL_mean": 2.1923751831054688,
      "KL/std": 3.7432785034179688,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9566397666931152,
      "fcm_dpo/q_t": 0.4926159679889679,
      "grad_norm": 14.656742095947266,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": 1.586308240890503,
      "logits/rejected": 1.50998055934906,
      "logps/chosen": -243.00326538085938,
      "logps/ref_chosen": -248.1522979736328,
      "logps/ref_rejected": -209.12808227539062,
      "logps/rejected": -206.93568420410156,
      "loss": 5.4307,
      "margin_dpo/margin_mean": 2.956638813018799,
      "margin_dpo/margin_std": 5.299832820892334,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 5.036802291870117,
      "KL/mean": 3.534621238708496,
      "KL/rejected_KL_mean": 2.0324363708496094,
      "KL/std": 4.84501838684082,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0043625831604004,
      "fcm_dpo/q_t": 0.49248701333999634,
      "grad_norm": 16.782224655151367,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": 1.8136450052261353,
      "logits/rejected": 1.8777077198028564,
      "logps/chosen": -254.4375762939453,
      "logps/ref_chosen": -259.4743957519531,
      "logps/ref_rejected": -246.9627227783203,
      "logps/rejected": -244.93026733398438,
      "loss": 5.4307,
      "margin_dpo/margin_mean": 3.0043623447418213,
      "margin_dpo/margin_std": 6.361856937408447,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 5.390218734741211,
      "KL/mean": 4.0547027587890625,
      "KL/rejected_KL_mean": 2.7191925048828125,
      "KL/std": 4.7152323722839355,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.6710262298583984,
      "fcm_dpo/q_t": 0.4933294653892517,
      "grad_norm": 15.686224937438965,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.6375982761383057,
      "logits/rejected": 1.626039743423462,
      "logps/chosen": -280.53631591796875,
      "logps/ref_chosen": -285.926513671875,
      "logps/ref_rejected": -253.77996826171875,
      "logps/rejected": -251.06077575683594,
      "loss": 5.4428,
      "margin_dpo/margin_mean": 2.6710267066955566,
      "margin_dpo/margin_std": 5.671087265014648,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 6.038295745849609,
      "KL/mean": 4.603489398956299,
      "KL/rejected_KL_mean": 3.168682098388672,
      "KL/std": 4.497054100036621,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.8696088790893555,
      "fcm_dpo/q_t": 0.49283161759376526,
      "grad_norm": 14.713410377502441,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 1.928248405456543,
      "logits/rejected": 1.88520348072052,
      "logps/chosen": -256.555908203125,
      "logps/ref_chosen": -262.5942077636719,
      "logps/ref_rejected": -236.687744140625,
      "logps/rejected": -233.51905822753906,
      "loss": 5.4357,
      "margin_dpo/margin_mean": 2.8696110248565674,
      "margin_dpo/margin_std": 6.379678726196289,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 6.572536468505859,
      "KL/mean": 5.037225246429443,
      "KL/rejected_KL_mean": 3.501911163330078,
      "KL/std": 4.380051136016846,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.070629358291626,
      "fcm_dpo/q_t": 0.4923320412635803,
      "grad_norm": 15.652230262756348,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": 1.8196052312850952,
      "logits/rejected": 1.7735058069229126,
      "logps/chosen": -264.47064208984375,
      "logps/ref_chosen": -271.04315185546875,
      "logps/ref_rejected": -234.03359985351562,
      "logps/rejected": -230.53167724609375,
      "loss": 5.4268,
      "margin_dpo/margin_mean": 3.070629358291626,
      "margin_dpo/margin_std": 5.494808197021484,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 6.641376495361328,
      "KL/mean": 4.8719635009765625,
      "KL/rejected_KL_mean": 3.1025524139404297,
      "KL/std": 4.892078876495361,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.538829803466797,
      "fcm_dpo/q_t": 0.49116700887680054,
      "grad_norm": 15.459914207458496,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": 1.555117130279541,
      "logits/rejected": 1.620307445526123,
      "logps/chosen": -259.6439208984375,
      "logps/ref_chosen": -266.2852783203125,
      "logps/ref_rejected": -274.32501220703125,
      "logps/rejected": -271.22247314453125,
      "loss": 5.4097,
      "margin_dpo/margin_mean": 3.5388307571411133,
      "margin_dpo/margin_std": 6.5827765464782715,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 6.8608245849609375,
      "KL/mean": 5.45859432220459,
      "KL/rejected_KL_mean": 4.056364059448242,
      "KL/std": 5.145233154296875,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.8044631481170654,
      "fcm_dpo/q_t": 0.4929939806461334,
      "grad_norm": 15.205020904541016,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": 1.717816710472107,
      "logits/rejected": 1.696789026260376,
      "logps/chosen": -279.9197998046875,
      "logps/ref_chosen": -286.7806091308594,
      "logps/ref_rejected": -276.1176452636719,
      "logps/rejected": -272.0613098144531,
      "loss": 5.4388,
      "margin_dpo/margin_mean": 2.8044614791870117,
      "margin_dpo/margin_std": 6.724213600158691,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": 6.483240127563477,
      "KL/mean": 5.469961166381836,
      "KL/rejected_KL_mean": 4.456682205200195,
      "KL/std": 4.455060958862305,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.026555061340332,
      "fcm_dpo/q_t": 0.4949374496936798,
      "grad_norm": 13.927360534667969,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": 1.8325409889221191,
      "logits/rejected": 1.9331657886505127,
      "logps/chosen": -228.546630859375,
      "logps/ref_chosen": -235.02987670898438,
      "logps/ref_rejected": -259.80999755859375,
      "logps/rejected": -255.35330200195312,
      "loss": 5.468,
      "margin_dpo/margin_mean": 2.0265555381774902,
      "margin_dpo/margin_std": 5.47263240814209,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 8.016817092895508,
      "KL/mean": 6.161609649658203,
      "KL/rejected_KL_mean": 4.3064117431640625,
      "KL/std": 5.604544639587402,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.7104101181030273,
      "fcm_dpo/q_t": 0.49073925614356995,
      "grad_norm": 16.24886131286621,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": 1.8381941318511963,
      "logits/rejected": 1.8788025379180908,
      "logps/chosen": -293.2959289550781,
      "logps/ref_chosen": -301.312744140625,
      "logps/ref_rejected": -288.8368225097656,
      "logps/rejected": -284.5304260253906,
      "loss": 5.4042,
      "margin_dpo/margin_mean": 3.7104103565216064,
      "margin_dpo/margin_std": 7.605499267578125,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": 8.245185852050781,
      "KL/mean": 6.189609527587891,
      "KL/rejected_KL_mean": 4.134031295776367,
      "KL/std": 5.4706220626831055,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.111157417297363,
      "fcm_dpo/q_t": 0.4897368550300598,
      "grad_norm": 15.45569133758545,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": 1.6008427143096924,
      "logits/rejected": 1.5762137174606323,
      "logps/chosen": -257.4208068847656,
      "logps/ref_chosen": -265.6659851074219,
      "logps/ref_rejected": -242.0697479248047,
      "logps/rejected": -237.9357147216797,
      "loss": 5.3868,
      "margin_dpo/margin_mean": 4.11115837097168,
      "margin_dpo/margin_std": 6.445609092712402,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": 7.379281997680664,
      "KL/mean": 5.766687870025635,
      "KL/rejected_KL_mean": 4.154087066650391,
      "KL/std": 5.497285842895508,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.2251944541931152,
      "fcm_dpo/q_t": 0.49194782972335815,
      "grad_norm": 14.074041366577148,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 1.8008042573928833,
      "logits/rejected": 1.8226231336593628,
      "logps/chosen": -253.3421630859375,
      "logps/ref_chosen": -260.721435546875,
      "logps/ref_rejected": -262.6593322753906,
      "logps/rejected": -258.5052185058594,
      "loss": 5.4225,
      "margin_dpo/margin_mean": 3.2251956462860107,
      "margin_dpo/margin_std": 7.017040252685547,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": 9.12533950805664,
      "KL/mean": 7.186056613922119,
      "KL/rejected_KL_mean": 5.246770858764648,
      "KL/std": 5.278353691101074,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.878573417663574,
      "fcm_dpo/q_t": 0.4903184175491333,
      "grad_norm": 15.827242851257324,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": 1.7324349880218506,
      "logits/rejected": 1.6902742385864258,
      "logps/chosen": -247.87745666503906,
      "logps/ref_chosen": -257.0028076171875,
      "logps/ref_rejected": -254.66232299804688,
      "logps/rejected": -249.41555786132812,
      "loss": 5.396,
      "margin_dpo/margin_mean": 3.878573417663574,
      "margin_dpo/margin_std": 6.301517486572266,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": 9.00086784362793,
      "KL/mean": 7.2712602615356445,
      "KL/rejected_KL_mean": 5.541656494140625,
      "KL/std": 5.559556007385254,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.4592151641845703,
      "fcm_dpo/q_t": 0.491364449262619,
      "grad_norm": 15.268774032592773,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 2.0697224140167236,
      "logits/rejected": 2.0166680812835693,
      "logps/chosen": -289.4095153808594,
      "logps/ref_chosen": -298.4103698730469,
      "logps/ref_rejected": -262.8761901855469,
      "logps/rejected": -257.3345642089844,
      "loss": 5.4132,
      "margin_dpo/margin_mean": 3.4592151641845703,
      "margin_dpo/margin_std": 6.73825216293335,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": 8.879470825195312,
      "KL/mean": 7.150986671447754,
      "KL/rejected_KL_mean": 5.422502517700195,
      "KL/std": 5.440098285675049,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.4569625854492188,
      "fcm_dpo/q_t": 0.491374671459198,
      "grad_norm": 14.684395790100098,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 1.6278072595596313,
      "logits/rejected": 1.6702879667282104,
      "logps/chosen": -218.6444091796875,
      "logps/ref_chosen": -227.5238800048828,
      "logps/ref_rejected": -227.11428833007812,
      "logps/rejected": -221.69178771972656,
      "loss": 5.4139,
      "margin_dpo/margin_mean": 3.4569625854492188,
      "margin_dpo/margin_std": 7.350286960601807,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": 8.972318649291992,
      "KL/mean": 7.081892013549805,
      "KL/rejected_KL_mean": 5.191457748413086,
      "KL/std": 6.646557807922363,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.780872344970703,
      "fcm_dpo/q_t": 0.49055975675582886,
      "grad_norm": 15.561689376831055,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": 1.6864891052246094,
      "logits/rejected": 1.5810626745224,
      "logps/chosen": -266.409912109375,
      "logps/ref_chosen": -275.3822326660156,
      "logps/ref_rejected": -269.8356018066406,
      "logps/rejected": -264.6441345214844,
      "loss": 5.4017,
      "margin_dpo/margin_mean": 3.780871868133545,
      "margin_dpo/margin_std": 7.682947158813477,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": 9.347759246826172,
      "KL/mean": 7.530686378479004,
      "KL/rejected_KL_mean": 5.713617324829102,
      "KL/std": 5.773220062255859,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.634143590927124,
      "fcm_dpo/q_t": 0.4909318685531616,
      "grad_norm": 14.155987739562988,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": 1.8359055519104004,
      "logits/rejected": 1.8261051177978516,
      "logps/chosen": -218.76861572265625,
      "logps/ref_chosen": -228.1163787841797,
      "logps/ref_rejected": -228.26266479492188,
      "logps/rejected": -222.54904174804688,
      "loss": 5.4068,
      "margin_dpo/margin_mean": 3.6341428756713867,
      "margin_dpo/margin_std": 7.2140374183654785,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": 10.118505477905273,
      "KL/mean": 8.063931465148926,
      "KL/rejected_KL_mean": 6.00935173034668,
      "KL/std": 6.673925876617432,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.109146595001221,
      "fcm_dpo/q_t": 0.4897528886795044,
      "grad_norm": 15.576948165893555,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": 1.64767324924469,
      "logits/rejected": 1.7312625646591187,
      "logps/chosen": -273.90875244140625,
      "logps/ref_chosen": -284.0272521972656,
      "logps/ref_rejected": -282.1720275878906,
      "logps/rejected": -276.16265869140625,
      "loss": 5.3908,
      "margin_dpo/margin_mean": 4.109147071838379,
      "margin_dpo/margin_std": 8.688535690307617,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": 9.934755325317383,
      "KL/mean": 7.648684501647949,
      "KL/rejected_KL_mean": 5.362621307373047,
      "KL/std": 6.8923139572143555,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.572126388549805,
      "fcm_dpo/q_t": 0.48859941959381104,
      "grad_norm": 26.926774978637695,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": 1.8692549467086792,
      "logits/rejected": 1.8034648895263672,
      "logps/chosen": -293.1150817871094,
      "logps/ref_chosen": -303.0498352050781,
      "logps/ref_rejected": -259.8980712890625,
      "logps/rejected": -254.53546142578125,
      "loss": 5.3727,
      "margin_dpo/margin_mean": 4.5721259117126465,
      "margin_dpo/margin_std": 8.830195426940918,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": 9.973358154296875,
      "KL/mean": 7.850517749786377,
      "KL/rejected_KL_mean": 5.727680206298828,
      "KL/std": 7.086856842041016,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.245677471160889,
      "fcm_dpo/q_t": 0.4894119203090668,
      "grad_norm": 16.132535934448242,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": 1.862799882888794,
      "logits/rejected": 1.8252724409103394,
      "logps/chosen": -260.61431884765625,
      "logps/ref_chosen": -270.58770751953125,
      "logps/ref_rejected": -238.90069580078125,
      "logps/rejected": -233.17300415039062,
      "loss": 5.3844,
      "margin_dpo/margin_mean": 4.245677947998047,
      "margin_dpo/margin_std": 8.014387130737305,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": 9.782859802246094,
      "KL/mean": 7.483928680419922,
      "KL/rejected_KL_mean": 5.185003280639648,
      "KL/std": 6.922455787658691,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.597842216491699,
      "fcm_dpo/q_t": 0.48854711651802063,
      "grad_norm": 15.29644775390625,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": 1.8745461702346802,
      "logits/rejected": 1.7610818147659302,
      "logps/chosen": -286.4026794433594,
      "logps/ref_chosen": -296.1855773925781,
      "logps/ref_rejected": -260.35333251953125,
      "logps/rejected": -255.16831970214844,
      "loss": 5.3733,
      "margin_dpo/margin_mean": 4.597841262817383,
      "margin_dpo/margin_std": 9.688837051391602,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": 10.038862228393555,
      "KL/mean": 7.679838180541992,
      "KL/rejected_KL_mean": 5.320802688598633,
      "KL/std": 7.337795257568359,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.7180633544921875,
      "fcm_dpo/q_t": 0.48824232816696167,
      "grad_norm": 15.715185165405273,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": 1.865659236907959,
      "logits/rejected": 1.8456714153289795,
      "logps/chosen": -267.6764831542969,
      "logps/ref_chosen": -277.7153625488281,
      "logps/ref_rejected": -272.9094543457031,
      "logps/rejected": -267.588623046875,
      "loss": 5.3692,
      "margin_dpo/margin_mean": 4.718062877655029,
      "margin_dpo/margin_std": 9.412782669067383,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": 10.27444076538086,
      "KL/mean": 7.701909065246582,
      "KL/rejected_KL_mean": 5.12938117980957,
      "KL/std": 7.546491622924805,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.145062446594238,
      "fcm_dpo/q_t": 0.4871649146080017,
      "grad_norm": 15.160264015197754,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": 1.7857860326766968,
      "logits/rejected": 1.7604609727859497,
      "logps/chosen": -262.26336669921875,
      "logps/ref_chosen": -272.5378112792969,
      "logps/ref_rejected": -244.6329345703125,
      "logps/rejected": -239.5035400390625,
      "loss": 5.3531,
      "margin_dpo/margin_mean": 5.145061492919922,
      "margin_dpo/margin_std": 9.944149017333984,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": 9.20852279663086,
      "KL/mean": 7.010412693023682,
      "KL/rejected_KL_mean": 4.812313079833984,
      "KL/std": 7.8776044845581055,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.3962178230285645,
      "fcm_dpo/q_t": 0.4890461564064026,
      "grad_norm": 15.500208854675293,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.7058637142181396,
      "logits/rejected": 1.628740906715393,
      "logps/chosen": -280.60382080078125,
      "logps/ref_chosen": -289.8123474121094,
      "logps/ref_rejected": -274.29071044921875,
      "logps/rejected": -269.4783935546875,
      "loss": 5.3809,
      "margin_dpo/margin_mean": 4.396217346191406,
      "margin_dpo/margin_std": 9.179187774658203,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": 9.915292739868164,
      "KL/mean": 7.689485549926758,
      "KL/rejected_KL_mean": 5.463672637939453,
      "KL/std": 8.085762977600098,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.451606750488281,
      "fcm_dpo/q_t": 0.4889028072357178,
      "grad_norm": 16.171855926513672,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 1.8361562490463257,
      "logits/rejected": 2.0114328861236572,
      "logps/chosen": -235.79782104492188,
      "logps/ref_chosen": -245.71311950683594,
      "logps/ref_rejected": -272.08935546875,
      "logps/rejected": -266.62567138671875,
      "loss": 5.3798,
      "margin_dpo/margin_mean": 4.451605796813965,
      "margin_dpo/margin_std": 9.735191345214844,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": 10.312219619750977,
      "KL/mean": 7.111192226409912,
      "KL/rejected_KL_mean": 3.910175323486328,
      "KL/std": 7.533254623413086,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.402047157287598,
      "fcm_dpo/q_t": 0.48403850197792053,
      "grad_norm": 15.211944580078125,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": 1.2617944478988647,
      "logits/rejected": 1.342819094657898,
      "logps/chosen": -199.63018798828125,
      "logps/ref_chosen": -209.94239807128906,
      "logps/ref_rejected": -230.66897583007812,
      "logps/rejected": -226.7587890625,
      "loss": 5.3023,
      "margin_dpo/margin_mean": 6.402046203613281,
      "margin_dpo/margin_std": 9.010238647460938,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": 9.309700012207031,
      "KL/mean": 6.913294792175293,
      "KL/rejected_KL_mean": 4.516889572143555,
      "KL/std": 8.310725212097168,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.792799472808838,
      "fcm_dpo/q_t": 0.48806390166282654,
      "grad_norm": 15.824963569641113,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": 1.6791523694992065,
      "logits/rejected": 1.7743861675262451,
      "logps/chosen": -241.9899444580078,
      "logps/ref_chosen": -251.2996368408203,
      "logps/ref_rejected": -277.4847412109375,
      "logps/rejected": -272.96783447265625,
      "loss": 5.3672,
      "margin_dpo/margin_mean": 4.79279899597168,
      "margin_dpo/margin_std": 10.305977821350098,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": 8.255512237548828,
      "KL/mean": 6.077991485595703,
      "KL/rejected_KL_mean": 3.900470733642578,
      "KL/std": 8.576713562011719,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.355037689208984,
      "fcm_dpo/q_t": 0.48915961384773254,
      "grad_norm": 14.31108283996582,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": 1.7941694259643555,
      "logits/rejected": 1.837496042251587,
      "logps/chosen": -249.52391052246094,
      "logps/ref_chosen": -257.7794189453125,
      "logps/ref_rejected": -227.91546630859375,
      "logps/rejected": -224.01498413085938,
      "loss": 5.386,
      "margin_dpo/margin_mean": 4.355037689208984,
      "margin_dpo/margin_std": 11.071287155151367,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": 10.358102798461914,
      "KL/mean": 6.291618347167969,
      "KL/rejected_KL_mean": 2.225147247314453,
      "KL/std": 9.65793514251709,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.132970809936523,
      "fcm_dpo/q_t": 0.479738712310791,
      "grad_norm": 17.929983139038086,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": 1.7260948419570923,
      "logits/rejected": 1.671138882637024,
      "logps/chosen": -301.7359313964844,
      "logps/ref_chosen": -312.09405517578125,
      "logps/ref_rejected": -277.97332763671875,
      "logps/rejected": -275.7481689453125,
      "loss": 5.2389,
      "margin_dpo/margin_mean": 8.132969856262207,
      "margin_dpo/margin_std": 10.61925220489502,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": 6.6581573486328125,
      "KL/mean": 3.9260501861572266,
      "KL/rejected_KL_mean": 1.1939315795898438,
      "KL/std": 8.93375015258789,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.464231491088867,
      "fcm_dpo/q_t": 0.4863970875740051,
      "grad_norm": 16.593677520751953,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": 1.8459594249725342,
      "logits/rejected": 1.8656595945358276,
      "logps/chosen": -278.70819091796875,
      "logps/ref_chosen": -285.3663330078125,
      "logps/ref_rejected": -262.2779541015625,
      "logps/rejected": -261.0840148925781,
      "loss": 5.3433,
      "margin_dpo/margin_mean": 5.464231014251709,
      "margin_dpo/margin_std": 11.167790412902832,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": 9.227272033691406,
      "KL/mean": 5.580292224884033,
      "KL/rejected_KL_mean": 1.933319091796875,
      "KL/std": 9.534920692443848,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.293954849243164,
      "fcm_dpo/q_t": 0.4818180501461029,
      "grad_norm": 16.601537704467773,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": 1.8770956993103027,
      "logits/rejected": 1.9497736692428589,
      "logps/chosen": -287.0663146972656,
      "logps/ref_chosen": -296.2935791015625,
      "logps/ref_rejected": -291.5162353515625,
      "logps/rejected": -289.5829162597656,
      "loss": 5.2739,
      "margin_dpo/margin_mean": 7.293955326080322,
      "margin_dpo/margin_std": 11.617576599121094,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": 9.50808334350586,
      "KL/mean": 5.426386833190918,
      "KL/rejected_KL_mean": 1.344686508178711,
      "KL/std": 8.935835838317871,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.163402557373047,
      "fcm_dpo/q_t": 0.47966885566711426,
      "grad_norm": 16.07965850830078,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": 1.8798167705535889,
      "logits/rejected": 1.743872880935669,
      "logps/chosen": -278.61627197265625,
      "logps/ref_chosen": -288.12432861328125,
      "logps/ref_rejected": -232.35720825195312,
      "logps/rejected": -231.01254272460938,
      "loss": 5.2373,
      "margin_dpo/margin_mean": 8.16340160369873,
      "margin_dpo/margin_std": 10.561053276062012,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": 8.060312271118164,
      "KL/mean": 3.82403826713562,
      "KL/rejected_KL_mean": -0.41223716735839844,
      "KL/std": 10.222774505615234,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.47254753112793,
      "fcm_dpo/q_t": 0.47896280884742737,
      "grad_norm": 17.64111328125,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": 1.7059346437454224,
      "logits/rejected": 1.6543306112289429,
      "logps/chosen": -309.1412353515625,
      "logps/ref_chosen": -317.2015686035156,
      "logps/ref_rejected": -278.0653381347656,
      "logps/rejected": -278.4775695800781,
      "loss": 5.2323,
      "margin_dpo/margin_mean": 8.47254467010498,
      "margin_dpo/margin_std": 13.026542663574219,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": 8.631385803222656,
      "KL/mean": 4.653246879577637,
      "KL/rejected_KL_mean": 0.6751174926757812,
      "KL/std": 10.123923301696777,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.956274032592773,
      "fcm_dpo/q_t": 0.4802025556564331,
      "grad_norm": 15.937657356262207,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.5779426097869873,
      "logits/rejected": 1.5542681217193604,
      "logps/chosen": -254.3080596923828,
      "logps/ref_chosen": -262.939453125,
      "logps/ref_rejected": -257.82275390625,
      "logps/rejected": -257.1476135253906,
      "loss": 5.2514,
      "margin_dpo/margin_mean": 7.956274032592773,
      "margin_dpo/margin_std": 13.098845481872559,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": 9.17288589477539,
      "KL/mean": 4.813394546508789,
      "KL/rejected_KL_mean": 0.45391082763671875,
      "KL/std": 11.314221382141113,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.718976020812988,
      "fcm_dpo/q_t": 0.47833994030952454,
      "grad_norm": 18.31784439086914,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 1.9188674688339233,
      "logits/rejected": 1.8243145942687988,
      "logps/chosen": -309.9809265136719,
      "logps/ref_chosen": -319.15380859375,
      "logps/ref_rejected": -281.7733154296875,
      "logps/rejected": -281.31939697265625,
      "loss": 5.2227,
      "margin_dpo/margin_mean": 8.718975067138672,
      "margin_dpo/margin_std": 12.87901496887207,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": 5.493339538574219,
      "KL/mean": 3.3263332843780518,
      "KL/rejected_KL_mean": 1.1593284606933594,
      "KL/std": 10.5056791305542,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.334020137786865,
      "fcm_dpo/q_t": 0.48921144008636475,
      "grad_norm": 14.63753890991211,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": 1.674668788909912,
      "logits/rejected": 1.7735837697982788,
      "logps/chosen": -245.007080078125,
      "logps/ref_chosen": -250.50042724609375,
      "logps/ref_rejected": -268.32635498046875,
      "logps/rejected": -267.16705322265625,
      "loss": 5.397,
      "margin_dpo/margin_mean": 4.334019660949707,
      "margin_dpo/margin_std": 14.161848068237305,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": 6.8038177490234375,
      "KL/mean": 1.7576650381088257,
      "KL/rejected_KL_mean": -3.288480758666992,
      "KL/std": 11.677337646484375,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.092288970947266,
      "fcm_dpo/q_t": 0.47495004534721375,
      "grad_norm": 16.81232452392578,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": 1.563971757888794,
      "logits/rejected": 1.5967552661895752,
      "logps/chosen": -272.5977783203125,
      "logps/ref_chosen": -279.4015808105469,
      "logps/ref_rejected": -266.6392822265625,
      "logps/rejected": -269.92779541015625,
      "loss": 5.1741,
      "margin_dpo/margin_mean": 10.092288970947266,
      "margin_dpo/margin_std": 14.342607498168945,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": 5.405511856079102,
      "KL/mean": 2.837066650390625,
      "KL/rejected_KL_mean": 0.2686290740966797,
      "KL/std": 10.736865043640137,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.136881351470947,
      "fcm_dpo/q_t": 0.4872317314147949,
      "grad_norm": 15.741600036621094,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 1.6113207340240479,
      "logits/rejected": 1.5922027826309204,
      "logps/chosen": -248.85498046875,
      "logps/ref_chosen": -254.26048278808594,
      "logps/ref_rejected": -241.8769989013672,
      "logps/rejected": -241.60836791992188,
      "loss": 5.3613,
      "margin_dpo/margin_mean": 5.136881351470947,
      "margin_dpo/margin_std": 13.077017784118652,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": 5.655580520629883,
      "KL/mean": 2.7717509269714355,
      "KL/rejected_KL_mean": -0.11208534240722656,
      "KL/std": 11.94007396697998,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.767664432525635,
      "fcm_dpo/q_t": 0.48570767045021057,
      "grad_norm": 16.217500686645508,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": 1.7174509763717651,
      "logits/rejected": 1.7513399124145508,
      "logps/chosen": -283.14892578125,
      "logps/ref_chosen": -288.80450439453125,
      "logps/ref_rejected": -266.8573913574219,
      "logps/rejected": -266.969482421875,
      "loss": 5.3411,
      "margin_dpo/margin_mean": 5.767664909362793,
      "margin_dpo/margin_std": 14.457511901855469,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": 7.043796539306641,
      "KL/mean": 3.014029026031494,
      "KL/rejected_KL_mean": -1.0157394409179688,
      "KL/std": 11.147762298583984,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.059527397155762,
      "fcm_dpo/q_t": 0.47997426986694336,
      "grad_norm": 16.38408851623535,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": 1.654149055480957,
      "logits/rejected": 1.6289184093475342,
      "logps/chosen": -265.14617919921875,
      "logps/ref_chosen": -272.18994140625,
      "logps/ref_rejected": -253.14334106445312,
      "logps/rejected": -254.15908813476562,
      "loss": 5.2506,
      "margin_dpo/margin_mean": 8.059527397155762,
      "margin_dpo/margin_std": 14.118448257446289,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": 5.619565963745117,
      "KL/mean": 1.4217474460601807,
      "KL/rejected_KL_mean": -2.7760753631591797,
      "KL/std": 11.071073532104492,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.39564323425293,
      "fcm_dpo/q_t": 0.4791334271430969,
      "grad_norm": 16.76996421813965,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": 1.6662051677703857,
      "logits/rejected": 1.6618272066116333,
      "logps/chosen": -244.7830810546875,
      "logps/ref_chosen": -250.4026336669922,
      "logps/ref_rejected": -229.63853454589844,
      "logps/rejected": -232.4146270751953,
      "loss": 5.2354,
      "margin_dpo/margin_mean": 8.39564323425293,
      "margin_dpo/margin_std": 12.537582397460938,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": 5.568138122558594,
      "KL/mean": 1.048189640045166,
      "KL/rejected_KL_mean": -3.4717636108398438,
      "KL/std": 12.760374069213867,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.039901733398438,
      "fcm_dpo/q_t": 0.4775538146495819,
      "grad_norm": 20.32367706298828,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": 1.7151572704315186,
      "logits/rejected": 1.7640178203582764,
      "logps/chosen": -252.8277587890625,
      "logps/ref_chosen": -258.3959045410156,
      "logps/ref_rejected": -268.84228515625,
      "logps/rejected": -272.3140563964844,
      "loss": 5.2147,
      "margin_dpo/margin_mean": 9.039901733398438,
      "margin_dpo/margin_std": 14.558975219726562,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": 6.093051910400391,
      "KL/mean": 2.3288393020629883,
      "KL/rejected_KL_mean": -1.4353694915771484,
      "KL/std": 14.14476203918457,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.528421401977539,
      "fcm_dpo/q_t": 0.4813995063304901,
      "grad_norm": 15.904483795166016,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": 2.0412864685058594,
      "logits/rejected": 2.026611566543579,
      "logps/chosen": -311.2095031738281,
      "logps/ref_chosen": -317.30255126953125,
      "logps/ref_rejected": -300.47552490234375,
      "logps/rejected": -301.9108581542969,
      "loss": 5.2826,
      "margin_dpo/margin_mean": 7.528423309326172,
      "margin_dpo/margin_std": 16.701452255249023,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": 7.061199188232422,
      "KL/mean": 2.542219877243042,
      "KL/rejected_KL_mean": -1.9767589569091797,
      "KL/std": 13.43856143951416,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.0379638671875,
      "fcm_dpo/q_t": 0.47759774327278137,
      "grad_norm": 16.14647102355957,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.7863283157348633,
      "logits/rejected": 1.737242341041565,
      "logps/chosen": -298.1932373046875,
      "logps/ref_chosen": -305.2544250488281,
      "logps/ref_rejected": -259.40716552734375,
      "logps/rejected": -261.3839111328125,
      "loss": 5.2213,
      "margin_dpo/margin_mean": 9.037964820861816,
      "margin_dpo/margin_std": 16.464614868164062,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": 3.467111587524414,
      "KL/mean": 0.779504120349884,
      "KL/rejected_KL_mean": -1.9080982208251953,
      "KL/std": 11.781448364257812,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.375213623046875,
      "fcm_dpo/q_t": 0.4866599440574646,
      "grad_norm": 14.629935264587402,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 1.71998131275177,
      "logits/rejected": 1.7435311079025269,
      "logps/chosen": -213.17361450195312,
      "logps/ref_chosen": -216.64073181152344,
      "logps/ref_rejected": -224.36422729492188,
      "logps/rejected": -226.2723388671875,
      "loss": 5.3561,
      "margin_dpo/margin_mean": 5.375214099884033,
      "margin_dpo/margin_std": 14.35720157623291,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": 5.773719787597656,
      "KL/mean": 2.054203510284424,
      "KL/rejected_KL_mean": -1.6653079986572266,
      "KL/std": 13.969862937927246,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.439023971557617,
      "fcm_dpo/q_t": 0.48157572746276855,
      "grad_norm": 15.743058204650879,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": 1.6342281103134155,
      "logits/rejected": 1.6392768621444702,
      "logps/chosen": -263.0450744628906,
      "logps/ref_chosen": -268.8188171386719,
      "logps/ref_rejected": -272.55853271484375,
      "logps/rejected": -274.2238464355469,
      "loss": 5.2863,
      "margin_dpo/margin_mean": 7.439024448394775,
      "margin_dpo/margin_std": 17.4293270111084,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": 8.53335189819336,
      "KL/mean": 3.257288932800293,
      "KL/rejected_KL_mean": -2.0187644958496094,
      "KL/std": 14.992415428161621,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.552115440368652,
      "fcm_dpo/q_t": 0.4738881587982178,
      "grad_norm": 18.32449722290039,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": 1.7882142066955566,
      "logits/rejected": 1.77853524684906,
      "logps/chosen": -313.1584167480469,
      "logps/ref_chosen": -321.6917419433594,
      "logps/ref_rejected": -279.1517333984375,
      "logps/rejected": -281.17047119140625,
      "loss": 5.1688,
      "margin_dpo/margin_mean": 10.552115440368652,
      "margin_dpo/margin_std": 18.21430206298828,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": 6.041261672973633,
      "KL/mean": 1.2617852687835693,
      "KL/rejected_KL_mean": -3.517695426940918,
      "KL/std": 14.469627380371094,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.5589599609375,
      "fcm_dpo/q_t": 0.47629186511039734,
      "grad_norm": 16.784204483032227,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": 1.6344691514968872,
      "logits/rejected": 1.5178453922271729,
      "logps/chosen": -265.6339416503906,
      "logps/ref_chosen": -271.6752014160156,
      "logps/ref_rejected": -221.85577392578125,
      "logps/rejected": -225.37347412109375,
      "loss": 5.2018,
      "margin_dpo/margin_mean": 9.55896282196045,
      "margin_dpo/margin_std": 15.9717435836792,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": 5.53581428527832,
      "KL/mean": 0.8782179355621338,
      "KL/rejected_KL_mean": -3.7793827056884766,
      "KL/std": 14.102092742919922,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.315194129943848,
      "fcm_dpo/q_t": 0.47698307037353516,
      "grad_norm": 16.259740829467773,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": 1.6127885580062866,
      "logits/rejected": 1.617489218711853,
      "logps/chosen": -264.86322021484375,
      "logps/ref_chosen": -270.3990478515625,
      "logps/ref_rejected": -264.16925048828125,
      "logps/rejected": -267.9486389160156,
      "loss": 5.2171,
      "margin_dpo/margin_mean": 9.315194129943848,
      "margin_dpo/margin_std": 18.472980499267578,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": 6.752374649047852,
      "KL/mean": 1.4570807218551636,
      "KL/rejected_KL_mean": -3.838207244873047,
      "KL/std": 15.533653259277344,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.010113801807165146,
      "fcm_dpo/delta": 0.016152551397681236,
      "fcm_dpo/margin": 10.59058666229248,
      "fcm_dpo/q_t": 0.4735642671585083,
      "grad_norm": 16.89833641052246,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": 1.7655162811279297,
      "logits/rejected": 1.7442115545272827,
      "logps/chosen": -306.4989318847656,
      "logps/ref_chosen": -313.25128173828125,
      "logps/ref_rejected": -283.0433349609375,
      "logps/rejected": -286.88153076171875,
      "loss": 5.1656,
      "margin_dpo/margin_mean": 10.59058666229248,
      "margin_dpo/margin_std": 18.256122589111328,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": 8.108327865600586,
      "KL/mean": 1.9179469347000122,
      "KL/rejected_KL_mean": -4.272436141967773,
      "KL/std": 16.06947898864746,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.01021901611238718,
      "fcm_dpo/delta": 0.03447946906089783,
      "fcm_dpo/margin": 12.380764961242676,
      "fcm_dpo/q_t": 0.4687623381614685,
      "grad_norm": 17.13775062561035,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 1.6967636346817017,
      "logits/rejected": 1.6207829713821411,
      "logps/chosen": -290.4556579589844,
      "logps/ref_chosen": -298.5639953613281,
      "logps/ref_rejected": -274.021728515625,
      "logps/rejected": -278.294189453125,
      "loss": 5.0914,
      "margin_dpo/margin_mean": 12.38076400756836,
      "margin_dpo/margin_std": 18.18212127685547,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": 8.716808319091797,
      "KL/mean": 1.9333598613739014,
      "KL/rejected_KL_mean": -4.8500823974609375,
      "KL/std": 16.91305923461914,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.010525355115532875,
      "fcm_dpo/delta": 0.017795873805880547,
      "fcm_dpo/margin": 13.566892623901367,
      "fcm_dpo/q_t": 0.4648301303386688,
      "grad_norm": 16.85113525390625,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": 1.82450532913208,
      "logits/rejected": 1.8800344467163086,
      "logps/chosen": -248.4140625,
      "logps/ref_chosen": -257.130859375,
      "logps/ref_rejected": -247.71636962890625,
      "logps/rejected": -252.56643676757812,
      "loss": 5.0413,
      "margin_dpo/margin_mean": 13.566890716552734,
      "margin_dpo/margin_std": 20.04707145690918,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": 9.113378524780273,
      "KL/mean": 2.767698049545288,
      "KL/rejected_KL_mean": -3.5779762268066406,
      "KL/std": 17.078147888183594,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.010695034638047218,
      "fcm_dpo/delta": 0.017773177474737167,
      "fcm_dpo/margin": 12.69135570526123,
      "fcm_dpo/q_t": 0.4666440486907959,
      "grad_norm": 17.96302604675293,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": 1.8288559913635254,
      "logits/rejected": 1.8840014934539795,
      "logps/chosen": -285.8471984863281,
      "logps/ref_chosen": -294.9605712890625,
      "logps/ref_rejected": -272.6249694824219,
      "logps/rejected": -276.2029113769531,
      "loss": 5.0725,
      "margin_dpo/margin_mean": 12.691354751586914,
      "margin_dpo/margin_std": 20.305936813354492,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": 7.700517654418945,
      "KL/mean": 1.8539153337478638,
      "KL/rejected_KL_mean": -3.992704391479492,
      "KL/std": 15.489526748657227,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.010713942348957062,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.693220138549805,
      "fcm_dpo/q_t": 0.4690232276916504,
      "grad_norm": 22.417810440063477,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 1.768460988998413,
      "logits/rejected": 1.8505364656448364,
      "logps/chosen": -268.74169921875,
      "logps/ref_chosen": -276.4422302246094,
      "logps/ref_rejected": -290.241455078125,
      "logps/rejected": -294.234130859375,
      "loss": 5.1037,
      "margin_dpo/margin_mean": 11.693222045898438,
      "margin_dpo/margin_std": 19.240259170532227,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": 4.924707412719727,
      "KL/mean": 0.12944117188453674,
      "KL/rejected_KL_mean": -4.665821075439453,
      "KL/std": 16.500263214111328,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.010808400809764862,
      "fcm_dpo/delta": 0.017509354278445244,
      "fcm_dpo/margin": 9.59052848815918,
      "fcm_dpo/q_t": 0.47460803389549255,
      "grad_norm": 18.86319923400879,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 1.5692379474639893,
      "logits/rejected": 1.568664789199829,
      "logps/chosen": -299.9000244140625,
      "logps/ref_chosen": -304.82470703125,
      "logps/ref_rejected": -289.9726867675781,
      "logps/rejected": -294.63848876953125,
      "loss": 5.1974,
      "margin_dpo/margin_mean": 9.59052848815918,
      "margin_dpo/margin_std": 21.24658966064453,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": 7.181467056274414,
      "KL/mean": 0.7200897932052612,
      "KL/rejected_KL_mean": -5.74128532409668,
      "KL/std": 18.5306339263916,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.010921559296548367,
      "fcm_dpo/delta": 0.023388568311929703,
      "fcm_dpo/margin": 12.922744750976562,
      "fcm_dpo/q_t": 0.4656592011451721,
      "grad_norm": 19.313886642456055,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": 1.6452537775039673,
      "logits/rejected": 1.756277322769165,
      "logps/chosen": -253.1332550048828,
      "logps/ref_chosen": -260.31475830078125,
      "logps/ref_rejected": -280.93756103515625,
      "logps/rejected": -286.6788330078125,
      "loss": 5.0719,
      "margin_dpo/margin_mean": 12.922745704650879,
      "margin_dpo/margin_std": 23.427160263061523,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": 7.666341781616211,
      "KL/mean": 0.9123910069465637,
      "KL/rejected_KL_mean": -5.841558456420898,
      "KL/std": 17.18130111694336,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.011093483306467533,
      "fcm_dpo/delta": 0.01043512299656868,
      "fcm_dpo/margin": 13.50790786743164,
      "fcm_dpo/q_t": 0.463068425655365,
      "grad_norm": 17.83781623840332,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": 1.6816298961639404,
      "logits/rejected": 1.6599150896072388,
      "logps/chosen": -271.4542541503906,
      "logps/ref_chosen": -279.12060546875,
      "logps/ref_rejected": -247.22982788085938,
      "logps/rejected": -253.07139587402344,
      "loss": 5.0282,
      "margin_dpo/margin_mean": 13.507908821105957,
      "margin_dpo/margin_std": 20.930997848510742,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": 1.5965232849121094,
      "KL/mean": -3.083186388015747,
      "KL/rejected_KL_mean": -7.76289176940918,
      "KL/std": 17.569765090942383,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.011244647204875946,
      "fcm_dpo/delta": 0.018133504316210747,
      "fcm_dpo/margin": 9.359416961669922,
      "fcm_dpo/q_t": 0.47423726320266724,
      "grad_norm": 22.97232437133789,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": 1.6623162031173706,
      "logits/rejected": 1.6896274089813232,
      "logps/chosen": -314.007568359375,
      "logps/ref_chosen": -315.6040954589844,
      "logps/ref_rejected": -291.45635986328125,
      "logps/rejected": -299.21923828125,
      "loss": 5.1908,
      "margin_dpo/margin_mean": 9.359418869018555,
      "margin_dpo/margin_std": 20.196651458740234,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": 5.185831069946289,
      "KL/mean": -1.4466923475265503,
      "KL/rejected_KL_mean": -8.079217910766602,
      "KL/std": 18.054054260253906,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.011457690969109535,
      "fcm_dpo/delta": 0.02747393399477005,
      "fcm_dpo/margin": 13.265047073364258,
      "fcm_dpo/q_t": 0.4627569317817688,
      "grad_norm": 19.105628967285156,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": 1.2857714891433716,
      "logits/rejected": 1.2516728639602661,
      "logps/chosen": -277.6871337890625,
      "logps/ref_chosen": -282.87298583984375,
      "logps/ref_rejected": -265.0682373046875,
      "logps/rejected": -273.1474304199219,
      "loss": 5.0165,
      "margin_dpo/margin_mean": 13.265047073364258,
      "margin_dpo/margin_std": 19.352157592773438,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": 4.274782180786133,
      "KL/mean": -1.9806631803512573,
      "KL/rejected_KL_mean": -8.236104965209961,
      "KL/std": 17.65324592590332,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.011614818125963211,
      "fcm_dpo/delta": 0.0105828158557415,
      "fcm_dpo/margin": 12.510882377624512,
      "fcm_dpo/q_t": 0.4644309878349304,
      "grad_norm": 21.815135955810547,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": 1.3217121362686157,
      "logits/rejected": 1.4803383350372314,
      "logps/chosen": -259.1671447753906,
      "logps/ref_chosen": -263.4419250488281,
      "logps/ref_rejected": -251.43150329589844,
      "logps/rejected": -259.6676025390625,
      "loss": 5.0561,
      "margin_dpo/margin_mean": 12.510882377624512,
      "margin_dpo/margin_std": 21.966407775878906,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": 2.2735843658447266,
      "KL/mean": -3.2860350608825684,
      "KL/rejected_KL_mean": -8.84565544128418,
      "KL/std": 19.090728759765625,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.011755144223570824,
      "fcm_dpo/delta": 0.013425348326563835,
      "fcm_dpo/margin": 11.119251251220703,
      "fcm_dpo/q_t": 0.4679127633571625,
      "grad_norm": 21.923294067382812,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": 1.6070241928100586,
      "logits/rejected": 1.600968599319458,
      "logps/chosen": -292.9971618652344,
      "logps/ref_chosen": -295.2707214355469,
      "logps/ref_rejected": -283.9851989746094,
      "logps/rejected": -292.8309020996094,
      "loss": 5.1037,
      "margin_dpo/margin_mean": 11.119251251220703,
      "margin_dpo/margin_std": 20.58676528930664,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": 1.0548057556152344,
      "KL/mean": -4.511653423309326,
      "KL/rejected_KL_mean": -10.078113555908203,
      "KL/std": 18.390064239501953,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.011882021091878414,
      "fcm_dpo/delta": 0.008401725441217422,
      "fcm_dpo/margin": 11.132911682128906,
      "fcm_dpo/q_t": 0.467870831489563,
      "grad_norm": 19.541961669921875,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": 1.5609619617462158,
      "logits/rejected": 1.5564669370651245,
      "logps/chosen": -281.5030517578125,
      "logps/ref_chosen": -282.5578918457031,
      "logps/ref_rejected": -262.4772644042969,
      "logps/rejected": -272.55535888671875,
      "loss": 5.1173,
      "margin_dpo/margin_mean": 11.132911682128906,
      "margin_dpo/margin_std": 23.500316619873047,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": 2.172739028930664,
      "KL/mean": -1.9332672357559204,
      "KL/rejected_KL_mean": -6.039266586303711,
      "KL/std": 19.162372589111328,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.011882021091878414,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.212000846862793,
      "fcm_dpo/q_t": 0.4761759638786316,
      "grad_norm": 19.020008087158203,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": 1.545659065246582,
      "logits/rejected": 1.6172683238983154,
      "logps/chosen": -261.4780578613281,
      "logps/ref_chosen": -263.65081787109375,
      "logps/ref_rejected": -266.2420959472656,
      "logps/rejected": -272.2813720703125,
      "loss": 5.246,
      "margin_dpo/margin_mean": 8.212000846862793,
      "margin_dpo/margin_std": 23.70122528076172,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": 5.729040145874023,
      "KL/mean": -1.7547959089279175,
      "KL/rejected_KL_mean": -9.238622665405273,
      "KL/std": 19.02861976623535,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.012009905651211739,
      "fcm_dpo/delta": 0.035113364458084106,
      "fcm_dpo/margin": 14.967662811279297,
      "fcm_dpo/q_t": 0.4561896324157715,
      "grad_norm": 19.189489364624023,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 1.6304552555084229,
      "logits/rejected": 1.5864876508712769,
      "logps/chosen": -266.6534118652344,
      "logps/ref_chosen": -272.3824462890625,
      "logps/ref_rejected": -248.96444702148438,
      "logps/rejected": -258.2030944824219,
      "loss": 4.9285,
      "margin_dpo/margin_mean": 14.967662811279297,
      "margin_dpo/margin_std": 21.21372413635254,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": 3.447650909423828,
      "KL/mean": -3.7205750942230225,
      "KL/rejected_KL_mean": -10.888784408569336,
      "KL/std": 20.16835594177246,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.012517844326794147,
      "fcm_dpo/delta": 0.03850012272596359,
      "fcm_dpo/margin": 14.336435317993164,
      "fcm_dpo/q_t": 0.4564206898212433,
      "grad_norm": 28.656044006347656,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 1.6075314283370972,
      "logits/rejected": 1.6587722301483154,
      "logps/chosen": -284.6233825683594,
      "logps/ref_chosen": -288.071044921875,
      "logps/ref_rejected": -274.83477783203125,
      "logps/rejected": -285.7235107421875,
      "loss": 4.9571,
      "margin_dpo/margin_mean": 14.336435317993164,
      "margin_dpo/margin_std": 24.259016036987305,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": 6.4539337158203125,
      "KL/mean": -0.5791962146759033,
      "KL/rejected_KL_mean": -7.612331390380859,
      "KL/std": 19.99506187438965,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.012752669863402843,
      "fcm_dpo/delta": 0.029137257486581802,
      "fcm_dpo/margin": 14.066269874572754,
      "fcm_dpo/q_t": 0.456595778465271,
      "grad_norm": 41.597877502441406,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": 1.6526694297790527,
      "logits/rejected": 1.553966999053955,
      "logps/chosen": -282.53790283203125,
      "logps/ref_chosen": -288.9918518066406,
      "logps/ref_rejected": -228.26071166992188,
      "logps/rejected": -235.87303161621094,
      "loss": 4.9609,
      "margin_dpo/margin_mean": 14.066269874572754,
      "margin_dpo/margin_std": 24.100385665893555,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": 6.067161560058594,
      "KL/mean": -1.965949535369873,
      "KL/rejected_KL_mean": -9.999053955078125,
      "KL/std": 21.08316993713379,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.01323370449244976,
      "fcm_dpo/delta": 0.05701703205704689,
      "fcm_dpo/margin": 16.066207885742188,
      "fcm_dpo/q_t": 0.44896399974823,
      "grad_norm": 23.682424545288086,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": 1.4129558801651,
      "logits/rejected": 1.4318358898162842,
      "logps/chosen": -283.4073181152344,
      "logps/ref_chosen": -289.4744873046875,
      "logps/ref_rejected": -265.130615234375,
      "logps/rejected": -275.12969970703125,
      "loss": 4.868,
      "margin_dpo/margin_mean": 16.066205978393555,
      "margin_dpo/margin_std": 26.51406478881836,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": 7.387163162231445,
      "KL/mean": 0.720440685749054,
      "KL/rejected_KL_mean": -5.946269989013672,
      "KL/std": 20.3001708984375,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.01383878942579031,
      "fcm_dpo/delta": 0.04417652636766434,
      "fcm_dpo/margin": 13.333430290222168,
      "fcm_dpo/q_t": 0.45594191551208496,
      "grad_norm": 26.49911117553711,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": 1.6475634574890137,
      "logits/rejected": 1.6261649131774902,
      "logps/chosen": -283.1991882324219,
      "logps/ref_chosen": -290.5863342285156,
      "logps/ref_rejected": -263.41046142578125,
      "logps/rejected": -269.3567199707031,
      "loss": 4.9609,
      "margin_dpo/margin_mean": 13.333430290222168,
      "margin_dpo/margin_std": 23.95872688293457,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": 7.559547424316406,
      "KL/mean": -1.1047616004943848,
      "KL/rejected_KL_mean": -9.769071578979492,
      "KL/std": 19.370868682861328,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.014158430509269238,
      "fcm_dpo/delta": 0.01770552061498165,
      "fcm_dpo/margin": 17.32861328125,
      "fcm_dpo/q_t": 0.44128453731536865,
      "grad_norm": 23.04567527770996,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": 1.6042205095291138,
      "logits/rejected": 1.5154917240142822,
      "logps/chosen": -256.980712890625,
      "logps/ref_chosen": -264.5402526855469,
      "logps/ref_rejected": -214.7958221435547,
      "logps/rejected": -224.56488037109375,
      "loss": 4.7571,
      "margin_dpo/margin_mean": 17.32861328125,
      "margin_dpo/margin_std": 24.243574142456055,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": 3.1682052612304688,
      "KL/mean": -3.7291977405548096,
      "KL/rejected_KL_mean": -10.626607894897461,
      "KL/std": 19.541095733642578,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.014321361668407917,
      "fcm_dpo/delta": 0.03937491774559021,
      "fcm_dpo/margin": 13.794812202453613,
      "fcm_dpo/q_t": 0.452516108751297,
      "grad_norm": 38.470664978027344,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": 1.6716407537460327,
      "logits/rejected": 1.8063669204711914,
      "logps/chosen": -267.01153564453125,
      "logps/ref_chosen": -270.17974853515625,
      "logps/ref_rejected": -261.2323913574219,
      "logps/rejected": -271.8589782714844,
      "loss": 4.9206,
      "margin_dpo/margin_mean": 13.794812202453613,
      "margin_dpo/margin_std": 23.947978973388672,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": 1.9022941589355469,
      "KL/mean": -3.944277763366699,
      "KL/rejected_KL_mean": -9.790847778320312,
      "KL/std": 21.820999145507812,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.014738228172063828,
      "fcm_dpo/delta": 0.019158843904733658,
      "fcm_dpo/margin": 11.693140983581543,
      "fcm_dpo/q_t": 0.4587726294994354,
      "grad_norm": 24.157381057739258,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": 1.4907667636871338,
      "logits/rejected": 1.4633166790008545,
      "logps/chosen": -280.8782958984375,
      "logps/ref_chosen": -282.7806091308594,
      "logps/ref_rejected": -249.23399353027344,
      "logps/rejected": -259.02484130859375,
      "loss": 5.0526,
      "margin_dpo/margin_mean": 11.693140983581543,
      "margin_dpo/margin_std": 26.260303497314453,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": 2.2874679565429688,
      "KL/mean": -3.8896989822387695,
      "KL/rejected_KL_mean": -10.066858291625977,
      "KL/std": 20.542917251586914,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.015006192028522491,
      "fcm_dpo/delta": 0.027173461392521858,
      "fcm_dpo/margin": 12.354327201843262,
      "fcm_dpo/q_t": 0.45590898394584656,
      "grad_norm": 23.897001266479492,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": 1.569932460784912,
      "logits/rejected": 1.5858418941497803,
      "logps/chosen": -226.62985229492188,
      "logps/ref_chosen": -228.9173126220703,
      "logps/ref_rejected": -226.81222534179688,
      "logps/rejected": -236.8791046142578,
      "loss": 4.9943,
      "margin_dpo/margin_mean": 12.354328155517578,
      "margin_dpo/margin_std": 25.71002197265625,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": 3.6988162994384766,
      "KL/mean": -4.007805347442627,
      "KL/rejected_KL_mean": -11.714437484741211,
      "KL/std": 21.272350311279297,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.015581812709569931,
      "fcm_dpo/delta": 0.04428521916270256,
      "fcm_dpo/margin": 15.413250923156738,
      "fcm_dpo/q_t": 0.4430851638317108,
      "grad_norm": 27.52985191345215,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": 1.796948790550232,
      "logits/rejected": 1.780381441116333,
      "logps/chosen": -307.3871154785156,
      "logps/ref_chosen": -311.0859375,
      "logps/ref_rejected": -303.225341796875,
      "logps/rejected": -314.9397888183594,
      "loss": 4.8032,
      "margin_dpo/margin_mean": 15.413253784179688,
      "margin_dpo/margin_std": 25.31744384765625,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": 1.0658245086669922,
      "KL/mean": -6.1691436767578125,
      "KL/rejected_KL_mean": -13.404109954833984,
      "KL/std": 20.473461151123047,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.0160695631057024,
      "fcm_dpo/delta": 0.05122996121644974,
      "fcm_dpo/margin": 14.469940185546875,
      "fcm_dpo/q_t": 0.44468462467193604,
      "grad_norm": 27.378890991210938,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 1.5738961696624756,
      "logits/rejected": 1.5868630409240723,
      "logps/chosen": -269.4176330566406,
      "logps/ref_chosen": -270.48345947265625,
      "logps/ref_rejected": -269.0491638183594,
      "logps/rejected": -282.4532775878906,
      "loss": 4.8253,
      "margin_dpo/margin_mean": 14.469941139221191,
      "margin_dpo/margin_std": 24.180871963500977,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": 4.359842300415039,
      "KL/mean": -6.759664535522461,
      "KL/rejected_KL_mean": -17.879169464111328,
      "KL/std": 22.557899475097656,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.016263432800769806,
      "fcm_dpo/delta": -0.012315181083977222,
      "fcm_dpo/margin": 22.23900032043457,
      "fcm_dpo/q_t": 0.4145720601081848,
      "grad_norm": 29.0079345703125,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.487717628479004,
      "logits/rejected": 1.5022878646850586,
      "logps/chosen": -283.8058776855469,
      "logps/ref_chosen": -288.16571044921875,
      "logps/ref_rejected": -264.7412414550781,
      "logps/rejected": -282.62042236328125,
      "loss": 4.435,
      "margin_dpo/margin_mean": 22.23900032043457,
      "margin_dpo/margin_std": 27.829801559448242,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": 3.211273193359375,
      "KL/mean": -5.202370643615723,
      "KL/rejected_KL_mean": -13.61601448059082,
      "KL/std": 22.641939163208008,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.016290651634335518,
      "fcm_dpo/delta": 0.01069331169128418,
      "fcm_dpo/margin": 16.827287673950195,
      "fcm_dpo/q_t": 0.43520352244377136,
      "grad_norm": 28.72869873046875,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": 1.3529834747314453,
      "logits/rejected": 1.429661512374878,
      "logps/chosen": -256.72723388671875,
      "logps/ref_chosen": -259.9385070800781,
      "logps/ref_rejected": -266.2453308105469,
      "logps/rejected": -279.86138916015625,
      "loss": 4.7321,
      "margin_dpo/margin_mean": 16.827287673950195,
      "margin_dpo/margin_std": 27.113182067871094,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": 2.228120803833008,
      "KL/mean": -9.826213836669922,
      "KL/rejected_KL_mean": -21.880542755126953,
      "KL/std": 23.97818946838379,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.0159678403288126,
      "fcm_dpo/delta": -0.036996521055698395,
      "fcm_dpo/margin": 24.108661651611328,
      "fcm_dpo/q_t": 0.40847939252853394,
      "grad_norm": 30.931665420532227,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 1.5791618824005127,
      "logits/rejected": 1.6448591947555542,
      "logps/chosen": -263.3489074707031,
      "logps/ref_chosen": -265.5769958496094,
      "logps/ref_rejected": -277.6246032714844,
      "logps/rejected": -299.505126953125,
      "loss": 4.3277,
      "margin_dpo/margin_mean": 24.108661651611328,
      "margin_dpo/margin_std": 26.158790588378906,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": 0.5362224578857422,
      "KL/mean": -8.268867492675781,
      "KL/rejected_KL_mean": -17.073957443237305,
      "KL/std": 24.662582397460938,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.015902649611234665,
      "fcm_dpo/delta": -0.003240179270505905,
      "fcm_dpo/margin": 17.610187530517578,
      "fcm_dpo/q_t": 0.43371790647506714,
      "grad_norm": 29.98178482055664,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": 1.6312828063964844,
      "logits/rejected": 1.6175562143325806,
      "logps/chosen": -281.7371826171875,
      "logps/ref_chosen": -282.2734069824219,
      "logps/ref_rejected": -261.5135498046875,
      "logps/rejected": -278.5875244140625,
      "loss": 4.7495,
      "margin_dpo/margin_mean": 17.610187530517578,
      "margin_dpo/margin_std": 30.49097442626953,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -0.6938209533691406,
      "KL/mean": -7.089329719543457,
      "KL/rejected_KL_mean": -13.484844207763672,
      "KL/std": 22.324426651000977,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.016145823523402214,
      "fcm_dpo/delta": 0.040474582463502884,
      "fcm_dpo/margin": 12.791024208068848,
      "fcm_dpo/q_t": 0.45139747858047485,
      "grad_norm": 29.488374710083008,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": 1.5053040981292725,
      "logits/rejected": 1.598483920097351,
      "logps/chosen": -279.4053955078125,
      "logps/ref_chosen": -278.7115783691406,
      "logps/ref_rejected": -298.0945739746094,
      "logps/rejected": -311.57940673828125,
      "loss": 4.972,
      "margin_dpo/margin_mean": 12.79102611541748,
      "margin_dpo/margin_std": 27.407451629638672,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": 0.7116966247558594,
      "KL/mean": -10.146078109741211,
      "KL/rejected_KL_mean": -21.00385856628418,
      "KL/std": 24.634361267089844,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.016242455691099167,
      "fcm_dpo/delta": -0.02400655299425125,
      "fcm_dpo/margin": 21.715553283691406,
      "fcm_dpo/q_t": 0.4174003005027771,
      "grad_norm": 27.25838279724121,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": 1.5964107513427734,
      "logits/rejected": 1.6219264268875122,
      "logps/chosen": -267.6394348144531,
      "logps/ref_chosen": -268.35113525390625,
      "logps/ref_rejected": -241.9660186767578,
      "logps/rejected": -262.9698791503906,
      "loss": 4.5326,
      "margin_dpo/margin_mean": 21.71555519104004,
      "margin_dpo/margin_std": 32.56122589111328,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -0.2618694305419922,
      "KL/mean": -7.795019626617432,
      "KL/rejected_KL_mean": -15.328165054321289,
      "KL/std": 23.94432830810547,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.01612035743892193,
      "fcm_dpo/delta": 0.012129014357924461,
      "fcm_dpo/margin": 15.066286087036133,
      "fcm_dpo/q_t": 0.44286805391311646,
      "grad_norm": 37.04217529296875,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": 1.4239239692687988,
      "logits/rejected": 1.484088659286499,
      "logps/chosen": -265.66552734375,
      "logps/ref_chosen": -265.40362548828125,
      "logps/ref_rejected": -257.82440185546875,
      "logps/rejected": -273.1525573730469,
      "loss": 4.8497,
      "margin_dpo/margin_mean": 15.066286087036133,
      "margin_dpo/margin_std": 27.817197799682617,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": 0.8738937377929688,
      "KL/mean": -7.52041482925415,
      "KL/rejected_KL_mean": -15.914718627929688,
      "KL/std": 23.788265228271484,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.016244836151599884,
      "fcm_dpo/delta": 0.001793057657778263,
      "fcm_dpo/margin": 16.788618087768555,
      "fcm_dpo/q_t": 0.43545839190483093,
      "grad_norm": 28.794185638427734,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": 1.4181725978851318,
      "logits/rejected": 1.459049940109253,
      "logps/chosen": -280.28839111328125,
      "logps/ref_chosen": -281.1622619628906,
      "logps/ref_rejected": -268.2899475097656,
      "logps/rejected": -284.20465087890625,
      "loss": 4.7504,
      "margin_dpo/margin_mean": 16.788619995117188,
      "margin_dpo/margin_std": 28.30937385559082,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -0.3008537292480469,
      "KL/mean": -6.822476863861084,
      "KL/rejected_KL_mean": -13.344091415405273,
      "KL/std": 24.671764373779297,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.016288867220282555,
      "fcm_dpo/delta": 0.012477658689022064,
      "fcm_dpo/margin": 13.043231010437012,
      "fcm_dpo/q_t": 0.45106634497642517,
      "grad_norm": 32.98561096191406,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": 1.786769986152649,
      "logits/rejected": 1.7041187286376953,
      "logps/chosen": -311.6258544921875,
      "logps/ref_chosen": -311.32501220703125,
      "logps/ref_rejected": -248.85350036621094,
      "logps/rejected": -262.19757080078125,
      "loss": 5.0423,
      "margin_dpo/margin_mean": 13.043231964111328,
      "margin_dpo/margin_std": 32.63959503173828,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": 3.5245132446289062,
      "KL/mean": -6.554137229919434,
      "KL/rejected_KL_mean": -16.63279151916504,
      "KL/std": 24.576709747314453,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.016245365142822266,
      "fcm_dpo/delta": -0.024331897497177124,
      "fcm_dpo/margin": 20.15730857849121,
      "fcm_dpo/q_t": 0.42349135875701904,
      "grad_norm": 28.755945205688477,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": 1.5738838911056519,
      "logits/rejected": 1.5880122184753418,
      "logps/chosen": -263.58447265625,
      "logps/ref_chosen": -267.10894775390625,
      "logps/ref_rejected": -258.66162109375,
      "logps/rejected": -275.29443359375,
      "loss": 4.5715,
      "margin_dpo/margin_mean": 20.157310485839844,
      "margin_dpo/margin_std": 28.514074325561523,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": 2.8593711853027344,
      "KL/mean": -7.777968883514404,
      "KL/rejected_KL_mean": -18.415300369262695,
      "KL/std": 25.572751998901367,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.01632966287434101,
      "fcm_dpo/delta": 0.002343408763408661,
      "fcm_dpo/margin": 21.27467918395996,
      "fcm_dpo/q_t": 0.4193387031555176,
      "grad_norm": 38.54991912841797,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 1.5916173458099365,
      "logits/rejected": 1.4707765579223633,
      "logps/chosen": -299.03302001953125,
      "logps/ref_chosen": -301.89239501953125,
      "logps/ref_rejected": -247.645263671875,
      "logps/rejected": -266.0605773925781,
      "loss": 4.5507,
      "margin_dpo/margin_mean": 21.274681091308594,
      "margin_dpo/margin_std": 32.15087127685547,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": 4.2616729736328125,
      "KL/mean": -5.310194492340088,
      "KL/rejected_KL_mean": -14.882072448730469,
      "KL/std": 24.701536178588867,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.01573173888027668,
      "fcm_dpo/delta": -0.02281479723751545,
      "fcm_dpo/margin": 19.14374351501465,
      "fcm_dpo/q_t": 0.4286409914493561,
      "grad_norm": 37.029823303222656,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": 1.3683712482452393,
      "logits/rejected": 1.3342028856277466,
      "logps/chosen": -283.97137451171875,
      "logps/ref_chosen": -288.2330627441406,
      "logps/ref_rejected": -252.5148162841797,
      "logps/rejected": -267.3968811035156,
      "loss": 4.6671,
      "margin_dpo/margin_mean": 19.14374351501465,
      "margin_dpo/margin_std": 29.76852798461914,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": 2.193357467651367,
      "KL/mean": -8.963401794433594,
      "KL/rejected_KL_mean": -20.12016487121582,
      "KL/std": 24.072208404541016,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.015881981700658798,
      "fcm_dpo/delta": -0.00502699613571167,
      "fcm_dpo/margin": 22.313512802124023,
      "fcm_dpo/q_t": 0.41592657566070557,
      "grad_norm": 37.18266296386719,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": 1.5910364389419556,
      "logits/rejected": 1.5665528774261475,
      "logps/chosen": -284.5704345703125,
      "logps/ref_chosen": -286.7638244628906,
      "logps/ref_rejected": -262.8302917480469,
      "logps/rejected": -282.950439453125,
      "loss": 4.4422,
      "margin_dpo/margin_mean": 22.313512802124023,
      "margin_dpo/margin_std": 26.97385025024414,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -5.206485748291016,
      "KL/mean": -14.868182182312012,
      "KL/rejected_KL_mean": -24.52988624572754,
      "KL/std": 25.66176414489746,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.015808280557394028,
      "fcm_dpo/delta": -0.011972883716225624,
      "fcm_dpo/margin": 19.32340431213379,
      "fcm_dpo/q_t": 0.42815569043159485,
      "grad_norm": 31.49512481689453,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": 1.7359358072280884,
      "logits/rejected": 1.704094409942627,
      "logps/chosen": -265.4849548339844,
      "logps/ref_chosen": -260.2784729003906,
      "logps/ref_rejected": -217.10556030273438,
      "logps/rejected": -241.63543701171875,
      "loss": 4.6352,
      "margin_dpo/margin_mean": 19.32340431213379,
      "margin_dpo/margin_std": 29.102746963500977,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -0.7911815643310547,
      "KL/mean": -8.85163688659668,
      "KL/rejected_KL_mean": -16.91208267211914,
      "KL/std": 23.83381462097168,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.015692435204982758,
      "fcm_dpo/delta": 0.011795670725405216,
      "fcm_dpo/margin": 16.12088966369629,
      "fcm_dpo/q_t": 0.4402773380279541,
      "grad_norm": 31.20162582397461,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": 1.7102930545806885,
      "logits/rejected": 1.7582740783691406,
      "logps/chosen": -271.3454895019531,
      "logps/ref_chosen": -270.5543212890625,
      "logps/ref_rejected": -259.80218505859375,
      "logps/rejected": -276.7142333984375,
      "loss": 4.7942,
      "margin_dpo/margin_mean": 16.120891571044922,
      "margin_dpo/margin_std": 27.525470733642578,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -3.1071395874023438,
      "KL/mean": -10.454446792602539,
      "KL/rejected_KL_mean": -17.80175018310547,
      "KL/std": 26.35810089111328,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.01568167470395565,
      "fcm_dpo/delta": 0.0011580949649214745,
      "fcm_dpo/margin": 14.694615364074707,
      "fcm_dpo/q_t": 0.44691964983940125,
      "grad_norm": 30.55211639404297,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 1.635843276977539,
      "logits/rejected": 1.626558780670166,
      "logps/chosen": -273.4015808105469,
      "logps/ref_chosen": -270.29443359375,
      "logps/ref_rejected": -230.40151977539062,
      "logps/rejected": -248.20326232910156,
      "loss": 4.9668,
      "margin_dpo/margin_mean": 14.69461727142334,
      "margin_dpo/margin_std": 33.95439529418945,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -3.550943374633789,
      "KL/mean": -12.279394149780273,
      "KL/rejected_KL_mean": -21.007844924926758,
      "KL/std": 25.653182983398438,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.016268886625766754,
      "fcm_dpo/delta": 0.03632211312651634,
      "fcm_dpo/margin": 17.45690155029297,
      "fcm_dpo/q_t": 0.43382585048675537,
      "grad_norm": 29.154111862182617,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": 1.7360961437225342,
      "logits/rejected": 1.6948254108428955,
      "logps/chosen": -277.01153564453125,
      "logps/ref_chosen": -273.4605712890625,
      "logps/ref_rejected": -248.86997985839844,
      "logps/rejected": -269.8778076171875,
      "loss": 4.7471,
      "margin_dpo/margin_mean": 17.45690155029297,
      "margin_dpo/margin_std": 30.755687713623047,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": 0.11563491821289062,
      "KL/mean": -11.461945533752441,
      "KL/rejected_KL_mean": -23.039531707763672,
      "KL/std": 27.769437789916992,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.015757236629724503,
      "fcm_dpo/delta": -0.05506858974695206,
      "fcm_dpo/margin": 23.15516471862793,
      "fcm_dpo/q_t": 0.41362375020980835,
      "grad_norm": 29.7043514251709,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": 1.5427602529525757,
      "logits/rejected": 1.5361279249191284,
      "logps/chosen": -305.60906982421875,
      "logps/ref_chosen": -305.7247009277344,
      "logps/ref_rejected": -290.16650390625,
      "logps/rejected": -313.2060546875,
      "loss": 4.4585,
      "margin_dpo/margin_mean": 23.15516471862793,
      "margin_dpo/margin_std": 30.023990631103516,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -0.1829700469970703,
      "KL/mean": -13.410947799682617,
      "KL/rejected_KL_mean": -26.638919830322266,
      "KL/std": 27.48219871520996,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.015085851773619652,
      "fcm_dpo/delta": -0.08233089745044708,
      "fcm_dpo/margin": 26.455947875976562,
      "fcm_dpo/q_t": 0.40659284591674805,
      "grad_norm": 26.725299835205078,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": 1.5867146253585815,
      "logits/rejected": 1.6717966794967651,
      "logps/chosen": -252.35324096679688,
      "logps/ref_chosen": -252.1702880859375,
      "logps/ref_rejected": -233.34750366210938,
      "logps/rejected": -259.98638916015625,
      "loss": 4.3479,
      "margin_dpo/margin_mean": 26.455947875976562,
      "margin_dpo/margin_std": 30.786144256591797,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": 0.48781394958496094,
      "KL/mean": -10.798688888549805,
      "KL/rejected_KL_mean": -22.08519744873047,
      "KL/std": 25.979764938354492,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.014495628885924816,
      "fcm_dpo/delta": 0.004030448384582996,
      "fcm_dpo/margin": 22.573015213012695,
      "fcm_dpo/q_t": 0.4230634868144989,
      "grad_norm": 29.26913070678711,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": 1.5492959022521973,
      "logits/rejected": 1.5965802669525146,
      "logps/chosen": -301.9905700683594,
      "logps/ref_chosen": -302.4783935546875,
      "logps/ref_rejected": -259.40777587890625,
      "logps/rejected": -281.49298095703125,
      "loss": 4.5404,
      "margin_dpo/margin_mean": 22.573015213012695,
      "margin_dpo/margin_std": 31.016324996948242,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": 0.39650535583496094,
      "KL/mean": -11.26140308380127,
      "KL/rejected_KL_mean": -22.919309616088867,
      "KL/std": 26.687667846679688,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.014312580227851868,
      "fcm_dpo/delta": -0.023077016696333885,
      "fcm_dpo/margin": 23.315814971923828,
      "fcm_dpo/q_t": 0.42124834656715393,
      "grad_norm": 29.14301872253418,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 1.384608507156372,
      "logits/rejected": 1.494786024093628,
      "logps/chosen": -250.25021362304688,
      "logps/ref_chosen": -250.646728515625,
      "logps/ref_rejected": -259.1659851074219,
      "logps/rejected": -282.0853271484375,
      "loss": 4.52,
      "margin_dpo/margin_mean": 23.315814971923828,
      "margin_dpo/margin_std": 29.931957244873047,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": 1.2880935668945312,
      "KL/mean": -6.481405735015869,
      "KL/rejected_KL_mean": -14.250913619995117,
      "KL/std": 27.44673728942871,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.01443471573293209,
      "fcm_dpo/delta": 0.007808614522218704,
      "fcm_dpo/margin": 15.538997650146484,
      "fcm_dpo/q_t": 0.44636446237564087,
      "grad_norm": 31.063236236572266,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": 1.7273176908493042,
      "logits/rejected": 1.742636799812317,
      "logps/chosen": -327.7368469238281,
      "logps/ref_chosen": -329.02490234375,
      "logps/ref_rejected": -293.0215759277344,
      "logps/rejected": -307.2724914550781,
      "loss": 4.9207,
      "margin_dpo/margin_mean": 15.538997650146484,
      "margin_dpo/margin_std": 31.846267700195312,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": 0.5969429016113281,
      "KL/mean": -9.004671096801758,
      "KL/rejected_KL_mean": -18.606285095214844,
      "KL/std": 27.319616317749023,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.014587562531232834,
      "fcm_dpo/delta": 0.014506502076983452,
      "fcm_dpo/margin": 19.203231811523438,
      "fcm_dpo/q_t": 0.4340902864933014,
      "grad_norm": 26.893781661987305,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": 1.4867792129516602,
      "logits/rejected": 1.454420804977417,
      "logps/chosen": -289.5436706542969,
      "logps/ref_chosen": -290.140625,
      "logps/ref_rejected": -271.474853515625,
      "logps/rejected": -290.08111572265625,
      "loss": 4.7115,
      "margin_dpo/margin_mean": 19.203231811523438,
      "margin_dpo/margin_std": 30.385982513427734,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": 1.6991462707519531,
      "KL/mean": -10.475726127624512,
      "KL/rejected_KL_mean": -22.650583267211914,
      "KL/std": 26.607711791992188,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.014571263454854488,
      "fcm_dpo/delta": -0.005590707994997501,
      "fcm_dpo/margin": 24.349727630615234,
      "fcm_dpo/q_t": 0.41688546538352966,
      "grad_norm": 28.60854721069336,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": 1.60545814037323,
      "logits/rejected": 1.533188819885254,
      "logps/chosen": -327.6081237792969,
      "logps/ref_chosen": -329.3072509765625,
      "logps/ref_rejected": -256.6839599609375,
      "logps/rejected": -279.3345642089844,
      "loss": 4.4583,
      "margin_dpo/margin_mean": 24.349727630615234,
      "margin_dpo/margin_std": 30.5372371673584,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -0.6039772033691406,
      "KL/mean": -10.574316024780273,
      "KL/rejected_KL_mean": -20.54465675354004,
      "KL/std": 27.627817153930664,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.014636460691690445,
      "fcm_dpo/delta": 0.0071708569303154945,
      "fcm_dpo/margin": 19.940677642822266,
      "fcm_dpo/q_t": 0.43096181750297546,
      "grad_norm": 28.59958267211914,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": 1.5082128047943115,
      "logits/rejected": 1.4971444606781006,
      "logps/chosen": -276.5062561035156,
      "logps/ref_chosen": -275.9023132324219,
      "logps/ref_rejected": -281.4163513183594,
      "logps/rejected": -301.96099853515625,
      "loss": 4.7007,
      "margin_dpo/margin_mean": 19.940677642822266,
      "margin_dpo/margin_std": 32.68938446044922,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -2.598093032836914,
      "KL/mean": -9.85112476348877,
      "KL/rejected_KL_mean": -17.10416030883789,
      "KL/std": 27.142044067382812,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.014547966420650482,
      "fcm_dpo/delta": -0.022165747359395027,
      "fcm_dpo/margin": 14.506064414978027,
      "fcm_dpo/q_t": 0.44962435960769653,
      "grad_norm": 33.748779296875,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": 1.2248111963272095,
      "logits/rejected": 1.2754470109939575,
      "logps/chosen": -246.16229248046875,
      "logps/ref_chosen": -243.56423950195312,
      "logps/ref_rejected": -239.12905883789062,
      "logps/rejected": -256.23321533203125,
      "loss": 5.0071,
      "margin_dpo/margin_mean": 14.506065368652344,
      "margin_dpo/margin_std": 33.28834915161133,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -3.4494762420654297,
      "KL/mean": -10.746357917785645,
      "KL/rejected_KL_mean": -18.043254852294922,
      "KL/std": 28.15262222290039,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.014648891054093838,
      "fcm_dpo/delta": 0.03360263630747795,
      "fcm_dpo/margin": 14.593791007995605,
      "fcm_dpo/q_t": 0.4496957063674927,
      "grad_norm": 49.765586853027344,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": 1.6462645530700684,
      "logits/rejected": 1.5723247528076172,
      "logps/chosen": -296.7531433105469,
      "logps/ref_chosen": -293.30364990234375,
      "logps/ref_rejected": -292.293701171875,
      "logps/rejected": -310.3369445800781,
      "loss": 5.0421,
      "margin_dpo/margin_mean": 14.593791961669922,
      "margin_dpo/margin_std": 36.45195007324219,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": 0.7383861541748047,
      "KL/mean": -10.880173683166504,
      "KL/rejected_KL_mean": -22.498729705810547,
      "KL/std": 29.536632537841797,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.014561534859240055,
      "fcm_dpo/delta": -0.01900009997189045,
      "fcm_dpo/margin": 23.23711585998535,
      "fcm_dpo/q_t": 0.4215569496154785,
      "grad_norm": 34.658103942871094,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": 1.553000569343567,
      "logits/rejected": 1.5740586519241333,
      "logps/chosen": -275.3878479003906,
      "logps/ref_chosen": -276.126220703125,
      "logps/ref_rejected": -240.31179809570312,
      "logps/rejected": -262.8105163574219,
      "loss": 4.57,
      "margin_dpo/margin_mean": 23.23711585998535,
      "margin_dpo/margin_std": 35.19696044921875,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": 0.16250228881835938,
      "KL/mean": -8.079819679260254,
      "KL/rejected_KL_mean": -16.3221435546875,
      "KL/std": 28.571683883666992,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.014571775682270527,
      "fcm_dpo/delta": 0.011283012107014656,
      "fcm_dpo/margin": 16.484657287597656,
      "fcm_dpo/q_t": 0.4438911974430084,
      "grad_norm": 40.181644439697266,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 1.2723461389541626,
      "logits/rejected": 1.3840264081954956,
      "logps/chosen": -249.69992065429688,
      "logps/ref_chosen": -249.8624267578125,
      "logps/ref_rejected": -271.18603515625,
      "logps/rejected": -287.5082092285156,
      "loss": 4.8972,
      "margin_dpo/margin_mean": 16.484657287597656,
      "margin_dpo/margin_std": 33.349483489990234,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": 0.8390045166015625,
      "KL/mean": -10.299476623535156,
      "KL/rejected_KL_mean": -21.437957763671875,
      "KL/std": 28.051979064941406,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.014693993143737316,
      "fcm_dpo/delta": -0.005156148225069046,
      "fcm_dpo/margin": 22.27696990966797,
      "fcm_dpo/q_t": 0.4239422082901001,
      "grad_norm": 31.738719940185547,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": 1.4623596668243408,
      "logits/rejected": 1.4917339086532593,
      "logps/chosen": -259.8301086425781,
      "logps/ref_chosen": -260.66912841796875,
      "logps/ref_rejected": -229.25491333007812,
      "logps/rejected": -250.69287109375,
      "loss": 4.5894,
      "margin_dpo/margin_mean": 22.27696990966797,
      "margin_dpo/margin_std": 33.356170654296875,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": 2.601388931274414,
      "KL/mean": -7.989866256713867,
      "KL/rejected_KL_mean": -18.581104278564453,
      "KL/std": 28.16393280029297,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.014405172318220139,
      "fcm_dpo/delta": -0.02288603037595749,
      "fcm_dpo/margin": 21.182498931884766,
      "fcm_dpo/q_t": 0.42837420105934143,
      "grad_norm": 29.90167999267578,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.4487864971160889,
      "logits/rejected": 1.4095501899719238,
      "logps/chosen": -288.28717041015625,
      "logps/ref_chosen": -290.8885803222656,
      "logps/ref_rejected": -275.1922302246094,
      "logps/rejected": -293.7733154296875,
      "loss": 4.6795,
      "margin_dpo/margin_mean": 21.182497024536133,
      "margin_dpo/margin_std": 34.764488220214844,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": 0.3690147399902344,
      "KL/mean": -12.685291290283203,
      "KL/rejected_KL_mean": -25.739601135253906,
      "KL/std": 28.62614631652832,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.014169503934681416,
      "fcm_dpo/delta": -0.03983201086521149,
      "fcm_dpo/margin": 26.108627319335938,
      "fcm_dpo/q_t": 0.41208019852638245,
      "grad_norm": 30.358043670654297,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": 1.4860947132110596,
      "logits/rejected": 1.5060608386993408,
      "logps/chosen": -270.19781494140625,
      "logps/ref_chosen": -270.56683349609375,
      "logps/ref_rejected": -260.3321533203125,
      "logps/rejected": -286.0717468261719,
      "loss": 4.398,
      "margin_dpo/margin_mean": 26.108627319335938,
      "margin_dpo/margin_std": 30.690927505493164,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": 0.6691818237304688,
      "KL/mean": -10.100210189819336,
      "KL/rejected_KL_mean": -20.86960220336914,
      "KL/std": 28.66571807861328,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.013951760716736317,
      "fcm_dpo/delta": 0.03157544136047363,
      "fcm_dpo/margin": 21.53877830505371,
      "fcm_dpo/q_t": 0.4296523928642273,
      "grad_norm": 30.643178939819336,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": 1.6787718534469604,
      "logits/rejected": 1.732492446899414,
      "logps/chosen": -260.3587951660156,
      "logps/ref_chosen": -261.0279846191406,
      "logps/ref_rejected": -240.72361755371094,
      "logps/rejected": -261.5932312011719,
      "loss": 4.6675,
      "margin_dpo/margin_mean": 21.53877830505371,
      "margin_dpo/margin_std": 34.69694519042969,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": 4.1945953369140625,
      "KL/mean": -11.411913871765137,
      "KL/rejected_KL_mean": -27.018423080444336,
      "KL/std": 32.246070861816406,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.013546817936003208,
      "fcm_dpo/delta": -0.07761034369468689,
      "fcm_dpo/margin": 31.213027954101562,
      "fcm_dpo/q_t": 0.4009802043437958,
      "grad_norm": 45.62461853027344,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": 1.6049247980117798,
      "logits/rejected": 1.5934560298919678,
      "logps/chosen": -304.8892517089844,
      "logps/ref_chosen": -309.0838317871094,
      "logps/ref_rejected": -263.6435241699219,
      "logps/rejected": -290.6619873046875,
      "loss": 4.3006,
      "margin_dpo/margin_mean": 31.213024139404297,
      "margin_dpo/margin_std": 38.831932067871094,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -1.7257041931152344,
      "KL/mean": -12.899564743041992,
      "KL/rejected_KL_mean": -24.07342529296875,
      "KL/std": 30.48215675354004,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.013345280662178993,
      "fcm_dpo/delta": -0.0015684561803936958,
      "fcm_dpo/margin": 22.347719192504883,
      "fcm_dpo/q_t": 0.4304111897945404,
      "grad_norm": 31.25998878479004,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": 1.3531945943832397,
      "logits/rejected": 1.4149812459945679,
      "logps/chosen": -267.42742919921875,
      "logps/ref_chosen": -265.70172119140625,
      "logps/ref_rejected": -257.5074157714844,
      "logps/rejected": -281.5808410644531,
      "loss": 4.6939,
      "margin_dpo/margin_mean": 22.34771728515625,
      "margin_dpo/margin_std": 36.67365646362305,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -4.714805603027344,
      "KL/mean": -15.543889999389648,
      "KL/rejected_KL_mean": -26.37298011779785,
      "KL/std": 29.30013656616211,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.013270964846014977,
      "fcm_dpo/delta": -0.008265113458037376,
      "fcm_dpo/margin": 21.658170700073242,
      "fcm_dpo/q_t": 0.43215349316596985,
      "grad_norm": 30.679100036621094,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": 1.5358141660690308,
      "logits/rejected": 1.5627222061157227,
      "logps/chosen": -296.4397888183594,
      "logps/ref_chosen": -291.7249755859375,
      "logps/ref_rejected": -273.54010009765625,
      "logps/rejected": -299.91314697265625,
      "loss": 4.6825,
      "margin_dpo/margin_mean": 21.658174514770508,
      "margin_dpo/margin_std": 33.335670471191406,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -4.497890472412109,
      "KL/mean": -15.550649642944336,
      "KL/rejected_KL_mean": -26.603410720825195,
      "KL/std": 31.563772201538086,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.013387581333518028,
      "fcm_dpo/delta": 0.028582941740751266,
      "fcm_dpo/margin": 22.10552406311035,
      "fcm_dpo/q_t": 0.4315081238746643,
      "grad_norm": 46.25345230102539,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": 1.5313571691513062,
      "logits/rejected": 1.5519829988479614,
      "logps/chosen": -326.1997375488281,
      "logps/ref_chosen": -321.70184326171875,
      "logps/ref_rejected": -261.8386535644531,
      "logps/rejected": -288.44207763671875,
      "loss": 4.7469,
      "margin_dpo/margin_mean": 22.105525970458984,
      "margin_dpo/margin_std": 40.669254302978516,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -5.854013442993164,
      "KL/mean": -17.669231414794922,
      "KL/rejected_KL_mean": -29.48446273803711,
      "KL/std": 31.350975036621094,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.013524826616048813,
      "fcm_dpo/delta": -0.011761566624045372,
      "fcm_dpo/margin": 23.630451202392578,
      "fcm_dpo/q_t": 0.4250749349594116,
      "grad_norm": 26.77386474609375,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": 1.4367499351501465,
      "logits/rejected": 1.4131824970245361,
      "logps/chosen": -261.7694396972656,
      "logps/ref_chosen": -255.91542053222656,
      "logps/ref_rejected": -245.1119842529297,
      "logps/rejected": -274.596435546875,
      "loss": 4.6478,
      "margin_dpo/margin_mean": 23.630451202392578,
      "margin_dpo/margin_std": 38.802860260009766,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": 0.6169719696044922,
      "KL/mean": -13.166799545288086,
      "KL/rejected_KL_mean": -26.950572967529297,
      "KL/std": 31.058868408203125,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.013076228089630604,
      "fcm_dpo/delta": -0.06478720903396606,
      "fcm_dpo/margin": 27.567546844482422,
      "fcm_dpo/q_t": 0.41436100006103516,
      "grad_norm": 27.982282638549805,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": 1.4188218116760254,
      "logits/rejected": 1.447735071182251,
      "logps/chosen": -265.40155029296875,
      "logps/ref_chosen": -266.0185546875,
      "logps/ref_rejected": -272.862548828125,
      "logps/rejected": -299.8131408691406,
      "loss": 4.4161,
      "margin_dpo/margin_mean": 27.567550659179688,
      "margin_dpo/margin_std": 31.97284507751465,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -2.2392826080322266,
      "KL/mean": -15.607118606567383,
      "KL/rejected_KL_mean": -28.97494888305664,
      "KL/std": 31.12480926513672,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.01282543409615755,
      "fcm_dpo/delta": 0.006976674310863018,
      "fcm_dpo/margin": 26.735673904418945,
      "fcm_dpo/q_t": 0.41944795846939087,
      "grad_norm": 30.025171279907227,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 1.3961399793624878,
      "logits/rejected": 1.3307793140411377,
      "logps/chosen": -252.3902587890625,
      "logps/ref_chosen": -250.15098571777344,
      "logps/ref_rejected": -223.34393310546875,
      "logps/rejected": -252.31884765625,
      "loss": 4.5429,
      "margin_dpo/margin_mean": 26.735673904418945,
      "margin_dpo/margin_std": 38.9410514831543,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -1.7710304260253906,
      "KL/mean": -13.45197868347168,
      "KL/rejected_KL_mean": -25.132925033569336,
      "KL/std": 32.31299591064453,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.01287173293530941,
      "fcm_dpo/delta": 0.014381160959601402,
      "fcm_dpo/margin": 23.361881256103516,
      "fcm_dpo/q_t": 0.42989999055862427,
      "grad_norm": 28.308029174804688,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 1.1891112327575684,
      "logits/rejected": 1.243423581123352,
      "logps/chosen": -240.16285705566406,
      "logps/ref_chosen": -238.3917999267578,
      "logps/ref_rejected": -243.70272827148438,
      "logps/rejected": -268.8356628417969,
      "loss": 4.7073,
      "margin_dpo/margin_mean": 23.36188316345215,
      "margin_dpo/margin_std": 40.21757507324219,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -7.069040298461914,
      "KL/mean": -17.562023162841797,
      "KL/rejected_KL_mean": -28.054996490478516,
      "KL/std": 31.5565185546875,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.012763913720846176,
      "fcm_dpo/delta": -0.01644499972462654,
      "fcm_dpo/margin": 20.985958099365234,
      "fcm_dpo/q_t": 0.43665024638175964,
      "grad_norm": 34.7735481262207,
      "learning_rate": 3.75e-07,
      "logits/chosen": 1.4003510475158691,
      "logits/rejected": 1.4679447412490845,
      "logps/chosen": -263.7060241699219,
      "logps/ref_chosen": -256.6369934082031,
      "logps/ref_rejected": -256.8894348144531,
      "logps/rejected": -284.9444274902344,
      "loss": 4.8249,
      "margin_dpo/margin_mean": 20.985958099365234,
      "margin_dpo/margin_std": 39.33759689331055,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -9.22610855102539,
      "KL/mean": -20.209728240966797,
      "KL/rejected_KL_mean": -31.193355560302734,
      "KL/std": 33.76824951171875,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.01292492263019085,
      "fcm_dpo/delta": 0.019002258777618408,
      "fcm_dpo/margin": 21.967243194580078,
      "fcm_dpo/q_t": 0.43383902311325073,
      "grad_norm": 30.886518478393555,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": 1.5099260807037354,
      "logits/rejected": 1.6060292720794678,
      "logps/chosen": -282.0257873535156,
      "logps/ref_chosen": -272.7996826171875,
      "logps/ref_rejected": -289.2391052246094,
      "logps/rejected": -320.4324645996094,
      "loss": 4.7457,
      "margin_dpo/margin_mean": 21.967243194580078,
      "margin_dpo/margin_std": 39.193119049072266,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -7.873512268066406,
      "KL/mean": -16.123220443725586,
      "KL/rejected_KL_mean": -24.37293243408203,
      "KL/std": 33.67116165161133,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.013049191795289516,
      "fcm_dpo/delta": 0.0064622750505805016,
      "fcm_dpo/margin": 16.499408721923828,
      "fcm_dpo/q_t": 0.4502105116844177,
      "grad_norm": 28.202543258666992,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": 1.3253309726715088,
      "logits/rejected": 1.3660714626312256,
      "logps/chosen": -289.18255615234375,
      "logps/ref_chosen": -281.30902099609375,
      "logps/ref_rejected": -246.3243408203125,
      "logps/rejected": -270.6972961425781,
      "loss": 4.9697,
      "margin_dpo/margin_mean": 16.499406814575195,
      "margin_dpo/margin_std": 36.595218658447266,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -8.23170280456543,
      "KL/mean": -16.38392448425293,
      "KL/rejected_KL_mean": -24.536142349243164,
      "KL/std": 33.066192626953125,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.013172554783523083,
      "fcm_dpo/delta": 0.018493063747882843,
      "fcm_dpo/margin": 16.304454803466797,
      "fcm_dpo/q_t": 0.4498530328273773,
      "grad_norm": 32.54745864868164,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": 1.481441855430603,
      "logits/rejected": 1.5793960094451904,
      "logps/chosen": -275.08984375,
      "logps/ref_chosen": -266.8581237792969,
      "logps/ref_rejected": -276.0855712890625,
      "logps/rejected": -300.6217041015625,
      "loss": 4.971,
      "margin_dpo/margin_mean": 16.304454803466797,
      "margin_dpo/margin_std": 36.80573654174805,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -1.1319255828857422,
      "KL/mean": -14.280367851257324,
      "KL/rejected_KL_mean": -27.428813934326172,
      "KL/std": 31.425127029418945,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.012993020005524158,
      "fcm_dpo/delta": -0.056588925421237946,
      "fcm_dpo/margin": 26.296886444091797,
      "fcm_dpo/q_t": 0.4195958971977234,
      "grad_norm": 34.21297836303711,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": 1.349816083908081,
      "logits/rejected": 1.4543536901474,
      "logps/chosen": -268.12689208984375,
      "logps/ref_chosen": -266.99493408203125,
      "logps/ref_rejected": -248.60513305664062,
      "logps/rejected": -276.0339660644531,
      "loss": 4.5208,
      "margin_dpo/margin_mean": 26.296886444091797,
      "margin_dpo/margin_std": 35.68197250366211,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": 1.521078109741211,
      "KL/mean": -15.074859619140625,
      "KL/rejected_KL_mean": -31.67078971862793,
      "KL/std": 34.696231842041016,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.012350260280072689,
      "fcm_dpo/delta": -0.09239321947097778,
      "fcm_dpo/margin": 33.191864013671875,
      "fcm_dpo/q_t": 0.4033791422843933,
      "grad_norm": 32.241180419921875,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": 1.5245367288589478,
      "logits/rejected": 1.5082204341888428,
      "logps/chosen": -301.4687194824219,
      "logps/ref_chosen": -302.98980712890625,
      "logps/ref_rejected": -262.8002624511719,
      "logps/rejected": -294.47100830078125,
      "loss": 4.3021,
      "margin_dpo/margin_mean": 33.191864013671875,
      "margin_dpo/margin_std": 37.09834289550781,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -1.5893135070800781,
      "KL/mean": -13.996271133422852,
      "KL/rejected_KL_mean": -26.403228759765625,
      "KL/std": 34.424808502197266,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.011691112071275711,
      "fcm_dpo/delta": 0.007009413093328476,
      "fcm_dpo/margin": 24.813915252685547,
      "fcm_dpo/q_t": 0.4319559335708618,
      "grad_norm": 27.28823471069336,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": 1.4097182750701904,
      "logits/rejected": 1.4914370775222778,
      "logps/chosen": -297.80108642578125,
      "logps/ref_chosen": -296.2117614746094,
      "logps/ref_rejected": -289.8357238769531,
      "logps/rejected": -316.23895263671875,
      "loss": 4.7397,
      "margin_dpo/margin_mean": 24.81391716003418,
      "margin_dpo/margin_std": 43.79566955566406,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -2.698129653930664,
      "KL/mean": -15.316537857055664,
      "KL/rejected_KL_mean": -27.934938430786133,
      "KL/std": 32.815879821777344,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.012067675590515137,
      "fcm_dpo/delta": 0.019923735409975052,
      "fcm_dpo/margin": 25.23680877685547,
      "fcm_dpo/q_t": 0.42850154638290405,
      "grad_norm": 25.719758987426758,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": 1.2751790285110474,
      "logits/rejected": 1.202856421470642,
      "logps/chosen": -257.9907531738281,
      "logps/ref_chosen": -255.29263305664062,
      "logps/ref_rejected": -212.69761657714844,
      "logps/rejected": -240.632568359375,
      "loss": 4.6557,
      "margin_dpo/margin_mean": 25.23680877685547,
      "margin_dpo/margin_std": 40.285091400146484,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -1.8063220977783203,
      "KL/mean": -12.176982879638672,
      "KL/rejected_KL_mean": -22.54764175415039,
      "KL/std": 33.8546142578125,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.012231405824422836,
      "fcm_dpo/delta": 0.03720745071768761,
      "fcm_dpo/margin": 20.741317749023438,
      "fcm_dpo/q_t": 0.44171106815338135,
      "grad_norm": 32.25979232788086,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 1.5977997779846191,
      "logits/rejected": 1.5860803127288818,
      "logps/chosen": -315.3427429199219,
      "logps/ref_chosen": -313.5364074707031,
      "logps/ref_rejected": -274.349609375,
      "logps/rejected": -296.897216796875,
      "loss": 4.9323,
      "margin_dpo/margin_mean": 20.741317749023438,
      "margin_dpo/margin_std": 47.0469856262207,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_KL/chosen_KL_mean": -5.410290241241455,
      "eval_KL/mean": -17.659828186035156,
      "eval_KL/rejected_KL_mean": -29.909364700317383,
      "eval_KL/std": 33.77998352050781,
      "eval_fcm_dpo/beta": 0.01241496205329895,
      "eval_logits/chosen": 1.5348118543624878,
      "eval_logits/rejected": 1.565679669380188,
      "eval_logps/chosen": -285.8269958496094,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -294.5549011230469,
      "eval_loss": 0.5859270095825195,
      "eval_margin_dpo/margin_mean": 24.499073028564453,
      "eval_margin_dpo/margin_std": 38.586021423339844,
      "eval_runtime": 88.4745,
      "eval_samples_per_second": 22.605,
      "eval_steps_per_second": 2.826,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -3.7221412658691406,
      "KL/mean": -17.519411087036133,
      "KL/rejected_KL_mean": -31.316675186157227,
      "KL/std": 34.73529815673828,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.012373468838632107,
      "fcm_dpo/delta": -0.02532346174120903,
      "fcm_dpo/margin": 27.594532012939453,
      "fcm_dpo/q_t": 0.4196230173110962,
      "grad_norm": 31.977867126464844,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 1.2489526271820068,
      "logits/rejected": 1.2757447957992554,
      "logps/chosen": -266.4293212890625,
      "logps/ref_chosen": -262.7071838378906,
      "logps/ref_rejected": -243.85879516601562,
      "logps/rejected": -275.17547607421875,
      "loss": 4.5411,
      "margin_dpo/margin_mean": 27.594532012939453,
      "margin_dpo/margin_std": 38.8919677734375,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -7.029844284057617,
      "KL/mean": -19.553714752197266,
      "KL/rejected_KL_mean": -32.07758331298828,
      "KL/std": 34.74762725830078,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.012009150348603725,
      "fcm_dpo/delta": -0.020928572863340378,
      "fcm_dpo/margin": 25.0477352142334,
      "fcm_dpo/q_t": 0.42958664894104004,
      "grad_norm": 28.7772216796875,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": 1.413314700126648,
      "logits/rejected": 1.3837385177612305,
      "logps/chosen": -271.31658935546875,
      "logps/ref_chosen": -264.2867431640625,
      "logps/ref_rejected": -231.6894989013672,
      "logps/rejected": -263.76708984375,
      "loss": 4.7293,
      "margin_dpo/margin_mean": 25.04773712158203,
      "margin_dpo/margin_std": 43.96614074707031,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -5.873542785644531,
      "KL/mean": -16.45172691345215,
      "KL/rejected_KL_mean": -27.029909133911133,
      "KL/std": 33.4597053527832,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.012204562313854694,
      "fcm_dpo/delta": 0.04473528265953064,
      "fcm_dpo/margin": 21.156368255615234,
      "fcm_dpo/q_t": 0.4388829469680786,
      "grad_norm": 44.532501220703125,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": 1.445522427558899,
      "logits/rejected": 1.4157874584197998,
      "logps/chosen": -277.15771484375,
      "logps/ref_chosen": -271.2842102050781,
      "logps/ref_rejected": -247.95462036132812,
      "logps/rejected": -274.9845275878906,
      "loss": 4.7564,
      "margin_dpo/margin_mean": 21.156368255615234,
      "margin_dpo/margin_std": 33.940940856933594,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -4.11598014831543,
      "KL/mean": -19.07703971862793,
      "KL/rejected_KL_mean": -34.03810501098633,
      "KL/std": 34.45749282836914,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.012221751734614372,
      "fcm_dpo/delta": -0.016550766304135323,
      "fcm_dpo/margin": 29.922134399414062,
      "fcm_dpo/q_t": 0.41452717781066895,
      "grad_norm": 31.315671920776367,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": 1.4726027250289917,
      "logits/rejected": 1.4374736547470093,
      "logps/chosen": -314.2464294433594,
      "logps/ref_chosen": -310.13043212890625,
      "logps/ref_rejected": -267.3905334472656,
      "logps/rejected": -301.4286193847656,
      "loss": 4.4531,
      "margin_dpo/margin_mean": 29.92213249206543,
      "margin_dpo/margin_std": 40.28830337524414,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -2.5549163818359375,
      "KL/mean": -15.990964889526367,
      "KL/rejected_KL_mean": -29.4270076751709,
      "KL/std": 34.03913116455078,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.012213467620313168,
      "fcm_dpo/delta": -0.02551940456032753,
      "fcm_dpo/margin": 26.87209701538086,
      "fcm_dpo/q_t": 0.42326200008392334,
      "grad_norm": 31.436506271362305,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 1.3607242107391357,
      "logits/rejected": 1.3596073389053345,
      "logps/chosen": -279.7156066894531,
      "logps/ref_chosen": -277.16064453125,
      "logps/ref_rejected": -292.91815185546875,
      "logps/rejected": -322.34521484375,
      "loss": 4.6082,
      "margin_dpo/margin_mean": 26.872095108032227,
      "margin_dpo/margin_std": 41.10820770263672,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -2.8938941955566406,
      "KL/mean": -15.508560180664062,
      "KL/rejected_KL_mean": -28.123228073120117,
      "KL/std": 34.658504486083984,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.011895447969436646,
      "fcm_dpo/delta": 0.0004592742770910263,
      "fcm_dpo/margin": 25.229337692260742,
      "fcm_dpo/q_t": 0.4292401075363159,
      "grad_norm": 29.424240112304688,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": 1.5487111806869507,
      "logits/rejected": 1.6584646701812744,
      "logps/chosen": -289.6764831542969,
      "logps/ref_chosen": -286.7825927734375,
      "logps/ref_rejected": -289.08270263671875,
      "logps/rejected": -317.2059020996094,
      "loss": 4.6833,
      "margin_dpo/margin_mean": 25.229339599609375,
      "margin_dpo/margin_std": 40.591800689697266,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -6.0487213134765625,
      "KL/mean": -17.73938751220703,
      "KL/rejected_KL_mean": -29.430057525634766,
      "KL/std": 36.98566436767578,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.011911369860172272,
      "fcm_dpo/delta": -0.004130927380174398,
      "fcm_dpo/margin": 23.381338119506836,
      "fcm_dpo/q_t": 0.43563973903656006,
      "grad_norm": 30.175479888916016,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": 1.336180567741394,
      "logits/rejected": 1.441645860671997,
      "logps/chosen": -272.58099365234375,
      "logps/ref_chosen": -266.5322570800781,
      "logps/ref_rejected": -275.20672607421875,
      "logps/rejected": -304.6368103027344,
      "loss": 4.8291,
      "margin_dpo/margin_mean": 23.38134002685547,
      "margin_dpo/margin_std": 47.05997848510742,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -5.421468734741211,
      "KL/mean": -18.70594596862793,
      "KL/rejected_KL_mean": -31.99042510986328,
      "KL/std": 36.047332763671875,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.011926165781915188,
      "fcm_dpo/delta": -0.02367500774562359,
      "fcm_dpo/margin": 26.56895637512207,
      "fcm_dpo/q_t": 0.425921231508255,
      "grad_norm": 27.188289642333984,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": 1.530231237411499,
      "logits/rejected": 1.5391017198562622,
      "logps/chosen": -278.2228088378906,
      "logps/ref_chosen": -272.80133056640625,
      "logps/ref_rejected": -288.4167785644531,
      "logps/rejected": -320.40716552734375,
      "loss": 4.641,
      "margin_dpo/margin_mean": 26.568958282470703,
      "margin_dpo/margin_std": 42.082916259765625,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -8.834991455078125,
      "KL/mean": -20.253042221069336,
      "KL/rejected_KL_mean": -31.671104431152344,
      "KL/std": 36.089778900146484,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.011618104763329029,
      "fcm_dpo/delta": -0.01005207933485508,
      "fcm_dpo/margin": 22.836122512817383,
      "fcm_dpo/q_t": 0.4380141496658325,
      "grad_norm": 28.982921600341797,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": 1.5916130542755127,
      "logits/rejected": 1.4594653844833374,
      "logps/chosen": -319.514404296875,
      "logps/ref_chosen": -310.679443359375,
      "logps/ref_rejected": -259.3792419433594,
      "logps/rejected": -291.0503234863281,
      "loss": 4.8111,
      "margin_dpo/margin_mean": 22.836122512817383,
      "margin_dpo/margin_std": 41.5211067199707,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -4.1964263916015625,
      "KL/mean": -17.632701873779297,
      "KL/rejected_KL_mean": -31.068981170654297,
      "KL/std": 36.62057113647461,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.011459911242127419,
      "fcm_dpo/delta": -0.012191078625619411,
      "fcm_dpo/margin": 26.872543334960938,
      "fcm_dpo/q_t": 0.4275331199169159,
      "grad_norm": 38.898990631103516,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 1.5657176971435547,
      "logits/rejected": 1.6077154874801636,
      "logps/chosen": -279.1645812988281,
      "logps/ref_chosen": -274.9681701660156,
      "logps/ref_rejected": -283.346923828125,
      "logps/rejected": -314.41595458984375,
      "loss": 4.6573,
      "margin_dpo/margin_mean": 26.872543334960938,
      "margin_dpo/margin_std": 42.73552322387695,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -2.3736629486083984,
      "KL/mean": -19.862415313720703,
      "KL/rejected_KL_mean": -37.351173400878906,
      "KL/std": 36.63434600830078,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.011161965318024158,
      "fcm_dpo/delta": -0.07134229689836502,
      "fcm_dpo/margin": 34.97750473022461,
      "fcm_dpo/q_t": 0.4074971377849579,
      "grad_norm": 25.51421546936035,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.5607993602752686,
      "logits/rejected": 1.5607770681381226,
      "logps/chosen": -268.65325927734375,
      "logps/ref_chosen": -266.2795715332031,
      "logps/ref_rejected": -249.7120819091797,
      "logps/rejected": -287.06329345703125,
      "loss": 4.3312,
      "margin_dpo/margin_mean": 34.97750473022461,
      "margin_dpo/margin_std": 38.056697845458984,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -5.078794479370117,
      "KL/mean": -18.220996856689453,
      "KL/rejected_KL_mean": -31.36318588256836,
      "KL/std": 36.19005584716797,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.010863772593438625,
      "fcm_dpo/delta": 0.009056318551301956,
      "fcm_dpo/margin": 26.284399032592773,
      "fcm_dpo/q_t": 0.432039350271225,
      "grad_norm": 33.2435417175293,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": 1.3537421226501465,
      "logits/rejected": 1.3926105499267578,
      "logps/chosen": -269.83160400390625,
      "logps/ref_chosen": -264.7528076171875,
      "logps/ref_rejected": -241.02842712402344,
      "logps/rejected": -272.3916015625,
      "loss": 4.7092,
      "margin_dpo/margin_mean": 26.284400939941406,
      "margin_dpo/margin_std": 42.68193054199219,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -10.200239181518555,
      "KL/mean": -22.536035537719727,
      "KL/rejected_KL_mean": -34.87183380126953,
      "KL/std": 36.69623565673828,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.010983395390212536,
      "fcm_dpo/delta": 0.004582123830914497,
      "fcm_dpo/margin": 24.671600341796875,
      "fcm_dpo/q_t": 0.43693676590919495,
      "grad_norm": 29.86541175842285,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": 1.427290439605713,
      "logits/rejected": 1.5478774309158325,
      "logps/chosen": -304.3388366699219,
      "logps/ref_chosen": -294.13861083984375,
      "logps/ref_rejected": -303.11669921875,
      "logps/rejected": -337.988525390625,
      "loss": 4.7606,
      "margin_dpo/margin_mean": 24.671600341796875,
      "margin_dpo/margin_std": 42.707393646240234,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -21.291706085205078,
      "KL/mean": -32.40495681762695,
      "KL/rejected_KL_mean": -43.51820755004883,
      "KL/std": 35.44430923461914,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.01094653271138668,
      "fcm_dpo/delta": -0.019153019413352013,
      "fcm_dpo/margin": 22.226491928100586,
      "fcm_dpo/q_t": 0.44278404116630554,
      "grad_norm": 32.5620231628418,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": 1.4371554851531982,
      "logits/rejected": 1.5856735706329346,
      "logps/chosen": -287.86846923828125,
      "logps/ref_chosen": -266.5767517089844,
      "logps/ref_rejected": -291.2264709472656,
      "logps/rejected": -334.74468994140625,
      "loss": 4.8475,
      "margin_dpo/margin_mean": 22.226491928100586,
      "margin_dpo/margin_std": 41.1622200012207,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -14.265382766723633,
      "KL/mean": -26.52931785583496,
      "KL/rejected_KL_mean": -38.79326248168945,
      "KL/std": 39.53722381591797,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.010910561308264732,
      "fcm_dpo/delta": 0.022356519475579262,
      "fcm_dpo/margin": 24.527870178222656,
      "fcm_dpo/q_t": 0.4381750524044037,
      "grad_norm": 31.876588821411133,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": 1.5163779258728027,
      "logits/rejected": 1.5200090408325195,
      "logps/chosen": -299.3061218261719,
      "logps/ref_chosen": -285.040771484375,
      "logps/ref_rejected": -258.0254821777344,
      "logps/rejected": -296.8187255859375,
      "loss": 4.8135,
      "margin_dpo/margin_mean": 24.52787208557129,
      "margin_dpo/margin_std": 46.94688415527344,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -15.323986053466797,
      "KL/mean": -28.67882537841797,
      "KL/rejected_KL_mean": -42.033660888671875,
      "KL/std": 40.500614166259766,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.01104488130658865,
      "fcm_dpo/delta": -0.0009606573730707169,
      "fcm_dpo/margin": 26.70969009399414,
      "fcm_dpo/q_t": 0.43062928318977356,
      "grad_norm": 52.82644271850586,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": 1.6028574705123901,
      "logits/rejected": 1.6213407516479492,
      "logps/chosen": -348.0753479003906,
      "logps/ref_chosen": -332.7513732910156,
      "logps/ref_rejected": -302.9167785644531,
      "logps/rejected": -344.9504089355469,
      "loss": 4.7602,
      "margin_dpo/margin_mean": 26.709692001342773,
      "margin_dpo/margin_std": 49.15129470825195,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -15.361120223999023,
      "KL/mean": -33.94387435913086,
      "KL/rejected_KL_mean": -52.52662658691406,
      "KL/std": 39.59142303466797,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.010764097794890404,
      "fcm_dpo/delta": -0.05242647975683212,
      "fcm_dpo/margin": 37.16550827026367,
      "fcm_dpo/q_t": 0.4073862135410309,
      "grad_norm": 30.919113159179688,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": 1.4598994255065918,
      "logits/rejected": 1.4568374156951904,
      "logps/chosen": -301.82904052734375,
      "logps/ref_chosen": -286.46795654296875,
      "logps/ref_rejected": -253.99298095703125,
      "logps/rejected": -306.51959228515625,
      "loss": 4.3726,
      "margin_dpo/margin_mean": 37.16550827026367,
      "margin_dpo/margin_std": 48.50450134277344,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -12.18088150024414,
      "KL/mean": -27.138734817504883,
      "KL/rejected_KL_mean": -42.096580505371094,
      "KL/std": 39.389923095703125,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.010711118578910828,
      "fcm_dpo/delta": 0.009858770295977592,
      "fcm_dpo/margin": 29.915679931640625,
      "fcm_dpo/q_t": 0.42433613538742065,
      "grad_norm": 36.198455810546875,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 1.4772182703018188,
      "logits/rejected": 1.5240106582641602,
      "logps/chosen": -323.5594482421875,
      "logps/ref_chosen": -311.3785400390625,
      "logps/ref_rejected": -290.3092956542969,
      "logps/rejected": -332.4058837890625,
      "loss": 4.5745,
      "margin_dpo/margin_mean": 29.915679931640625,
      "margin_dpo/margin_std": 42.48519515991211,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -12.033224105834961,
      "KL/mean": -26.832868576049805,
      "KL/rejected_KL_mean": -41.63250732421875,
      "KL/std": 37.529998779296875,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.010566053912043571,
      "fcm_dpo/delta": -0.006896474398672581,
      "fcm_dpo/margin": 29.599288940429688,
      "fcm_dpo/q_t": 0.4260551929473877,
      "grad_norm": 41.803077697753906,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": 1.2191658020019531,
      "logits/rejected": 1.2737482786178589,
      "logps/chosen": -271.36614990234375,
      "logps/ref_chosen": -259.33294677734375,
      "logps/ref_rejected": -261.7661437988281,
      "logps/rejected": -303.3986511230469,
      "loss": 4.6202,
      "margin_dpo/margin_mean": 29.59929084777832,
      "margin_dpo/margin_std": 44.6679573059082,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -13.808561325073242,
      "KL/mean": -29.368690490722656,
      "KL/rejected_KL_mean": -44.92881774902344,
      "KL/std": 40.09233856201172,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.010643276385962963,
      "fcm_dpo/delta": 0.019416026771068573,
      "fcm_dpo/margin": 31.120250701904297,
      "fcm_dpo/q_t": 0.4224722683429718,
      "grad_norm": 56.81173324584961,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 1.7514309883117676,
      "logits/rejected": 1.7150810956954956,
      "logps/chosen": -320.012939453125,
      "logps/ref_chosen": -306.2043762207031,
      "logps/ref_rejected": -256.7926330566406,
      "logps/rejected": -301.7214660644531,
      "loss": 4.5919,
      "margin_dpo/margin_mean": 31.120250701904297,
      "margin_dpo/margin_std": 48.43251419067383,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -14.225717544555664,
      "KL/mean": -26.480388641357422,
      "KL/rejected_KL_mean": -38.73506164550781,
      "KL/std": 40.26365661621094,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.010754149407148361,
      "fcm_dpo/delta": -0.014078240841627121,
      "fcm_dpo/margin": 24.50934600830078,
      "fcm_dpo/q_t": 0.4375441372394562,
      "grad_norm": 26.611108779907227,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 1.1856746673583984,
      "logits/rejected": 1.3015198707580566,
      "logps/chosen": -307.55767822265625,
      "logps/ref_chosen": -293.3319396972656,
      "logps/ref_rejected": -282.7599182128906,
      "logps/rejected": -321.4949951171875,
      "loss": 4.8041,
      "margin_dpo/margin_mean": 24.50934600830078,
      "margin_dpo/margin_std": 44.65764617919922,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -10.101140975952148,
      "KL/mean": -26.346946716308594,
      "KL/rejected_KL_mean": -42.592742919921875,
      "KL/std": 38.68461990356445,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.010501356795430183,
      "fcm_dpo/delta": -0.010066326707601547,
      "fcm_dpo/margin": 32.49159622192383,
      "fcm_dpo/q_t": 0.4193086624145508,
      "grad_norm": 37.93665313720703,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": 1.5162588357925415,
      "logits/rejected": 1.4309250116348267,
      "logps/chosen": -297.1072998046875,
      "logps/ref_chosen": -287.0061340332031,
      "logps/ref_rejected": -233.23130798339844,
      "logps/rejected": -275.82403564453125,
      "loss": 4.5074,
      "margin_dpo/margin_mean": 32.491600036621094,
      "margin_dpo/margin_std": 42.84659194946289,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -10.967178344726562,
      "KL/mean": -28.739547729492188,
      "KL/rejected_KL_mean": -46.511924743652344,
      "KL/std": 41.87736129760742,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.010262366384267807,
      "fcm_dpo/delta": -0.05380725860595703,
      "fcm_dpo/margin": 35.54475402832031,
      "fcm_dpo/q_t": 0.4140002131462097,
      "grad_norm": 29.36161231994629,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": 1.5524566173553467,
      "logits/rejected": 1.4685381650924683,
      "logps/chosen": -286.782470703125,
      "logps/ref_chosen": -275.8153076171875,
      "logps/ref_rejected": -238.54286193847656,
      "logps/rejected": -285.0547790527344,
      "loss": 4.4655,
      "margin_dpo/margin_mean": 35.54475021362305,
      "margin_dpo/margin_std": 47.85310363769531,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -11.005252838134766,
      "KL/mean": -28.47077751159668,
      "KL/rejected_KL_mean": -45.936302185058594,
      "KL/std": 40.45540237426758,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.0100031653419137,
      "fcm_dpo/delta": -0.04593458026647568,
      "fcm_dpo/margin": 34.9310417175293,
      "fcm_dpo/q_t": 0.41721823811531067,
      "grad_norm": 52.13056564331055,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": 1.2748719453811646,
      "logits/rejected": 1.3322370052337646,
      "logps/chosen": -250.10565185546875,
      "logps/ref_chosen": -239.1003875732422,
      "logps/ref_rejected": -230.31900024414062,
      "logps/rejected": -276.2552795410156,
      "loss": 4.4815,
      "margin_dpo/margin_mean": 34.9310417175293,
      "margin_dpo/margin_std": 44.63822555541992,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -18.939266204833984,
      "KL/mean": -31.239116668701172,
      "KL/rejected_KL_mean": -43.538963317871094,
      "KL/std": 41.6203727722168,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.009852276183664799,
      "fcm_dpo/delta": 0.024417586624622345,
      "fcm_dpo/margin": 24.59969139099121,
      "fcm_dpo/q_t": 0.4442344307899475,
      "grad_norm": 26.851593017578125,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": 1.575951099395752,
      "logits/rejected": 1.601134181022644,
      "logps/chosen": -319.2686462402344,
      "logps/ref_chosen": -300.3293762207031,
      "logps/ref_rejected": -332.132568359375,
      "logps/rejected": -375.6715087890625,
      "loss": 4.9255,
      "margin_dpo/margin_mean": 24.59969139099121,
      "margin_dpo/margin_std": 54.19697952270508,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -13.297246932983398,
      "KL/mean": -25.346601486206055,
      "KL/rejected_KL_mean": -37.395938873291016,
      "KL/std": 39.70291519165039,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.010138127021491528,
      "fcm_dpo/delta": 0.05284906178712845,
      "fcm_dpo/margin": 24.098691940307617,
      "fcm_dpo/q_t": 0.4428808093070984,
      "grad_norm": 30.7982177734375,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": 1.4364606142044067,
      "logits/rejected": 1.501950979232788,
      "logps/chosen": -258.70941162109375,
      "logps/ref_chosen": -245.41213989257812,
      "logps/ref_rejected": -251.72999572753906,
      "logps/rejected": -289.1259460449219,
      "loss": 4.8668,
      "margin_dpo/margin_mean": 24.09869384765625,
      "margin_dpo/margin_std": 47.61665344238281,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -12.442529678344727,
      "KL/mean": -29.97757339477539,
      "KL/rejected_KL_mean": -47.512603759765625,
      "KL/std": 42.04408264160156,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.01016856636852026,
      "fcm_dpo/delta": -0.028599146753549576,
      "fcm_dpo/margin": 35.0700798034668,
      "fcm_dpo/q_t": 0.4160389304161072,
      "grad_norm": 29.33309555053711,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": 1.2065235376358032,
      "logits/rejected": 1.241371512413025,
      "logps/chosen": -289.3183288574219,
      "logps/ref_chosen": -276.87579345703125,
      "logps/ref_rejected": -259.0455017089844,
      "logps/rejected": -306.5581359863281,
      "loss": 4.4977,
      "margin_dpo/margin_mean": 35.0700798034668,
      "margin_dpo/margin_std": 48.75973892211914,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -10.04623031616211,
      "KL/mean": -27.58026695251465,
      "KL/rejected_KL_mean": -45.11430358886719,
      "KL/std": 41.352909088134766,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.010188644751906395,
      "fcm_dpo/delta": -0.008220143616199493,
      "fcm_dpo/margin": 35.06806945800781,
      "fcm_dpo/q_t": 0.4163851737976074,
      "grad_norm": 26.867780685424805,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": 1.5530842542648315,
      "logits/rejected": 1.5565985441207886,
      "logps/chosen": -314.01727294921875,
      "logps/ref_chosen": -303.9710693359375,
      "logps/ref_rejected": -255.62033081054688,
      "logps/rejected": -300.734619140625,
      "loss": 4.4857,
      "margin_dpo/margin_mean": 35.06806945800781,
      "margin_dpo/margin_std": 47.67494583129883,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -9.413045883178711,
      "KL/mean": -32.62232208251953,
      "KL/rejected_KL_mean": -55.83159255981445,
      "KL/std": 43.81716537475586,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.00961458869278431,
      "fcm_dpo/delta": -0.10150563716888428,
      "fcm_dpo/margin": 46.418540954589844,
      "fcm_dpo/q_t": 0.3945727050304413,
      "grad_norm": 24.265979766845703,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": 1.3851195573806763,
      "logits/rejected": 1.380741834640503,
      "logps/chosen": -291.1090087890625,
      "logps/ref_chosen": -281.6959533691406,
      "logps/ref_rejected": -258.93353271484375,
      "logps/rejected": -314.76513671875,
      "loss": 4.1928,
      "margin_dpo/margin_mean": 46.418540954589844,
      "margin_dpo/margin_std": 49.15439987182617,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -11.311918258666992,
      "KL/mean": -28.9129638671875,
      "KL/rejected_KL_mean": -46.51400375366211,
      "KL/std": 42.01262283325195,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.00928244274109602,
      "fcm_dpo/delta": -0.014752980321645737,
      "fcm_dpo/margin": 35.20207977294922,
      "fcm_dpo/q_t": 0.4226464629173279,
      "grad_norm": 26.019874572753906,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 1.049330711364746,
      "logits/rejected": 1.132533311843872,
      "logps/chosen": -235.26846313476562,
      "logps/ref_chosen": -223.95655822753906,
      "logps/ref_rejected": -238.95187377929688,
      "logps/rejected": -285.46588134765625,
      "loss": 4.5761,
      "margin_dpo/margin_mean": 35.20207977294922,
      "margin_dpo/margin_std": 50.57219314575195,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -17.906423568725586,
      "KL/mean": -32.054283142089844,
      "KL/rejected_KL_mean": -46.2021484375,
      "KL/std": 41.659996032714844,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.009253242053091526,
      "fcm_dpo/delta": 0.01831619255244732,
      "fcm_dpo/margin": 28.295734405517578,
      "fcm_dpo/q_t": 0.4376838207244873,
      "grad_norm": 29.27717399597168,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 1.3871787786483765,
      "logits/rejected": 1.4612462520599365,
      "logps/chosen": -332.8310241699219,
      "logps/ref_chosen": -314.92462158203125,
      "logps/ref_rejected": -279.6347351074219,
      "logps/rejected": -325.8368835449219,
      "loss": 4.7545,
      "margin_dpo/margin_mean": 28.295738220214844,
      "margin_dpo/margin_std": 46.122982025146484,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -10.566184997558594,
      "KL/mean": -26.787403106689453,
      "KL/rejected_KL_mean": -43.008628845214844,
      "KL/std": 43.76570129394531,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.009324965067207813,
      "fcm_dpo/delta": 0.008656857535243034,
      "fcm_dpo/margin": 32.44244384765625,
      "fcm_dpo/q_t": 0.42886555194854736,
      "grad_norm": 24.10577392578125,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": 1.208914041519165,
      "logits/rejected": 1.2646691799163818,
      "logps/chosen": -273.8857727050781,
      "logps/ref_chosen": -263.3196105957031,
      "logps/ref_rejected": -247.13612365722656,
      "logps/rejected": -290.1447448730469,
      "loss": 4.6441,
      "margin_dpo/margin_mean": 32.44244384765625,
      "margin_dpo/margin_std": 48.81672286987305,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -9.526803970336914,
      "KL/mean": -26.895198822021484,
      "KL/rejected_KL_mean": -44.26359558105469,
      "KL/std": 42.79859161376953,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.00938683282583952,
      "fcm_dpo/delta": 0.0017264587804675102,
      "fcm_dpo/margin": 34.736785888671875,
      "fcm_dpo/q_t": 0.42343154549598694,
      "grad_norm": 28.52617645263672,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": 1.343544363975525,
      "logits/rejected": 1.4157849550247192,
      "logps/chosen": -288.8890075683594,
      "logps/ref_chosen": -279.36224365234375,
      "logps/ref_rejected": -264.46112060546875,
      "logps/rejected": -308.7247314453125,
      "loss": 4.5635,
      "margin_dpo/margin_mean": 34.73678207397461,
      "margin_dpo/margin_std": 49.0056037902832,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -9.335607528686523,
      "KL/mean": -26.9256591796875,
      "KL/rejected_KL_mean": -44.515708923339844,
      "KL/std": 45.046573638916016,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.009366855956614017,
      "fcm_dpo/delta": -0.007984668016433716,
      "fcm_dpo/margin": 35.18009948730469,
      "fcm_dpo/q_t": 0.42302170395851135,
      "grad_norm": 27.110992431640625,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": 1.2010538578033447,
      "logits/rejected": 1.2763432264328003,
      "logps/chosen": -295.78472900390625,
      "logps/ref_chosen": -286.4491271972656,
      "logps/ref_rejected": -294.5469665527344,
      "logps/rejected": -339.06268310546875,
      "loss": 4.6291,
      "margin_dpo/margin_mean": 35.18009948730469,
      "margin_dpo/margin_std": 56.73601531982422,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -11.618865966796875,
      "KL/mean": -28.07540512084961,
      "KL/rejected_KL_mean": -44.531944274902344,
      "KL/std": 45.744510650634766,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.009503568522632122,
      "fcm_dpo/delta": 0.003584356978535652,
      "fcm_dpo/margin": 32.9130744934082,
      "fcm_dpo/q_t": 0.4265667498111725,
      "grad_norm": 58.758182525634766,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": 1.4884653091430664,
      "logits/rejected": 1.5040245056152344,
      "logps/chosen": -293.97894287109375,
      "logps/ref_chosen": -282.36004638671875,
      "logps/ref_rejected": -243.79052734375,
      "logps/rejected": -288.32244873046875,
      "loss": 4.6582,
      "margin_dpo/margin_mean": 32.9130744934082,
      "margin_dpo/margin_std": 53.42784118652344,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -6.206565856933594,
      "KL/mean": -24.582725524902344,
      "KL/rejected_KL_mean": -42.958885192871094,
      "KL/std": 43.78059387207031,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.009410029277205467,
      "fcm_dpo/delta": -0.013622024096548557,
      "fcm_dpo/margin": 36.752315521240234,
      "fcm_dpo/q_t": 0.41831323504447937,
      "grad_norm": 27.63625717163086,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": 1.2264595031738281,
      "logits/rejected": 1.2288726568222046,
      "logps/chosen": -285.3459777832031,
      "logps/ref_chosen": -279.139404296875,
      "logps/ref_rejected": -240.5850067138672,
      "logps/rejected": -283.54388427734375,
      "loss": 4.5203,
      "margin_dpo/margin_mean": 36.752315521240234,
      "margin_dpo/margin_std": 51.104835510253906,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -9.57513427734375,
      "KL/mean": -25.410507202148438,
      "KL/rejected_KL_mean": -41.24588394165039,
      "KL/std": 40.79557800292969,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.009222055785357952,
      "fcm_dpo/delta": -0.013315379619598389,
      "fcm_dpo/margin": 31.670761108398438,
      "fcm_dpo/q_t": 0.4315035939216614,
      "grad_norm": 27.917341232299805,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": 1.516631007194519,
      "logits/rejected": 1.5989162921905518,
      "logps/chosen": -268.38665771484375,
      "logps/ref_chosen": -258.8114929199219,
      "logps/ref_rejected": -266.3898010253906,
      "logps/rejected": -307.63568115234375,
      "loss": 4.6921,
      "margin_dpo/margin_mean": 31.670761108398438,
      "margin_dpo/margin_std": 51.152652740478516,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -9.69536018371582,
      "KL/mean": -27.4605770111084,
      "KL/rejected_KL_mean": -45.22578811645508,
      "KL/std": 45.36590576171875,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.009020301513373852,
      "fcm_dpo/delta": -0.019452054053544998,
      "fcm_dpo/margin": 35.530426025390625,
      "fcm_dpo/q_t": 0.4251117706298828,
      "grad_norm": 28.10435676574707,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": 1.361495852470398,
      "logits/rejected": 1.2905851602554321,
      "logps/chosen": -311.7615661621094,
      "logps/ref_chosen": -302.0661926269531,
      "logps/ref_rejected": -265.5518798828125,
      "logps/rejected": -310.77764892578125,
      "loss": 4.6025,
      "margin_dpo/margin_mean": 35.53042221069336,
      "margin_dpo/margin_std": 52.67932891845703,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -5.742082595825195,
      "KL/mean": -20.897375106811523,
      "KL/rejected_KL_mean": -36.05267333984375,
      "KL/std": 44.31733322143555,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.009107161313295364,
      "fcm_dpo/delta": 0.03577272593975067,
      "fcm_dpo/margin": 30.310583114624023,
      "fcm_dpo/q_t": 0.43593236804008484,
      "grad_norm": 27.373260498046875,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": 1.5845272541046143,
      "logits/rejected": 1.693718671798706,
      "logps/chosen": -264.74041748046875,
      "logps/ref_chosen": -258.99835205078125,
      "logps/ref_rejected": -273.7834777832031,
      "logps/rejected": -309.836181640625,
      "loss": 4.779,
      "margin_dpo/margin_mean": 30.310585021972656,
      "margin_dpo/margin_std": 56.40108871459961,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -3.5544681549072266,
      "KL/mean": -22.997289657592773,
      "KL/rejected_KL_mean": -42.44010925292969,
      "KL/std": 43.98283004760742,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.009086307138204575,
      "fcm_dpo/delta": -0.032155584543943405,
      "fcm_dpo/margin": 38.88564682006836,
      "fcm_dpo/q_t": 0.4165016710758209,
      "grad_norm": 33.51439666748047,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 1.2379156351089478,
      "logits/rejected": 1.3163321018218994,
      "logps/chosen": -279.0775146484375,
      "logps/ref_chosen": -275.5230407714844,
      "logps/ref_rejected": -255.50784301757812,
      "logps/rejected": -297.9479675292969,
      "loss": 4.4702,
      "margin_dpo/margin_mean": 38.88564682006836,
      "margin_dpo/margin_std": 50.45158767700195,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -9.96323013305664,
      "KL/mean": -24.66914176940918,
      "KL/rejected_KL_mean": -39.375057220458984,
      "KL/std": 44.939361572265625,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.009113289415836334,
      "fcm_dpo/delta": 0.026187503710389137,
      "fcm_dpo/margin": 29.411834716796875,
      "fcm_dpo/q_t": 0.43763625621795654,
      "grad_norm": 27.15723419189453,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.4474784135818481,
      "logits/rejected": 1.4950228929519653,
      "logps/chosen": -263.32672119140625,
      "logps/ref_chosen": -253.36349487304688,
      "logps/ref_rejected": -247.1916961669922,
      "logps/rejected": -286.5667419433594,
      "loss": 4.8048,
      "margin_dpo/margin_mean": 29.411834716796875,
      "margin_dpo/margin_std": 55.104427337646484,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -7.69102668762207,
      "KL/mean": -25.5341796875,
      "KL/rejected_KL_mean": -43.37733459472656,
      "KL/std": 44.129642486572266,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.009203766472637653,
      "fcm_dpo/delta": -0.0036179821472615004,
      "fcm_dpo/margin": 35.68629837036133,
      "fcm_dpo/q_t": 0.4221925139427185,
      "grad_norm": 26.887231826782227,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": 1.424123764038086,
      "logits/rejected": 1.367086410522461,
      "logps/chosen": -272.6791076660156,
      "logps/ref_chosen": -264.9880676269531,
      "logps/ref_rejected": -243.07838439941406,
      "logps/rejected": -286.4556884765625,
      "loss": 4.5457,
      "margin_dpo/margin_mean": 35.68629837036133,
      "margin_dpo/margin_std": 48.56901168823242,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -2.119577407836914,
      "KL/mean": -23.20427703857422,
      "KL/rejected_KL_mean": -44.288978576660156,
      "KL/std": 43.80159378051758,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.009071076288819313,
      "fcm_dpo/delta": -0.06056726723909378,
      "fcm_dpo/margin": 42.16940689086914,
      "fcm_dpo/q_t": 0.4097389578819275,
      "grad_norm": 54.67237854003906,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": 1.5834507942199707,
      "logits/rejected": 1.6539833545684814,
      "logps/chosen": -317.990234375,
      "logps/ref_chosen": -315.8706970214844,
      "logps/ref_rejected": -304.3729248046875,
      "logps/rejected": -348.661865234375,
      "loss": 4.3688,
      "margin_dpo/margin_mean": 42.16940689086914,
      "margin_dpo/margin_std": 46.83339309692383,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -6.7764434814453125,
      "KL/mean": -27.922348022460938,
      "KL/rejected_KL_mean": -49.06826400756836,
      "KL/std": 46.5766487121582,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.008566658943891525,
      "fcm_dpo/delta": -0.04639684408903122,
      "fcm_dpo/margin": 42.29182052612305,
      "fcm_dpo/q_t": 0.4142807126045227,
      "grad_norm": 25.08127212524414,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 1.513807773590088,
      "logits/rejected": 1.4798811674118042,
      "logps/chosen": -317.87359619140625,
      "logps/ref_chosen": -311.09716796875,
      "logps/ref_rejected": -266.01959228515625,
      "logps/rejected": -315.08782958984375,
      "loss": 4.4476,
      "margin_dpo/margin_mean": 42.29182052612305,
      "margin_dpo/margin_std": 53.406185150146484,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -6.420351028442383,
      "KL/mean": -27.76439666748047,
      "KL/rejected_KL_mean": -49.10843276977539,
      "KL/std": 46.080867767333984,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.008325492031872272,
      "fcm_dpo/delta": -0.046551793813705444,
      "fcm_dpo/margin": 42.68809127807617,
      "fcm_dpo/q_t": 0.4156304597854614,
      "grad_norm": 34.8421630859375,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": 1.4574421644210815,
      "logits/rejected": 1.3477965593338013,
      "logps/chosen": -291.16082763671875,
      "logps/ref_chosen": -284.740478515625,
      "logps/ref_rejected": -237.64117431640625,
      "logps/rejected": -286.7496032714844,
      "loss": 4.4634,
      "margin_dpo/margin_mean": 42.68809127807617,
      "margin_dpo/margin_std": 53.2493782043457,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -17.11293601989746,
      "KL/mean": -31.10993194580078,
      "KL/rejected_KL_mean": -45.106929779052734,
      "KL/std": 45.12336349487305,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.008096544072031975,
      "fcm_dpo/delta": -0.007281060330569744,
      "fcm_dpo/margin": 27.99398422241211,
      "fcm_dpo/q_t": 0.44674554467201233,
      "grad_norm": 41.784725189208984,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": 1.5147217512130737,
      "logits/rejected": 1.5465182065963745,
      "logps/chosen": -289.725830078125,
      "logps/ref_chosen": -272.6128845214844,
      "logps/ref_rejected": -249.99725341796875,
      "logps/rejected": -295.10418701171875,
      "loss": 4.9097,
      "margin_dpo/margin_mean": 27.99398422241211,
      "margin_dpo/margin_std": 55.39048767089844,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -9.308063507080078,
      "KL/mean": -27.8387508392334,
      "KL/rejected_KL_mean": -46.369449615478516,
      "KL/std": 45.95248794555664,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.008128305897116661,
      "fcm_dpo/delta": 0.002714872360229492,
      "fcm_dpo/margin": 37.0613899230957,
      "fcm_dpo/q_t": 0.4285484850406647,
      "grad_norm": 25.302976608276367,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": 1.4533767700195312,
      "logits/rejected": 1.4659007787704468,
      "logps/chosen": -283.7438659667969,
      "logps/ref_chosen": -274.435791015625,
      "logps/ref_rejected": -257.8125305175781,
      "logps/rejected": -304.1819763183594,
      "loss": 4.6146,
      "margin_dpo/margin_mean": 37.0613899230957,
      "margin_dpo/margin_std": 52.04000473022461,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -14.35413932800293,
      "KL/mean": -29.608779907226562,
      "KL/rejected_KL_mean": -44.86341857910156,
      "KL/std": 45.581966400146484,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.00804627314209938,
      "fcm_dpo/delta": -0.008875997737050056,
      "fcm_dpo/margin": 30.509273529052734,
      "fcm_dpo/q_t": 0.4414757490158081,
      "grad_norm": 34.29328536987305,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": 1.290514588356018,
      "logits/rejected": 1.3305566310882568,
      "logps/chosen": -275.66546630859375,
      "logps/ref_chosen": -261.3113098144531,
      "logps/ref_rejected": -231.152587890625,
      "logps/rejected": -276.0160217285156,
      "loss": 4.8307,
      "margin_dpo/margin_mean": 30.509275436401367,
      "margin_dpo/margin_std": 54.97924041748047,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -16.31682777404785,
      "KL/mean": -33.82235336303711,
      "KL/rejected_KL_mean": -51.32787322998047,
      "KL/std": 48.20877456665039,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.008161664009094238,
      "fcm_dpo/delta": 0.028440292924642563,
      "fcm_dpo/margin": 35.011043548583984,
      "fcm_dpo/q_t": 0.4324631989002228,
      "grad_norm": 42.08315658569336,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": 1.3390055894851685,
      "logits/rejected": 1.2902629375457764,
      "logps/chosen": -284.7840576171875,
      "logps/ref_chosen": -268.46722412109375,
      "logps/ref_rejected": -240.27597045898438,
      "logps/rejected": -291.60382080078125,
      "loss": 4.6927,
      "margin_dpo/margin_mean": 35.011043548583984,
      "margin_dpo/margin_std": 54.89663314819336,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -17.609296798706055,
      "KL/mean": -35.390602111816406,
      "KL/rejected_KL_mean": -53.17192077636719,
      "KL/std": 49.72998046875,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.008215202949941158,
      "fcm_dpo/delta": 0.0022948868572711945,
      "fcm_dpo/margin": 35.56262969970703,
      "fcm_dpo/q_t": 0.43131887912750244,
      "grad_norm": 33.538795471191406,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 1.6498174667358398,
      "logits/rejected": 1.7886760234832764,
      "logps/chosen": -302.7145080566406,
      "logps/ref_chosen": -285.1051940917969,
      "logps/ref_rejected": -281.2331848144531,
      "logps/rejected": -334.4051208496094,
      "loss": 4.7111,
      "margin_dpo/margin_mean": 35.56262969970703,
      "margin_dpo/margin_std": 59.274070739746094,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -14.372152328491211,
      "KL/mean": -32.225440979003906,
      "KL/rejected_KL_mean": -50.0787353515625,
      "KL/std": 47.4111213684082,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.008327195420861244,
      "fcm_dpo/delta": -0.006058306898921728,
      "fcm_dpo/margin": 35.70658493041992,
      "fcm_dpo/q_t": 0.4304063320159912,
      "grad_norm": 30.19339370727539,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 1.4011435508728027,
      "logits/rejected": 1.3594019412994385,
      "logps/chosen": -288.4847717285156,
      "logps/ref_chosen": -274.1126403808594,
      "logps/ref_rejected": -256.7730407714844,
      "logps/rejected": -306.8517761230469,
      "loss": 4.66,
      "margin_dpo/margin_mean": 35.70658874511719,
      "margin_dpo/margin_std": 53.72241973876953,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -21.507081985473633,
      "KL/mean": -41.127342224121094,
      "KL/rejected_KL_mean": -60.74760055541992,
      "KL/std": 49.66435623168945,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.008040564134716988,
      "fcm_dpo/delta": -0.021318240091204643,
      "fcm_dpo/margin": 39.24053192138672,
      "fcm_dpo/q_t": 0.4255543351173401,
      "grad_norm": 25.63070297241211,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": 1.3852585554122925,
      "logits/rejected": 1.3525639772415161,
      "logps/chosen": -311.0727844238281,
      "logps/ref_chosen": -289.56573486328125,
      "logps/ref_rejected": -277.9891662597656,
      "logps/rejected": -338.73675537109375,
      "loss": 4.6379,
      "margin_dpo/margin_mean": 39.24052810668945,
      "margin_dpo/margin_std": 61.18446731567383,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -19.807077407836914,
      "KL/mean": -34.876522064208984,
      "KL/rejected_KL_mean": -49.94596481323242,
      "KL/std": 49.224517822265625,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.008184156380593777,
      "fcm_dpo/delta": 0.03253357857465744,
      "fcm_dpo/margin": 30.13889503479004,
      "fcm_dpo/q_t": 0.4421314001083374,
      "grad_norm": 30.542022705078125,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": 1.5468662977218628,
      "logits/rejected": 1.5475176572799683,
      "logps/chosen": -289.9909973144531,
      "logps/ref_chosen": -270.1839294433594,
      "logps/ref_rejected": -245.86949157714844,
      "logps/rejected": -295.8154296875,
      "loss": 4.8468,
      "margin_dpo/margin_mean": 30.138896942138672,
      "margin_dpo/margin_std": 57.33922576904297,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -15.34299087524414,
      "KL/mean": -34.77743911743164,
      "KL/rejected_KL_mean": -54.211883544921875,
      "KL/std": 49.53538513183594,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.008361018262803555,
      "fcm_dpo/delta": -0.019143983721733093,
      "fcm_dpo/margin": 38.86888122558594,
      "fcm_dpo/q_t": 0.423013299703598,
      "grad_norm": 28.539209365844727,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": 1.4679999351501465,
      "logits/rejected": 1.4704045057296753,
      "logps/chosen": -276.0294189453125,
      "logps/ref_chosen": -260.6864318847656,
      "logps/ref_rejected": -245.99508666992188,
      "logps/rejected": -300.20697021484375,
      "loss": 4.5542,
      "margin_dpo/margin_mean": 38.86888885498047,
      "margin_dpo/margin_std": 50.53099822998047,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -14.399248123168945,
      "KL/mean": -35.56013488769531,
      "KL/rejected_KL_mean": -56.72101974487305,
      "KL/std": 50.00802993774414,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.008184293285012245,
      "fcm_dpo/delta": -0.01608538255095482,
      "fcm_dpo/margin": 42.32177734375,
      "fcm_dpo/q_t": 0.41896212100982666,
      "grad_norm": 23.222185134887695,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": 1.5479485988616943,
      "logits/rejected": 1.4615684747695923,
      "logps/chosen": -312.3130798339844,
      "logps/ref_chosen": -297.9138488769531,
      "logps/ref_rejected": -253.39952087402344,
      "logps/rejected": -310.1205139160156,
      "loss": 4.5463,
      "margin_dpo/margin_mean": 42.32177734375,
      "margin_dpo/margin_std": 63.10238265991211,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -12.57583999633789,
      "KL/mean": -34.11717987060547,
      "KL/rejected_KL_mean": -55.65851974487305,
      "KL/std": 50.8270149230957,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.008083254098892212,
      "fcm_dpo/delta": -0.021567806601524353,
      "fcm_dpo/margin": 43.082672119140625,
      "fcm_dpo/q_t": 0.41805118322372437,
      "grad_norm": 37.054935455322266,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": 1.369890570640564,
      "logits/rejected": 1.32097589969635,
      "logps/chosen": -322.5022277832031,
      "logps/ref_chosen": -309.9263916015625,
      "logps/ref_rejected": -253.452392578125,
      "logps/rejected": -309.1109313964844,
      "loss": 4.5599,
      "margin_dpo/margin_mean": 43.082672119140625,
      "margin_dpo/margin_std": 63.74726867675781,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -16.547395706176758,
      "KL/mean": -36.08913040161133,
      "KL/rejected_KL_mean": -55.63086700439453,
      "KL/std": 50.99491500854492,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.007843622006475925,
      "fcm_dpo/delta": -0.017665142193436623,
      "fcm_dpo/margin": 39.083473205566406,
      "fcm_dpo/q_t": 0.42786192893981934,
      "grad_norm": 34.529075622558594,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 1.2697227001190186,
      "logits/rejected": 1.3054670095443726,
      "logps/chosen": -276.36102294921875,
      "logps/ref_chosen": -259.8136291503906,
      "logps/ref_rejected": -245.84947204589844,
      "logps/rejected": -301.4803466796875,
      "loss": 4.6232,
      "margin_dpo/margin_mean": 39.083473205566406,
      "margin_dpo/margin_std": 57.39265060424805,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -13.858589172363281,
      "KL/mean": -35.44044494628906,
      "KL/rejected_KL_mean": -57.02231216430664,
      "KL/std": 51.46019744873047,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.007777026854455471,
      "fcm_dpo/delta": 0.01463997270911932,
      "fcm_dpo/margin": 43.163719177246094,
      "fcm_dpo/q_t": 0.42022788524627686,
      "grad_norm": 34.798580169677734,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": 1.4866372346878052,
      "logits/rejected": 1.5739963054656982,
      "logps/chosen": -279.2122802734375,
      "logps/ref_chosen": -265.35369873046875,
      "logps/ref_rejected": -247.9174346923828,
      "logps/rejected": -304.9397277832031,
      "loss": 4.4969,
      "margin_dpo/margin_mean": 43.163719177246094,
      "margin_dpo/margin_std": 54.083045959472656,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -13.254034042358398,
      "KL/mean": -32.92533493041992,
      "KL/rejected_KL_mean": -52.59662628173828,
      "KL/std": 50.64497375488281,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.007970641367137432,
      "fcm_dpo/delta": 0.01746588572859764,
      "fcm_dpo/margin": 39.34259796142578,
      "fcm_dpo/q_t": 0.42596715688705444,
      "grad_norm": 37.043338775634766,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": 1.453195333480835,
      "logits/rejected": 1.4561303853988647,
      "logps/chosen": -301.4772644042969,
      "logps/ref_chosen": -288.2232360839844,
      "logps/ref_rejected": -270.7300109863281,
      "logps/rejected": -323.32666015625,
      "loss": 4.6078,
      "margin_dpo/margin_mean": 39.342594146728516,
      "margin_dpo/margin_std": 58.970062255859375,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -18.382680892944336,
      "KL/mean": -27.45463752746582,
      "KL/rejected_KL_mean": -36.5265998840332,
      "KL/std": 49.039588928222656,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.008133959025144577,
      "fcm_dpo/delta": 0.021044978871941566,
      "fcm_dpo/margin": 18.143917083740234,
      "fcm_dpo/q_t": 0.46542131900787354,
      "grad_norm": 37.91532897949219,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 1.3853825330734253,
      "logits/rejected": 1.426079511642456,
      "logps/chosen": -300.0024108886719,
      "logps/ref_chosen": -281.61968994140625,
      "logps/ref_rejected": -271.03973388671875,
      "logps/rejected": -307.56634521484375,
      "loss": 5.2073,
      "margin_dpo/margin_mean": 18.143917083740234,
      "margin_dpo/margin_std": 57.807655334472656,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -23.00128746032715,
      "KL/mean": -39.980648040771484,
      "KL/rejected_KL_mean": -56.96001052856445,
      "KL/std": 50.98291015625,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.008176068775355816,
      "fcm_dpo/delta": 0.02054738998413086,
      "fcm_dpo/margin": 33.95871353149414,
      "fcm_dpo/q_t": 0.43500789999961853,
      "grad_norm": 30.975357055664062,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 1.1824455261230469,
      "logits/rejected": 1.3351492881774902,
      "logps/chosen": -265.4644775390625,
      "logps/ref_chosen": -242.4631805419922,
      "logps/ref_rejected": -244.0020751953125,
      "logps/rejected": -300.96209716796875,
      "loss": 4.7799,
      "margin_dpo/margin_mean": 33.958717346191406,
      "margin_dpo/margin_std": 62.828582763671875,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -14.076541900634766,
      "KL/mean": -35.360504150390625,
      "KL/rejected_KL_mean": -56.644474029541016,
      "KL/std": 51.975372314453125,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.008335628546774387,
      "fcm_dpo/delta": -0.005399887450039387,
      "fcm_dpo/margin": 42.56793212890625,
      "fcm_dpo/q_t": 0.4167358875274658,
      "grad_norm": 30.83401107788086,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": 1.3769159317016602,
      "logits/rejected": 1.3988920450210571,
      "logps/chosen": -306.0651550292969,
      "logps/ref_chosen": -291.98858642578125,
      "logps/ref_rejected": -273.35955810546875,
      "logps/rejected": -330.0040588378906,
      "loss": 4.4754,
      "margin_dpo/margin_mean": 42.567928314208984,
      "margin_dpo/margin_std": 56.71961975097656,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -22.119665145874023,
      "KL/mean": -40.024654388427734,
      "KL/rejected_KL_mean": -57.929649353027344,
      "KL/std": 52.6969108581543,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.008227887563407421,
      "fcm_dpo/delta": -0.005060448311269283,
      "fcm_dpo/margin": 35.80997848510742,
      "fcm_dpo/q_t": 0.4311443567276001,
      "grad_norm": 36.8986701965332,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": 1.210425615310669,
      "logits/rejected": 1.1882951259613037,
      "logps/chosen": -297.1081237792969,
      "logps/ref_chosen": -274.98846435546875,
      "logps/ref_rejected": -254.75782775878906,
      "logps/rejected": -312.6875,
      "loss": 4.7026,
      "margin_dpo/margin_mean": 35.809974670410156,
      "margin_dpo/margin_std": 59.73834228515625,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -12.145683288574219,
      "KL/mean": -34.42516326904297,
      "KL/rejected_KL_mean": -56.70464324951172,
      "KL/std": 51.965606689453125,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.008038321509957314,
      "fcm_dpo/delta": -0.03860735520720482,
      "fcm_dpo/margin": 44.5589599609375,
      "fcm_dpo/q_t": 0.41594168543815613,
      "grad_norm": 26.068283081054688,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": 1.1392879486083984,
      "logits/rejected": 1.2066115140914917,
      "logps/chosen": -263.7718505859375,
      "logps/ref_chosen": -251.62615966796875,
      "logps/ref_rejected": -251.9549560546875,
      "logps/rejected": -308.65960693359375,
      "loss": 4.4551,
      "margin_dpo/margin_mean": 44.5589599609375,
      "margin_dpo/margin_std": 57.255672454833984,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -18.578876495361328,
      "KL/mean": -39.75908660888672,
      "KL/rejected_KL_mean": -60.939300537109375,
      "KL/std": 54.34898376464844,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.007775822654366493,
      "fcm_dpo/delta": -0.025865402072668076,
      "fcm_dpo/margin": 42.360435485839844,
      "fcm_dpo/q_t": 0.4223623275756836,
      "grad_norm": 31.938304901123047,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": 1.2756521701812744,
      "logits/rejected": 1.3228347301483154,
      "logps/chosen": -296.5155029296875,
      "logps/ref_chosen": -277.9366149902344,
      "logps/ref_rejected": -262.689453125,
      "logps/rejected": -323.6287536621094,
      "loss": 4.5659,
      "margin_dpo/margin_mean": 42.360435485839844,
      "margin_dpo/margin_std": 60.44560241699219,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -9.219978332519531,
      "KL/mean": -30.533493041992188,
      "KL/rejected_KL_mean": -51.847015380859375,
      "KL/std": 51.99962615966797,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.0078043402172625065,
      "fcm_dpo/delta": -0.01406799629330635,
      "fcm_dpo/margin": 42.627052307128906,
      "fcm_dpo/q_t": 0.42097949981689453,
      "grad_norm": 38.614402770996094,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": 1.5329761505126953,
      "logits/rejected": 1.5293965339660645,
      "logps/chosen": -279.6648864746094,
      "logps/ref_chosen": -270.4449157714844,
      "logps/ref_rejected": -259.5942077636719,
      "logps/rejected": -311.4412536621094,
      "loss": 4.5365,
      "margin_dpo/margin_mean": 42.627052307128906,
      "margin_dpo/margin_std": 56.017112731933594,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -17.516124725341797,
      "KL/mean": -35.28583908081055,
      "KL/rejected_KL_mean": -53.055564880371094,
      "KL/std": 53.478023529052734,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.007699913810938597,
      "fcm_dpo/delta": 0.005450897850096226,
      "fcm_dpo/margin": 35.539432525634766,
      "fcm_dpo/q_t": 0.43559616804122925,
      "grad_norm": 36.26092529296875,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": 1.3301708698272705,
      "logits/rejected": 1.4120111465454102,
      "logps/chosen": -287.35601806640625,
      "logps/ref_chosen": -269.83990478515625,
      "logps/ref_rejected": -246.33746337890625,
      "logps/rejected": -299.39300537109375,
      "loss": 4.7849,
      "margin_dpo/margin_mean": 35.539432525634766,
      "margin_dpo/margin_std": 65.43788146972656,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -12.048959732055664,
      "KL/mean": -32.75706100463867,
      "KL/rejected_KL_mean": -53.46516418457031,
      "KL/std": 52.84770965576172,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.007740322966128588,
      "fcm_dpo/delta": -0.0016885926015675068,
      "fcm_dpo/margin": 41.416194915771484,
      "fcm_dpo/q_t": 0.4247538149356842,
      "grad_norm": 43.43336868286133,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": 1.1642755270004272,
      "logits/rejected": 1.1353002786636353,
      "logps/chosen": -309.5829162597656,
      "logps/ref_chosen": -297.533935546875,
      "logps/ref_rejected": -270.3407897949219,
      "logps/rejected": -323.80596923828125,
      "loss": 4.5814,
      "margin_dpo/margin_mean": 41.416194915771484,
      "margin_dpo/margin_std": 59.4129638671875,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -16.36360740661621,
      "KL/mean": -34.944828033447266,
      "KL/rejected_KL_mean": -53.52605438232422,
      "KL/std": 53.50718307495117,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.007925559766590595,
      "fcm_dpo/delta": 0.03601551055908203,
      "fcm_dpo/margin": 37.16244125366211,
      "fcm_dpo/q_t": 0.43063509464263916,
      "grad_norm": 28.538761138916016,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": 1.107112169265747,
      "logits/rejected": 1.1485131978988647,
      "logps/chosen": -256.70245361328125,
      "logps/ref_chosen": -240.33883666992188,
      "logps/ref_rejected": -220.71717834472656,
      "logps/rejected": -274.24322509765625,
      "loss": 4.7177,
      "margin_dpo/margin_mean": 37.16244125366211,
      "margin_dpo/margin_std": 64.6333999633789,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -12.508075714111328,
      "KL/mean": -35.363121032714844,
      "KL/rejected_KL_mean": -58.218162536621094,
      "KL/std": 56.018768310546875,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.00781742762774229,
      "fcm_dpo/delta": -0.03789552301168442,
      "fcm_dpo/margin": 45.710079193115234,
      "fcm_dpo/q_t": 0.41605621576309204,
      "grad_norm": 26.576629638671875,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 1.64694082736969,
      "logits/rejected": 1.661696434020996,
      "logps/chosen": -321.40118408203125,
      "logps/ref_chosen": -308.8930969238281,
      "logps/ref_rejected": -278.47869873046875,
      "logps/rejected": -336.6968994140625,
      "loss": 4.5259,
      "margin_dpo/margin_mean": 45.71007537841797,
      "margin_dpo/margin_std": 67.13477325439453,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -13.733379364013672,
      "KL/mean": -34.902366638183594,
      "KL/rejected_KL_mean": -56.07133865356445,
      "KL/std": 52.8056526184082,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.007904794998466969,
      "fcm_dpo/delta": 0.01488709170371294,
      "fcm_dpo/margin": 42.337955474853516,
      "fcm_dpo/q_t": 0.42179206013679504,
      "grad_norm": 26.501750946044922,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.2430672645568848,
      "logits/rejected": 1.2517176866531372,
      "logps/chosen": -289.1615905761719,
      "logps/ref_chosen": -275.42822265625,
      "logps/ref_rejected": -251.50906372070312,
      "logps/rejected": -307.58038330078125,
      "loss": 4.5378,
      "margin_dpo/margin_mean": 42.33795166015625,
      "margin_dpo/margin_std": 58.434486389160156,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -12.444175720214844,
      "KL/mean": -31.635404586791992,
      "KL/rejected_KL_mean": -50.82664489746094,
      "KL/std": 52.990604400634766,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.007760303094983101,
      "fcm_dpo/delta": -0.015720102936029434,
      "fcm_dpo/margin": 38.3824577331543,
      "fcm_dpo/q_t": 0.42999643087387085,
      "grad_norm": 67.79286193847656,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": 1.540757417678833,
      "logits/rejected": 1.5696464776992798,
      "logps/chosen": -301.826904296875,
      "logps/ref_chosen": -289.38275146484375,
      "logps/ref_rejected": -274.895751953125,
      "logps/rejected": -325.7224426269531,
      "loss": 4.6742,
      "margin_dpo/margin_mean": 38.3824577331543,
      "margin_dpo/margin_std": 60.079891204833984,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -13.064891815185547,
      "KL/mean": -34.57106399536133,
      "KL/rejected_KL_mean": -56.07723617553711,
      "KL/std": 53.60449981689453,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.007559816353023052,
      "fcm_dpo/delta": -0.02358321100473404,
      "fcm_dpo/margin": 43.0123405456543,
      "fcm_dpo/q_t": 0.4236932396888733,
      "grad_norm": 30.624635696411133,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": 1.2134649753570557,
      "logits/rejected": 1.3226889371871948,
      "logps/chosen": -298.0533447265625,
      "logps/ref_chosen": -284.98846435546875,
      "logps/ref_rejected": -273.5077819824219,
      "logps/rejected": -329.58502197265625,
      "loss": 4.5951,
      "margin_dpo/margin_mean": 43.0123405456543,
      "margin_dpo/margin_std": 64.62885284423828,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -13.977079391479492,
      "KL/mean": -36.888465881347656,
      "KL/rejected_KL_mean": -59.79985427856445,
      "KL/std": 56.31623077392578,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.00738301919773221,
      "fcm_dpo/delta": -0.04748978465795517,
      "fcm_dpo/margin": 45.82276916503906,
      "fcm_dpo/q_t": 0.4198259115219116,
      "grad_norm": 35.57650375366211,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": 1.3637560606002808,
      "logits/rejected": 1.4287388324737549,
      "logps/chosen": -276.6566467285156,
      "logps/ref_chosen": -262.6795959472656,
      "logps/ref_rejected": -236.13792419433594,
      "logps/rejected": -295.9377746582031,
      "loss": 4.5283,
      "margin_dpo/margin_mean": 45.82277297973633,
      "margin_dpo/margin_std": 62.66472244262695,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -9.125805854797363,
      "KL/mean": -35.98506164550781,
      "KL/rejected_KL_mean": -62.84431457519531,
      "KL/std": 54.24895477294922,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.007173227611929178,
      "fcm_dpo/delta": -0.03702869266271591,
      "fcm_dpo/margin": 53.7185173034668,
      "fcm_dpo/q_t": 0.4086601436138153,
      "grad_norm": 24.493165969848633,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": 0.9603291749954224,
      "logits/rejected": 1.0559853315353394,
      "logps/chosen": -255.52532958984375,
      "logps/ref_chosen": -246.39950561523438,
      "logps/ref_rejected": -247.4836883544922,
      "logps/rejected": -310.3280029296875,
      "loss": 4.3468,
      "margin_dpo/margin_mean": 53.7185173034668,
      "margin_dpo/margin_std": 61.33988571166992,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -6.06883430480957,
      "KL/mean": -31.070690155029297,
      "KL/rejected_KL_mean": -56.072547912597656,
      "KL/std": 53.70794677734375,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.006930915173143148,
      "fcm_dpo/delta": -0.01743871159851551,
      "fcm_dpo/margin": 50.00370407104492,
      "fcm_dpo/q_t": 0.4177793860435486,
      "grad_norm": 26.09370231628418,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": 1.612465262413025,
      "logits/rejected": 1.7407917976379395,
      "logps/chosen": -294.9888916015625,
      "logps/ref_chosen": -288.9200439453125,
      "logps/ref_rejected": -291.69305419921875,
      "logps/rejected": -347.7655944824219,
      "loss": 4.4576,
      "margin_dpo/margin_mean": 50.00370407104492,
      "margin_dpo/margin_std": 60.121192932128906,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -17.49423599243164,
      "KL/mean": -35.93175506591797,
      "KL/rejected_KL_mean": -54.36928176879883,
      "KL/std": 54.084617614746094,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.0069658225402235985,
      "fcm_dpo/delta": -0.0002794163301587105,
      "fcm_dpo/margin": 36.87504577636719,
      "fcm_dpo/q_t": 0.438935786485672,
      "grad_norm": 41.52362823486328,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": 1.1239932775497437,
      "logits/rejected": 1.1675801277160645,
      "logps/chosen": -274.5791320800781,
      "logps/ref_chosen": -257.08489990234375,
      "logps/ref_rejected": -251.61532592773438,
      "logps/rejected": -305.984619140625,
      "loss": 4.8199,
      "margin_dpo/margin_mean": 36.87504959106445,
      "margin_dpo/margin_std": 68.62469482421875,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -23.4544620513916,
      "KL/mean": -39.10903549194336,
      "KL/rejected_KL_mean": -54.763614654541016,
      "KL/std": 56.21195983886719,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.007018493954092264,
      "fcm_dpo/delta": 0.01935981586575508,
      "fcm_dpo/margin": 31.309158325195312,
      "fcm_dpo/q_t": 0.44825783371925354,
      "grad_norm": 29.68076515197754,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": 1.321920394897461,
      "logits/rejected": 1.2831867933273315,
      "logps/chosen": -284.1958312988281,
      "logps/ref_chosen": -260.7413635253906,
      "logps/ref_rejected": -227.4325408935547,
      "logps/rejected": -282.1961669921875,
      "loss": 4.9723,
      "margin_dpo/margin_mean": 31.309158325195312,
      "margin_dpo/margin_std": 69.76156616210938,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -14.656320571899414,
      "KL/mean": -39.641693115234375,
      "KL/rejected_KL_mean": -64.6270751953125,
      "KL/std": 58.25289535522461,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.006963942665606737,
      "fcm_dpo/delta": -0.023014746606349945,
      "fcm_dpo/margin": 49.97075271606445,
      "fcm_dpo/q_t": 0.41770339012145996,
      "grad_norm": 28.83690071105957,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": 1.0891973972320557,
      "logits/rejected": 1.1760591268539429,
      "logps/chosen": -317.6038818359375,
      "logps/ref_chosen": -302.947509765625,
      "logps/ref_rejected": -294.1771545410156,
      "logps/rejected": -358.80426025390625,
      "loss": 4.4742,
      "margin_dpo/margin_mean": 49.97075271606445,
      "margin_dpo/margin_std": 63.19890594482422,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -14.967306137084961,
      "KL/mean": -37.44441223144531,
      "KL/rejected_KL_mean": -59.921531677246094,
      "KL/std": 54.44309997558594,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.006940196733921766,
      "fcm_dpo/delta": -0.007086487486958504,
      "fcm_dpo/margin": 44.9542350769043,
      "fcm_dpo/q_t": 0.4263109564781189,
      "grad_norm": 28.677061080932617,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 1.298783779144287,
      "logits/rejected": 1.3807445764541626,
      "logps/chosen": -272.8653869628906,
      "logps/ref_chosen": -257.8981018066406,
      "logps/ref_rejected": -244.9084014892578,
      "logps/rejected": -304.8299255371094,
      "loss": 4.6132,
      "margin_dpo/margin_mean": 44.9542350769043,
      "margin_dpo/margin_std": 66.73704528808594,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -25.362091064453125,
      "KL/mean": -47.759178161621094,
      "KL/rejected_KL_mean": -70.15626525878906,
      "KL/std": 58.21725845336914,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.0069414544850587845,
      "fcm_dpo/delta": 0.013901254162192345,
      "fcm_dpo/margin": 44.79418182373047,
      "fcm_dpo/q_t": 0.426907479763031,
      "grad_norm": 29.31087303161621,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 1.1630300283432007,
      "logits/rejected": 1.223859190940857,
      "logps/chosen": -315.5567626953125,
      "logps/ref_chosen": -290.1946716308594,
      "logps/ref_rejected": -290.98883056640625,
      "logps/rejected": -361.14508056640625,
      "loss": 4.5963,
      "margin_dpo/margin_mean": 44.79418182373047,
      "margin_dpo/margin_std": 61.97891616821289,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -23.467235565185547,
      "KL/mean": -44.098331451416016,
      "KL/rejected_KL_mean": -64.72943115234375,
      "KL/std": 58.591026306152344,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.006948751863092184,
      "fcm_dpo/delta": -0.007603822275996208,
      "fcm_dpo/margin": 41.26219940185547,
      "fcm_dpo/q_t": 0.4327255189418793,
      "grad_norm": 44.2260627746582,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": 1.2628419399261475,
      "logits/rejected": 1.2755869626998901,
      "logps/chosen": -332.3846435546875,
      "logps/ref_chosen": -308.91741943359375,
      "logps/ref_rejected": -272.2177734375,
      "logps/rejected": -336.94720458984375,
      "loss": 4.7278,
      "margin_dpo/margin_mean": 41.26219940185547,
      "margin_dpo/margin_std": 70.71784973144531,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -27.019088745117188,
      "KL/mean": -49.432090759277344,
      "KL/rejected_KL_mean": -71.84510803222656,
      "KL/std": 55.73960876464844,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.0069027189165353775,
      "fcm_dpo/delta": -0.022060176357626915,
      "fcm_dpo/margin": 44.82601547241211,
      "fcm_dpo/q_t": 0.4269099533557892,
      "grad_norm": 24.90456199645996,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 1.628061056137085,
      "logits/rejected": 1.705552101135254,
      "logps/chosen": -275.0411071777344,
      "logps/ref_chosen": -248.02200317382812,
      "logps/ref_rejected": -234.78541564941406,
      "logps/rejected": -306.63055419921875,
      "loss": 4.626,
      "margin_dpo/margin_mean": 44.82601547241211,
      "margin_dpo/margin_std": 66.48298645019531,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -28.33185577392578,
      "KL/mean": -50.24345016479492,
      "KL/rejected_KL_mean": -72.15504455566406,
      "KL/std": 60.20033645629883,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.006803111173212528,
      "fcm_dpo/delta": 0.007166975177824497,
      "fcm_dpo/margin": 43.823184967041016,
      "fcm_dpo/q_t": 0.4303523302078247,
      "grad_norm": 40.75434494018555,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": 1.2160999774932861,
      "logits/rejected": 1.2167224884033203,
      "logps/chosen": -292.82110595703125,
      "logps/ref_chosen": -264.4892578125,
      "logps/ref_rejected": -255.43092346191406,
      "logps/rejected": -327.58599853515625,
      "loss": 4.7216,
      "margin_dpo/margin_mean": 43.823184967041016,
      "margin_dpo/margin_std": 76.38203430175781,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -26.796810150146484,
      "KL/mean": -44.10403823852539,
      "KL/rejected_KL_mean": -61.41126251220703,
      "KL/std": 55.90229797363281,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.006924143061041832,
      "fcm_dpo/delta": -0.0020297749433666468,
      "fcm_dpo/margin": 34.614463806152344,
      "fcm_dpo/q_t": 0.4431176781654358,
      "grad_norm": 48.845001220703125,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": 1.5354719161987305,
      "logits/rejected": 1.5582504272460938,
      "logps/chosen": -328.2655029296875,
      "logps/ref_chosen": -301.46868896484375,
      "logps/ref_rejected": -274.2380676269531,
      "logps/rejected": -335.64935302734375,
      "loss": 4.8579,
      "margin_dpo/margin_mean": 34.614463806152344,
      "margin_dpo/margin_std": 63.19735336303711,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -32.05453109741211,
      "KL/mean": -49.327964782714844,
      "KL/rejected_KL_mean": -66.60139465332031,
      "KL/std": 56.85910415649414,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.00707277562469244,
      "fcm_dpo/delta": 0.06271536648273468,
      "fcm_dpo/margin": 34.54685974121094,
      "fcm_dpo/q_t": 0.4431869387626648,
      "grad_norm": 31.606082916259766,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": 1.4152065515518188,
      "logits/rejected": 1.4930853843688965,
      "logps/chosen": -298.61358642578125,
      "logps/ref_chosen": -266.5590515136719,
      "logps/ref_rejected": -280.12066650390625,
      "logps/rejected": -346.7220764160156,
      "loss": 4.8852,
      "margin_dpo/margin_mean": 34.54685974121094,
      "margin_dpo/margin_std": 71.22084045410156,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -27.66551971435547,
      "KL/mean": -47.93730926513672,
      "KL/rejected_KL_mean": -68.20911407470703,
      "KL/std": 54.153778076171875,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.007219557184726,
      "fcm_dpo/delta": 0.005961534101516008,
      "fcm_dpo/margin": 40.5435905456543,
      "fcm_dpo/q_t": 0.4306770861148834,
      "grad_norm": 44.460689544677734,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": 1.2364251613616943,
      "logits/rejected": 1.2954843044281006,
      "logps/chosen": -297.60186767578125,
      "logps/ref_chosen": -269.93634033203125,
      "logps/ref_rejected": -232.32330322265625,
      "logps/rejected": -300.53240966796875,
      "loss": 4.7062,
      "margin_dpo/margin_mean": 40.5435905456543,
      "margin_dpo/margin_std": 67.83841705322266,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -28.250913619995117,
      "KL/mean": -48.43796157836914,
      "KL/rejected_KL_mean": -68.62500762939453,
      "KL/std": 57.301856994628906,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.0073934514075517654,
      "fcm_dpo/delta": 0.008817121386528015,
      "fcm_dpo/margin": 40.37410354614258,
      "fcm_dpo/q_t": 0.4305746853351593,
      "grad_norm": 50.19603729248047,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": 1.2935190200805664,
      "logits/rejected": 1.249834418296814,
      "logps/chosen": -312.2964172363281,
      "logps/ref_chosen": -284.04547119140625,
      "logps/ref_rejected": -261.19378662109375,
      "logps/rejected": -329.8188171386719,
      "loss": 4.6954,
      "margin_dpo/margin_mean": 40.37410354614258,
      "margin_dpo/margin_std": 67.88856506347656,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -36.108951568603516,
      "KL/mean": -50.618778228759766,
      "KL/rejected_KL_mean": -65.12860107421875,
      "KL/std": 58.852088928222656,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.007400644011795521,
      "fcm_dpo/delta": 0.0379965677857399,
      "fcm_dpo/margin": 29.0196590423584,
      "fcm_dpo/q_t": 0.4496317505836487,
      "grad_norm": 26.830270767211914,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": 1.4068999290466309,
      "logits/rejected": 1.3869056701660156,
      "logps/chosen": -359.39019775390625,
      "logps/ref_chosen": -323.28125,
      "logps/ref_rejected": -274.2093200683594,
      "logps/rejected": -339.33795166015625,
      "loss": 5.0029,
      "margin_dpo/margin_mean": 29.019657135009766,
      "margin_dpo/margin_std": 69.8228530883789,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -28.93212127685547,
      "KL/mean": -46.7961311340332,
      "KL/rejected_KL_mean": -64.66014099121094,
      "KL/std": 57.06940841674805,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.007479534018784761,
      "fcm_dpo/delta": 0.007990166544914246,
      "fcm_dpo/margin": 35.72800827026367,
      "fcm_dpo/q_t": 0.437507301568985,
      "grad_norm": 44.01280212402344,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 1.1369706392288208,
      "logits/rejected": 1.0937857627868652,
      "logps/chosen": -346.36749267578125,
      "logps/ref_chosen": -317.4353942871094,
      "logps/ref_rejected": -309.32403564453125,
      "logps/rejected": -373.9841613769531,
      "loss": 4.7899,
      "margin_dpo/margin_mean": 35.72801208496094,
      "margin_dpo/margin_std": 64.40153503417969,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -26.466779708862305,
      "KL/mean": -52.29292678833008,
      "KL/rejected_KL_mean": -78.11907196044922,
      "KL/std": 62.020896911621094,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.0074017890729010105,
      "fcm_dpo/delta": -0.05687055364251137,
      "fcm_dpo/margin": 51.652286529541016,
      "fcm_dpo/q_t": 0.4103277921676636,
      "grad_norm": 26.82126808166504,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 1.2127907276153564,
      "logits/rejected": 1.3827784061431885,
      "logps/chosen": -291.97503662109375,
      "logps/ref_chosen": -265.50823974609375,
      "logps/ref_rejected": -275.1380310058594,
      "logps/rejected": -353.257080078125,
      "loss": 4.432,
      "margin_dpo/margin_mean": 51.65229034423828,
      "margin_dpo/margin_std": 69.08890533447266,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -21.15934944152832,
      "KL/mean": -42.26246643066406,
      "KL/rejected_KL_mean": -63.36557388305664,
      "KL/std": 57.55577087402344,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.00716756284236908,
      "fcm_dpo/delta": -0.0024852589704096317,
      "fcm_dpo/margin": 42.20623016357422,
      "fcm_dpo/q_t": 0.428485631942749,
      "grad_norm": 39.160423278808594,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": 1.171478509902954,
      "logits/rejected": 1.2960166931152344,
      "logps/chosen": -271.0606689453125,
      "logps/ref_chosen": -249.9013214111328,
      "logps/ref_rejected": -274.58245849609375,
      "logps/rejected": -337.94805908203125,
      "loss": 4.6581,
      "margin_dpo/margin_mean": 42.20622634887695,
      "margin_dpo/margin_std": 64.75778198242188,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -30.10618019104004,
      "KL/mean": -51.75135803222656,
      "KL/rejected_KL_mean": -73.39653015136719,
      "KL/std": 61.40587615966797,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.007188502699136734,
      "fcm_dpo/delta": -0.007652060128748417,
      "fcm_dpo/margin": 43.29034423828125,
      "fcm_dpo/q_t": 0.42725032567977905,
      "grad_norm": 54.156898498535156,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": 1.4280041456222534,
      "logits/rejected": 1.4016972780227661,
      "logps/chosen": -326.4121398925781,
      "logps/ref_chosen": -296.30596923828125,
      "logps/ref_rejected": -268.0640869140625,
      "logps/rejected": -341.4606018066406,
      "loss": 4.6406,
      "margin_dpo/margin_mean": 43.29034423828125,
      "margin_dpo/margin_std": 68.36395263671875,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -27.238351821899414,
      "KL/mean": -46.235416412353516,
      "KL/rejected_KL_mean": -65.23246765136719,
      "KL/std": 58.70721435546875,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.00720646558329463,
      "fcm_dpo/delta": 0.004427594132721424,
      "fcm_dpo/margin": 37.99412155151367,
      "fcm_dpo/q_t": 0.4357755780220032,
      "grad_norm": 58.681175231933594,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": 1.3519439697265625,
      "logits/rejected": 1.402977466583252,
      "logps/chosen": -261.121337890625,
      "logps/ref_chosen": -233.8829803466797,
      "logps/ref_rejected": -224.14334106445312,
      "logps/rejected": -289.37579345703125,
      "loss": 4.7709,
      "margin_dpo/margin_mean": 37.99412536621094,
      "margin_dpo/margin_std": 67.7559585571289,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -27.015514373779297,
      "KL/mean": -49.32575225830078,
      "KL/rejected_KL_mean": -71.635986328125,
      "KL/std": 59.171085357666016,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.007208208087831736,
      "fcm_dpo/delta": 0.011092734523117542,
      "fcm_dpo/margin": 44.620487213134766,
      "fcm_dpo/q_t": 0.4253423810005188,
      "grad_norm": 31.265945434570312,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": 1.3879395723342896,
      "logits/rejected": 1.407829761505127,
      "logps/chosen": -271.1268310546875,
      "logps/ref_chosen": -244.111328125,
      "logps/ref_rejected": -240.18373107910156,
      "logps/rejected": -311.8197021484375,
      "loss": 4.6319,
      "margin_dpo/margin_mean": 44.620487213134766,
      "margin_dpo/margin_std": 71.76419067382812,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -22.816198348999023,
      "KL/mean": -52.903900146484375,
      "KL/rejected_KL_mean": -82.99159240722656,
      "KL/std": 61.090755462646484,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.006941244937479496,
      "fcm_dpo/delta": -0.09451235085725784,
      "fcm_dpo/margin": 60.17542266845703,
      "fcm_dpo/q_t": 0.4009513258934021,
      "grad_norm": 32.76856994628906,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 1.2942861318588257,
      "logits/rejected": 1.4655033349990845,
      "logps/chosen": -307.49676513671875,
      "logps/ref_chosen": -284.6805725097656,
      "logps/ref_rejected": -282.1923828125,
      "logps/rejected": -365.1839599609375,
      "loss": 4.2914,
      "margin_dpo/margin_mean": 60.17542266845703,
      "margin_dpo/margin_std": 70.85232543945312,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -25.92462158203125,
      "KL/mean": -49.15663528442383,
      "KL/rejected_KL_mean": -72.38865661621094,
      "KL/std": 59.605716705322266,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.006682926788926125,
      "fcm_dpo/delta": -0.00022066128440201283,
      "fcm_dpo/margin": 46.464027404785156,
      "fcm_dpo/q_t": 0.4269474744796753,
      "grad_norm": 23.390378952026367,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": 1.6523057222366333,
      "logits/rejected": 1.6650694608688354,
      "logps/chosen": -306.15325927734375,
      "logps/ref_chosen": -280.2286376953125,
      "logps/ref_rejected": -280.8172912597656,
      "logps/rejected": -353.2059326171875,
      "loss": 4.6145,
      "margin_dpo/margin_mean": 46.464027404785156,
      "margin_dpo/margin_std": 68.29534912109375,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -19.456554412841797,
      "KL/mean": -45.69917297363281,
      "KL/rejected_KL_mean": -71.94178771972656,
      "KL/std": 62.74684143066406,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.00656413659453392,
      "fcm_dpo/delta": -0.056091997772455215,
      "fcm_dpo/margin": 52.48523712158203,
      "fcm_dpo/q_t": 0.4196234941482544,
      "grad_norm": 26.08226203918457,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": 1.2744899988174438,
      "logits/rejected": 1.2772325277328491,
      "logps/chosen": -316.4568786621094,
      "logps/ref_chosen": -297.0003356933594,
      "logps/ref_rejected": -279.1561279296875,
      "logps/rejected": -351.097900390625,
      "loss": 4.5314,
      "margin_dpo/margin_mean": 52.48523712158203,
      "margin_dpo/margin_std": 74.00574493408203,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -29.766115188598633,
      "KL/mean": -50.27949523925781,
      "KL/rejected_KL_mean": -70.7928695678711,
      "KL/std": 60.58844757080078,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.0064441789872944355,
      "fcm_dpo/delta": 0.0012970532989129424,
      "fcm_dpo/margin": 41.02676010131836,
      "fcm_dpo/q_t": 0.4368269443511963,
      "grad_norm": 30.30462646484375,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": 1.4438220262527466,
      "logits/rejected": 1.552324652671814,
      "logps/chosen": -301.2375793457031,
      "logps/ref_chosen": -271.471435546875,
      "logps/ref_rejected": -268.0870056152344,
      "logps/rejected": -338.8798828125,
      "loss": 4.7319,
      "margin_dpo/margin_mean": 41.02676010131836,
      "margin_dpo/margin_std": 61.7968864440918,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -29.983966827392578,
      "KL/mean": -51.26396560668945,
      "KL/rejected_KL_mean": -72.54396057128906,
      "KL/std": 59.726593017578125,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.0064831688068807125,
      "fcm_dpo/delta": 0.02090342529118061,
      "fcm_dpo/margin": 42.55999755859375,
      "fcm_dpo/q_t": 0.43513986468315125,
      "grad_norm": 38.59688186645508,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 1.3454720973968506,
      "logits/rejected": 1.338334321975708,
      "logps/chosen": -336.3418273925781,
      "logps/ref_chosen": -306.35784912109375,
      "logps/ref_rejected": -268.64776611328125,
      "logps/rejected": -341.19171142578125,
      "loss": 4.748,
      "margin_dpo/margin_mean": 42.55999755859375,
      "margin_dpo/margin_std": 73.57970428466797,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -28.815393447875977,
      "KL/mean": -49.97590637207031,
      "KL/rejected_KL_mean": -71.13642883300781,
      "KL/std": 60.828125,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.006697408854961395,
      "fcm_dpo/delta": 0.04768696054816246,
      "fcm_dpo/margin": 42.3210334777832,
      "fcm_dpo/q_t": 0.4334796667098999,
      "grad_norm": 33.95066833496094,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.2236063480377197,
      "logits/rejected": 1.279178500175476,
      "logps/chosen": -298.807861328125,
      "logps/ref_chosen": -269.9924621582031,
      "logps/ref_rejected": -271.3135681152344,
      "logps/rejected": -342.4500427246094,
      "loss": 4.7154,
      "margin_dpo/margin_mean": 42.32103729248047,
      "margin_dpo/margin_std": 70.6814193725586,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -27.1036319732666,
      "KL/mean": -52.12931823730469,
      "KL/rejected_KL_mean": -77.1550064086914,
      "KL/std": 60.031585693359375,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.006716761738061905,
      "fcm_dpo/delta": -0.009889008477330208,
      "fcm_dpo/margin": 50.051368713378906,
      "fcm_dpo/q_t": 0.4204072952270508,
      "grad_norm": 24.913196563720703,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": 1.4465467929840088,
      "logits/rejected": 1.5260837078094482,
      "logps/chosen": -287.5643310546875,
      "logps/ref_chosen": -260.460693359375,
      "logps/ref_rejected": -254.2005615234375,
      "logps/rejected": -331.3555908203125,
      "loss": 4.5517,
      "margin_dpo/margin_mean": 50.051368713378906,
      "margin_dpo/margin_std": 71.2435302734375,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -25.57321548461914,
      "KL/mean": -48.29783630371094,
      "KL/rejected_KL_mean": -71.02244567871094,
      "KL/std": 59.999168395996094,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.006709085311740637,
      "fcm_dpo/delta": -0.014878163114190102,
      "fcm_dpo/margin": 45.449241638183594,
      "fcm_dpo/q_t": 0.4275047183036804,
      "grad_norm": 42.935428619384766,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": 1.2820379734039307,
      "logits/rejected": 1.335497498512268,
      "logps/chosen": -321.66949462890625,
      "logps/ref_chosen": -296.09625244140625,
      "logps/ref_rejected": -264.7806701660156,
      "logps/rejected": -335.8031005859375,
      "loss": 4.6713,
      "margin_dpo/margin_mean": 45.449241638183594,
      "margin_dpo/margin_std": 72.53079986572266,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -38.16606521606445,
      "KL/mean": -58.339839935302734,
      "KL/rejected_KL_mean": -78.51362609863281,
      "KL/std": 63.94147872924805,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.00677832867950201,
      "fcm_dpo/delta": 0.021766573190689087,
      "fcm_dpo/margin": 40.34754943847656,
      "fcm_dpo/q_t": 0.43565672636032104,
      "grad_norm": 24.83973503112793,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": 1.1720811128616333,
      "logits/rejected": 1.1602578163146973,
      "logps/chosen": -320.0155334472656,
      "logps/ref_chosen": -281.8494873046875,
      "logps/ref_rejected": -234.3673858642578,
      "logps/rejected": -312.88104248046875,
      "loss": 4.7635,
      "margin_dpo/margin_mean": 40.34755325317383,
      "margin_dpo/margin_std": 71.7032241821289,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -20.941808700561523,
      "KL/mean": -50.60677719116211,
      "KL/rejected_KL_mean": -80.27175903320312,
      "KL/std": 64.1218490600586,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.006609754636883736,
      "fcm_dpo/delta": -0.0644335225224495,
      "fcm_dpo/margin": 59.329959869384766,
      "fcm_dpo/q_t": 0.4075421094894409,
      "grad_norm": 32.39091873168945,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": 1.1956496238708496,
      "logits/rejected": 1.3007614612579346,
      "logps/chosen": -268.98455810546875,
      "logps/ref_chosen": -248.0427703857422,
      "logps/ref_rejected": -263.464599609375,
      "logps/rejected": -343.7363586425781,
      "loss": 4.373,
      "margin_dpo/margin_mean": 59.329959869384766,
      "margin_dpo/margin_std": 72.90184020996094,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -20.12569808959961,
      "KL/mean": -45.739540100097656,
      "KL/rejected_KL_mean": -71.3533935546875,
      "KL/std": 60.92407989501953,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.006476250477135181,
      "fcm_dpo/delta": -0.008115990087389946,
      "fcm_dpo/margin": 51.227684020996094,
      "fcm_dpo/q_t": 0.4214398264884949,
      "grad_norm": 29.42223358154297,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": 1.4924051761627197,
      "logits/rejected": 1.4189156293869019,
      "logps/chosen": -331.41162109375,
      "logps/ref_chosen": -311.2859191894531,
      "logps/ref_rejected": -268.4949645996094,
      "logps/rejected": -339.84832763671875,
      "loss": 4.5461,
      "margin_dpo/margin_mean": 51.227691650390625,
      "margin_dpo/margin_std": 69.82916259765625,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -26.97306251525879,
      "KL/mean": -55.20698547363281,
      "KL/rejected_KL_mean": -83.44090270996094,
      "KL/std": 64.14069366455078,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.006342010572552681,
      "fcm_dpo/delta": -0.04554333537817001,
      "fcm_dpo/margin": 56.46784973144531,
      "fcm_dpo/q_t": 0.41460421681404114,
      "grad_norm": 22.003700256347656,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": 1.3358668088912964,
      "logits/rejected": 1.340239405632019,
      "logps/chosen": -297.185546875,
      "logps/ref_chosen": -270.21246337890625,
      "logps/ref_rejected": -239.1932830810547,
      "logps/rejected": -322.6341552734375,
      "loss": 4.4404,
      "margin_dpo/margin_mean": 56.46784973144531,
      "margin_dpo/margin_std": 68.10401153564453,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -34.194801330566406,
      "KL/mean": -52.87619400024414,
      "KL/rejected_KL_mean": -71.55757904052734,
      "KL/std": 61.622840881347656,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.0062793237157166,
      "fcm_dpo/delta": 0.037548113614320755,
      "fcm_dpo/margin": 37.362770080566406,
      "fcm_dpo/q_t": 0.44447165727615356,
      "grad_norm": 24.542198181152344,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": 1.2251380681991577,
      "logits/rejected": 1.2178269624710083,
      "logps/chosen": -322.7176513671875,
      "logps/ref_chosen": -288.5228271484375,
      "logps/ref_rejected": -253.90597534179688,
      "logps/rejected": -325.46356201171875,
      "loss": 4.8665,
      "margin_dpo/margin_mean": 37.36277389526367,
      "margin_dpo/margin_std": 70.34661102294922,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -32.58028793334961,
      "KL/mean": -60.30564880371094,
      "KL/rejected_KL_mean": -88.03099822998047,
      "KL/std": 65.30012512207031,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.006324210204184055,
      "fcm_dpo/delta": -0.023812919855117798,
      "fcm_dpo/margin": 55.450714111328125,
      "fcm_dpo/q_t": 0.4179151952266693,
      "grad_norm": 29.367076873779297,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 1.2297852039337158,
      "logits/rejected": 1.3383907079696655,
      "logps/chosen": -304.47503662109375,
      "logps/ref_chosen": -271.8947448730469,
      "logps/ref_rejected": -260.5709228515625,
      "logps/rejected": -348.60186767578125,
      "loss": 4.5065,
      "margin_dpo/margin_mean": 55.450714111328125,
      "margin_dpo/margin_std": 77.08302307128906,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -36.49464416503906,
      "KL/mean": -59.19923782348633,
      "KL/rejected_KL_mean": -81.90382385253906,
      "KL/std": 60.69124984741211,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.006272952537983656,
      "fcm_dpo/delta": -0.004127860069274902,
      "fcm_dpo/margin": 45.40919494628906,
      "fcm_dpo/q_t": 0.4325340688228607,
      "grad_norm": 27.314123153686523,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 1.1205840110778809,
      "logits/rejected": 1.2319813966751099,
      "logps/chosen": -306.6701354980469,
      "logps/ref_chosen": -270.17547607421875,
      "logps/ref_rejected": -257.6781311035156,
      "logps/rejected": -339.58197021484375,
      "loss": 4.6657,
      "margin_dpo/margin_mean": 45.4091911315918,
      "margin_dpo/margin_std": 65.9197006225586,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -34.93788146972656,
      "KL/mean": -59.768985748291016,
      "KL/rejected_KL_mean": -84.60008239746094,
      "KL/std": 64.38106536865234,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.006242044270038605,
      "fcm_dpo/delta": -0.007484890520572662,
      "fcm_dpo/margin": 49.66218948364258,
      "fcm_dpo/q_t": 0.4266931414604187,
      "grad_norm": 27.447628021240234,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 1.2971068620681763,
      "logits/rejected": 1.4518681764602661,
      "logps/chosen": -298.70013427734375,
      "logps/ref_chosen": -263.76226806640625,
      "logps/ref_rejected": -263.8758850097656,
      "logps/rejected": -348.4759826660156,
      "loss": 4.6582,
      "margin_dpo/margin_mean": 49.66218948364258,
      "margin_dpo/margin_std": 79.02595520019531,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -33.10550308227539,
      "KL/mean": -57.65279769897461,
      "KL/rejected_KL_mean": -82.20010375976562,
      "KL/std": 61.48524856567383,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.0061736032366752625,
      "fcm_dpo/delta": 0.00010113231837749481,
      "fcm_dpo/margin": 49.094608306884766,
      "fcm_dpo/q_t": 0.42795896530151367,
      "grad_norm": 31.530193328857422,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": 1.1550307273864746,
      "logits/rejected": 1.1102404594421387,
      "logps/chosen": -307.7779235839844,
      "logps/ref_chosen": -274.67242431640625,
      "logps/ref_rejected": -242.1553955078125,
      "logps/rejected": -324.3554992675781,
      "loss": 4.6451,
      "margin_dpo/margin_mean": 49.094608306884766,
      "margin_dpo/margin_std": 73.66265106201172,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -37.30099868774414,
      "KL/mean": -60.17115783691406,
      "KL/rejected_KL_mean": -83.04132843017578,
      "KL/std": 65.74032592773438,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.006277354899793863,
      "fcm_dpo/delta": -0.005351033061742783,
      "fcm_dpo/margin": 45.740333557128906,
      "fcm_dpo/q_t": 0.43227338790893555,
      "grad_norm": 25.331951141357422,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": 1.4583932161331177,
      "logits/rejected": 1.5462766885757446,
      "logps/chosen": -332.4888916015625,
      "logps/ref_chosen": -295.1878662109375,
      "logps/ref_rejected": -271.3431091308594,
      "logps/rejected": -354.3844909667969,
      "loss": 4.7275,
      "margin_dpo/margin_mean": 45.740333557128906,
      "margin_dpo/margin_std": 75.9927978515625,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -35.36128234863281,
      "KL/mean": -58.82329177856445,
      "KL/rejected_KL_mean": -82.28529357910156,
      "KL/std": 63.6338005065918,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.006193258333951235,
      "fcm_dpo/delta": 0.005469064228236675,
      "fcm_dpo/margin": 46.92401885986328,
      "fcm_dpo/q_t": 0.4310287535190582,
      "grad_norm": 51.12434005737305,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": 1.2167834043502808,
      "logits/rejected": 1.2604175806045532,
      "logps/chosen": -325.129150390625,
      "logps/ref_chosen": -289.767822265625,
      "logps/ref_rejected": -267.0824279785156,
      "logps/rejected": -349.3677062988281,
      "loss": 4.6999,
      "margin_dpo/margin_mean": 46.924015045166016,
      "margin_dpo/margin_std": 76.2000732421875,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -31.69390106201172,
      "KL/mean": -58.096961975097656,
      "KL/rejected_KL_mean": -84.50003051757812,
      "KL/std": 64.58732604980469,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.006193036679178476,
      "fcm_dpo/delta": -0.002978797070682049,
      "fcm_dpo/margin": 52.806121826171875,
      "fcm_dpo/q_t": 0.42314010858535767,
      "grad_norm": 26.359420776367188,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": 1.5111496448516846,
      "logits/rejected": 1.5261718034744263,
      "logps/chosen": -330.11297607421875,
      "logps/ref_chosen": -298.4190673828125,
      "logps/ref_rejected": -266.64447021484375,
      "logps/rejected": -351.1445007324219,
      "loss": 4.5644,
      "margin_dpo/margin_mean": 52.806121826171875,
      "margin_dpo/margin_std": 75.78223419189453,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -37.298099517822266,
      "KL/mean": -65.22569274902344,
      "KL/rejected_KL_mean": -93.15328216552734,
      "KL/std": 66.21449279785156,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.006165334489196539,
      "fcm_dpo/delta": -0.01280367560684681,
      "fcm_dpo/margin": 55.85517883300781,
      "fcm_dpo/q_t": 0.41917383670806885,
      "grad_norm": 26.289752960205078,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": 1.2990326881408691,
      "logits/rejected": 1.292647361755371,
      "logps/chosen": -339.5279541015625,
      "logps/ref_chosen": -302.2298583984375,
      "logps/ref_rejected": -272.7162170410156,
      "logps/rejected": -365.8694763183594,
      "loss": 4.5071,
      "margin_dpo/margin_mean": 55.85517883300781,
      "margin_dpo/margin_std": 76.14733123779297,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -29.202835083007812,
      "KL/mean": -54.92329025268555,
      "KL/rejected_KL_mean": -80.64374542236328,
      "KL/std": 62.955997467041016,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.006147629115730524,
      "fcm_dpo/delta": 0.01207443606108427,
      "fcm_dpo/margin": 51.440914154052734,
      "fcm_dpo/q_t": 0.4251144528388977,
      "grad_norm": 49.980812072753906,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": 1.1367402076721191,
      "logits/rejected": 1.1078227758407593,
      "logps/chosen": -276.0562438964844,
      "logps/ref_chosen": -246.85342407226562,
      "logps/ref_rejected": -216.5264892578125,
      "logps/rejected": -297.17022705078125,
      "loss": 4.5828,
      "margin_dpo/margin_mean": 51.44091796875,
      "margin_dpo/margin_std": 72.3245620727539,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -34.30552673339844,
      "KL/mean": -59.55128479003906,
      "KL/rejected_KL_mean": -84.79703521728516,
      "KL/std": 64.01325988769531,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.006172084715217352,
      "fcm_dpo/delta": -0.007514355704188347,
      "fcm_dpo/margin": 50.49151611328125,
      "fcm_dpo/q_t": 0.425422728061676,
      "grad_norm": 53.458770751953125,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": 1.3109852075576782,
      "logits/rejected": 1.397509217262268,
      "logps/chosen": -296.95513916015625,
      "logps/ref_chosen": -262.649658203125,
      "logps/ref_rejected": -251.24945068359375,
      "logps/rejected": -336.0464782714844,
      "loss": 4.5874,
      "margin_dpo/margin_mean": 50.49151611328125,
      "margin_dpo/margin_std": 69.40398406982422,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -28.571325302124023,
      "KL/mean": -56.571205139160156,
      "KL/rejected_KL_mean": -84.57109069824219,
      "KL/std": 63.24675750732422,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.006113649345934391,
      "fcm_dpo/delta": 0.0075470320880413055,
      "fcm_dpo/margin": 55.999759674072266,
      "fcm_dpo/q_t": 0.41817706823349,
      "grad_norm": 22.36319923400879,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": 1.4038994312286377,
      "logits/rejected": 1.4546418190002441,
      "logps/chosen": -321.95770263671875,
      "logps/ref_chosen": -293.3863220214844,
      "logps/ref_rejected": -238.86026000976562,
      "logps/rejected": -323.4313659667969,
      "loss": 4.4491,
      "margin_dpo/margin_mean": 55.999759674072266,
      "margin_dpo/margin_std": 63.672142028808594,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -27.266340255737305,
      "KL/mean": -52.95782470703125,
      "KL/rejected_KL_mean": -78.64930725097656,
      "KL/std": 63.449283599853516,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.006137266289442778,
      "fcm_dpo/delta": -0.023932842537760735,
      "fcm_dpo/margin": 51.382957458496094,
      "fcm_dpo/q_t": 0.4251866936683655,
      "grad_norm": 41.082489013671875,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 1.3946305513381958,
      "logits/rejected": 1.4241005182266235,
      "logps/chosen": -322.7701416015625,
      "logps/ref_chosen": -295.5038146972656,
      "logps/ref_rejected": -278.524169921875,
      "logps/rejected": -357.1734924316406,
      "loss": 4.631,
      "margin_dpo/margin_mean": 51.382957458496094,
      "margin_dpo/margin_std": 79.98564910888672,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -30.506221771240234,
      "KL/mean": -52.700050354003906,
      "KL/rejected_KL_mean": -74.89389038085938,
      "KL/std": 63.73116683959961,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.0062629529275000095,
      "fcm_dpo/delta": 0.04666643589735031,
      "fcm_dpo/margin": 44.38766098022461,
      "fcm_dpo/q_t": 0.43435966968536377,
      "grad_norm": 27.308677673339844,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 1.674647331237793,
      "logits/rejected": 1.6250500679016113,
      "logps/chosen": -311.2615661621094,
      "logps/ref_chosen": -280.7553405761719,
      "logps/ref_rejected": -237.47286987304688,
      "logps/rejected": -312.3667297363281,
      "loss": 4.7217,
      "margin_dpo/margin_mean": 44.387664794921875,
      "margin_dpo/margin_std": 73.47088623046875,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -30.787818908691406,
      "KL/mean": -57.94097137451172,
      "KL/rejected_KL_mean": -85.09412384033203,
      "KL/std": 65.82294464111328,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.006239354610443115,
      "fcm_dpo/delta": -0.021518852561712265,
      "fcm_dpo/margin": 54.306304931640625,
      "fcm_dpo/q_t": 0.41958463191986084,
      "grad_norm": 37.03125762939453,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 1.4301615953445435,
      "logits/rejected": 1.4399337768554688,
      "logps/chosen": -329.8791198730469,
      "logps/ref_chosen": -299.09130859375,
      "logps/ref_rejected": -268.9637756347656,
      "logps/rejected": -354.057861328125,
      "loss": 4.5192,
      "margin_dpo/margin_mean": 54.306304931640625,
      "margin_dpo/margin_std": 73.71610260009766,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -36.678306579589844,
      "KL/mean": -55.58849334716797,
      "KL/rejected_KL_mean": -74.49867248535156,
      "KL/std": 63.065025329589844,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.006224056705832481,
      "fcm_dpo/delta": 0.007301344536244869,
      "fcm_dpo/margin": 37.82036209106445,
      "fcm_dpo/q_t": 0.4444431960582733,
      "grad_norm": 28.335824966430664,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": 1.5918333530426025,
      "logits/rejected": 1.5504992008209229,
      "logps/chosen": -313.37432861328125,
      "logps/ref_chosen": -276.696044921875,
      "logps/ref_rejected": -225.78790283203125,
      "logps/rejected": -300.28656005859375,
      "loss": 4.9222,
      "margin_dpo/margin_mean": 37.82036209106445,
      "margin_dpo/margin_std": 81.20458221435547,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -23.867691040039062,
      "KL/mean": -53.865814208984375,
      "KL/rejected_KL_mean": -83.86392211914062,
      "KL/std": 64.13953399658203,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.006019862834364176,
      "fcm_dpo/delta": -0.03729663044214249,
      "fcm_dpo/margin": 59.9962158203125,
      "fcm_dpo/q_t": 0.41400039196014404,
      "grad_norm": 36.7513427734375,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": 1.3579412698745728,
      "logits/rejected": 1.4415276050567627,
      "logps/chosen": -254.4527130126953,
      "logps/ref_chosen": -230.58499145507812,
      "logps/ref_rejected": -229.8074493408203,
      "logps/rejected": -313.6713562011719,
      "loss": 4.4062,
      "margin_dpo/margin_mean": 59.9962158203125,
      "margin_dpo/margin_std": 67.14761352539062,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -28.211715698242188,
      "KL/mean": -52.65016555786133,
      "KL/rejected_KL_mean": -77.08860778808594,
      "KL/std": 68.05137634277344,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.005999343004077673,
      "fcm_dpo/delta": 0.008088069036602974,
      "fcm_dpo/margin": 48.87690734863281,
      "fcm_dpo/q_t": 0.43176984786987305,
      "grad_norm": 40.93552017211914,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": 1.361984133720398,
      "logits/rejected": 1.4012986421585083,
      "logps/chosen": -298.7677001953125,
      "logps/ref_chosen": -270.5559997558594,
      "logps/ref_rejected": -266.328369140625,
      "logps/rejected": -343.4169921875,
      "loss": 4.7308,
      "margin_dpo/margin_mean": 48.87691116333008,
      "margin_dpo/margin_std": 86.3935546875,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -24.686569213867188,
      "KL/mean": -50.134666442871094,
      "KL/rejected_KL_mean": -75.58274841308594,
      "KL/std": 62.49614715576172,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.006198465824127197,
      "fcm_dpo/delta": 0.03533268719911575,
      "fcm_dpo/margin": 50.89619064331055,
      "fcm_dpo/q_t": 0.4254573881626129,
      "grad_norm": 34.666107177734375,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": 1.274807095527649,
      "logits/rejected": 1.3614484071731567,
      "logps/chosen": -257.9251708984375,
      "logps/ref_chosen": -233.2386016845703,
      "logps/ref_rejected": -258.154541015625,
      "logps/rejected": -333.7372741699219,
      "loss": 4.6277,
      "margin_dpo/margin_mean": 50.89619064331055,
      "margin_dpo/margin_std": 79.92534637451172,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -14.76801872253418,
      "KL/mean": -43.20484161376953,
      "KL/rejected_KL_mean": -71.64167785644531,
      "KL/std": 63.20402145385742,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.006196317262947559,
      "fcm_dpo/delta": -0.02259736694395542,
      "fcm_dpo/margin": 56.8736572265625,
      "fcm_dpo/q_t": 0.41676342487335205,
      "grad_norm": 23.561872482299805,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": 1.4594719409942627,
      "logits/rejected": 1.4942654371261597,
      "logps/chosen": -283.314208984375,
      "logps/ref_chosen": -268.5461730957031,
      "logps/ref_rejected": -266.5316467285156,
      "logps/rejected": -338.17327880859375,
      "loss": 4.5067,
      "margin_dpo/margin_mean": 56.8736572265625,
      "margin_dpo/margin_std": 78.37486267089844,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -27.4244384765625,
      "KL/mean": -52.73503112792969,
      "KL/rejected_KL_mean": -78.04562377929688,
      "KL/std": 64.39427947998047,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.006048653740435839,
      "fcm_dpo/delta": -0.020154006779193878,
      "fcm_dpo/margin": 50.621185302734375,
      "fcm_dpo/q_t": 0.4272252321243286,
      "grad_norm": 56.88744354248047,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": 1.2021706104278564,
      "logits/rejected": 1.2242573499679565,
      "logps/chosen": -322.30120849609375,
      "logps/ref_chosen": -294.8768005371094,
      "logps/ref_rejected": -281.09771728515625,
      "logps/rejected": -359.14337158203125,
      "loss": 4.62,
      "margin_dpo/margin_mean": 50.621185302734375,
      "margin_dpo/margin_std": 74.09422302246094,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -36.27273178100586,
      "KL/mean": -57.75022506713867,
      "KL/rejected_KL_mean": -79.22772216796875,
      "KL/std": 69.6645278930664,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.006137675605714321,
      "fcm_dpo/delta": 0.03255471587181091,
      "fcm_dpo/margin": 42.95498275756836,
      "fcm_dpo/q_t": 0.43853747844696045,
      "grad_norm": 27.558162689208984,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": 1.430652141571045,
      "logits/rejected": 1.4459766149520874,
      "logps/chosen": -348.5705261230469,
      "logps/ref_chosen": -312.2977600097656,
      "logps/ref_rejected": -304.6536560058594,
      "logps/rejected": -383.88134765625,
      "loss": 4.8338,
      "margin_dpo/margin_mean": 42.95499038696289,
      "margin_dpo/margin_std": 84.05970001220703,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -32.838157653808594,
      "KL/mean": -51.564796447753906,
      "KL/rejected_KL_mean": -70.29144287109375,
      "KL/std": 65.51470947265625,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.006250401958823204,
      "fcm_dpo/delta": 0.02425036020576954,
      "fcm_dpo/margin": 37.45328903198242,
      "fcm_dpo/q_t": 0.4451409578323364,
      "grad_norm": 47.7509765625,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 1.270750641822815,
      "logits/rejected": 1.3350883722305298,
      "logps/chosen": -327.0423889160156,
      "logps/ref_chosen": -294.2042236328125,
      "logps/ref_rejected": -296.2102355957031,
      "logps/rejected": -366.5016784667969,
      "loss": 4.9031,
      "margin_dpo/margin_mean": 37.453285217285156,
      "margin_dpo/margin_std": 78.70518493652344,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -26.416156768798828,
      "KL/mean": -49.335060119628906,
      "KL/rejected_KL_mean": -72.25395202636719,
      "KL/std": 65.05891418457031,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.006297783460468054,
      "fcm_dpo/delta": 0.00228310888633132,
      "fcm_dpo/margin": 45.83780288696289,
      "fcm_dpo/q_t": 0.43208441138267517,
      "grad_norm": 31.611860275268555,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.392313838005066,
      "logits/rejected": 1.4106934070587158,
      "logps/chosen": -294.7287902832031,
      "logps/ref_chosen": -268.3126525878906,
      "logps/ref_rejected": -255.5543212890625,
      "logps/rejected": -327.8082580566406,
      "loss": 4.7045,
      "margin_dpo/margin_mean": 45.837799072265625,
      "margin_dpo/margin_std": 75.203369140625,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -26.727645874023438,
      "KL/mean": -55.6285514831543,
      "KL/rejected_KL_mean": -84.52947235107422,
      "KL/std": 66.19429779052734,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.006172165274620056,
      "fcm_dpo/delta": -0.026184778660535812,
      "fcm_dpo/margin": 57.80183029174805,
      "fcm_dpo/q_t": 0.41586801409721375,
      "grad_norm": 36.2138671875,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": 1.3578213453292847,
      "logits/rejected": 1.3975375890731812,
      "logps/chosen": -290.47808837890625,
      "logps/ref_chosen": -263.75042724609375,
      "logps/ref_rejected": -234.5144500732422,
      "logps/rejected": -319.0439147949219,
      "loss": 4.4997,
      "margin_dpo/margin_mean": 57.80183410644531,
      "margin_dpo/margin_std": 81.1626968383789,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -25.4312744140625,
      "KL/mean": -46.80036926269531,
      "KL/rejected_KL_mean": -68.16946411132812,
      "KL/std": 63.240413665771484,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.006313118617981672,
      "fcm_dpo/delta": 0.03730397671461105,
      "fcm_dpo/margin": 42.738189697265625,
      "fcm_dpo/q_t": 0.436068594455719,
      "grad_norm": 36.31412124633789,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": 1.5578030347824097,
      "logits/rejected": 1.5484049320220947,
      "logps/chosen": -312.2477111816406,
      "logps/ref_chosen": -286.81646728515625,
      "logps/ref_rejected": -260.4169616699219,
      "logps/rejected": -328.58642578125,
      "loss": 4.7539,
      "margin_dpo/margin_mean": 42.73819351196289,
      "margin_dpo/margin_std": 72.25802612304688,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -23.441322326660156,
      "KL/mean": -48.83234786987305,
      "KL/rejected_KL_mean": -74.22337341308594,
      "KL/std": 65.88062286376953,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.006354607176035643,
      "fcm_dpo/delta": -0.022333327680826187,
      "fcm_dpo/margin": 50.78205490112305,
      "fcm_dpo/q_t": 0.4244767129421234,
      "grad_norm": 33.03730773925781,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": 1.2370400428771973,
      "logits/rejected": 1.2361977100372314,
      "logps/chosen": -316.7522277832031,
      "logps/ref_chosen": -293.3109130859375,
      "logps/ref_rejected": -264.1449279785156,
      "logps/rejected": -338.3683166503906,
      "loss": 4.6508,
      "margin_dpo/margin_mean": 50.78205490112305,
      "margin_dpo/margin_std": 82.47505187988281,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -38.15849304199219,
      "KL/mean": -61.130126953125,
      "KL/rejected_KL_mean": -84.10176086425781,
      "KL/std": 64.27632141113281,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.006248917430639267,
      "fcm_dpo/delta": 0.003860275261104107,
      "fcm_dpo/margin": 45.943267822265625,
      "fcm_dpo/q_t": 0.4327232837677002,
      "grad_norm": 28.86574935913086,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 1.3502018451690674,
      "logits/rejected": 1.4506105184555054,
      "logps/chosen": -307.91485595703125,
      "logps/ref_chosen": -269.75634765625,
      "logps/ref_rejected": -241.08438110351562,
      "logps/rejected": -325.1861267089844,
      "loss": 4.7627,
      "margin_dpo/margin_mean": 45.943267822265625,
      "margin_dpo/margin_std": 82.85317993164062,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -28.533510208129883,
      "KL/mean": -50.69529724121094,
      "KL/rejected_KL_mean": -72.85708618164062,
      "KL/std": 64.56777954101562,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.006333356257528067,
      "fcm_dpo/delta": 0.01765347272157669,
      "fcm_dpo/margin": 44.323577880859375,
      "fcm_dpo/q_t": 0.434348464012146,
      "grad_norm": 34.51810073852539,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": 1.518268346786499,
      "logits/rejected": 1.5495967864990234,
      "logps/chosen": -265.9053039550781,
      "logps/ref_chosen": -237.37179565429688,
      "logps/ref_rejected": -253.5463409423828,
      "logps/rejected": -326.4034423828125,
      "loss": 4.7293,
      "margin_dpo/margin_mean": 44.32358169555664,
      "margin_dpo/margin_std": 75.06216430664062,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -34.02997589111328,
      "KL/mean": -59.638214111328125,
      "KL/rejected_KL_mean": -85.2464599609375,
      "KL/std": 64.21141052246094,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.006402851082384586,
      "fcm_dpo/delta": -0.007774224504828453,
      "fcm_dpo/margin": 51.216487884521484,
      "fcm_dpo/q_t": 0.42355290055274963,
      "grad_norm": 39.071144104003906,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": 1.0541033744812012,
      "logits/rejected": 1.1021695137023926,
      "logps/chosen": -295.3376159667969,
      "logps/ref_chosen": -261.3076477050781,
      "logps/ref_rejected": -252.3170166015625,
      "logps/rejected": -337.5634765625,
      "loss": 4.6154,
      "margin_dpo/margin_mean": 51.216487884521484,
      "margin_dpo/margin_std": 81.65045928955078,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -17.339065551757812,
      "KL/mean": -46.921913146972656,
      "KL/rejected_KL_mean": -76.50475311279297,
      "KL/std": 66.18382263183594,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.006291863974183798,
      "fcm_dpo/delta": -0.023192577064037323,
      "fcm_dpo/margin": 59.165679931640625,
      "fcm_dpo/q_t": 0.4126308560371399,
      "grad_norm": 25.905010223388672,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": 1.4315043687820435,
      "logits/rejected": 1.4169501066207886,
      "logps/chosen": -286.1108093261719,
      "logps/ref_chosen": -268.7717590332031,
      "logps/ref_rejected": -232.3352813720703,
      "logps/rejected": -308.84002685546875,
      "loss": 4.4326,
      "margin_dpo/margin_mean": 59.16568374633789,
      "margin_dpo/margin_std": 78.41838073730469,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -26.666561126708984,
      "KL/mean": -56.78811264038086,
      "KL/rejected_KL_mean": -86.90966033935547,
      "KL/std": 68.48039245605469,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.0061700306832790375,
      "fcm_dpo/delta": -0.022951878607273102,
      "fcm_dpo/margin": 60.24311065673828,
      "fcm_dpo/q_t": 0.4123905599117279,
      "grad_norm": 27.902175903320312,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": 1.1806966066360474,
      "logits/rejected": 1.1471039056777954,
      "logps/chosen": -320.2796630859375,
      "logps/ref_chosen": -293.6131286621094,
      "logps/ref_rejected": -281.87738037109375,
      "logps/rejected": -368.78704833984375,
      "loss": 4.4547,
      "margin_dpo/margin_mean": 60.24311065673828,
      "margin_dpo/margin_std": 82.73958587646484,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -32.590484619140625,
      "KL/mean": -56.8644905090332,
      "KL/rejected_KL_mean": -81.13849639892578,
      "KL/std": 67.71037292480469,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.006117869634181261,
      "fcm_dpo/delta": -0.00484220776706934,
      "fcm_dpo/margin": 48.54801559448242,
      "fcm_dpo/q_t": 0.43037620186805725,
      "grad_norm": 28.891117095947266,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 0.9012414216995239,
      "logits/rejected": 1.0123016834259033,
      "logps/chosen": -284.3475036621094,
      "logps/ref_chosen": -251.75704956054688,
      "logps/ref_rejected": -255.76441955566406,
      "logps/rejected": -336.90289306640625,
      "loss": 4.6896,
      "margin_dpo/margin_mean": 48.54801559448242,
      "margin_dpo/margin_std": 79.59148406982422,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -47.16057586669922,
      "KL/mean": -67.5002212524414,
      "KL/rejected_KL_mean": -87.83984375,
      "KL/std": 71.11100769042969,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.0061308396980166435,
      "fcm_dpo/delta": 0.02000846341252327,
      "fcm_dpo/margin": 40.679283142089844,
      "fcm_dpo/q_t": 0.44181251525878906,
      "grad_norm": 36.61376190185547,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 1.4778717756271362,
      "logits/rejected": 1.592700481414795,
      "logps/chosen": -353.8082275390625,
      "logps/ref_chosen": -306.64764404296875,
      "logps/ref_rejected": -301.4248352050781,
      "logps/rejected": -389.2646789550781,
      "loss": 4.8817,
      "margin_dpo/margin_mean": 40.679283142089844,
      "margin_dpo/margin_std": 83.81196594238281,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -29.27410316467285,
      "KL/mean": -54.556968688964844,
      "KL/rejected_KL_mean": -79.83984375,
      "KL/std": 67.4007568359375,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.006202759686857462,
      "fcm_dpo/delta": -0.015592677518725395,
      "fcm_dpo/margin": 50.56574249267578,
      "fcm_dpo/q_t": 0.4261421263217926,
      "grad_norm": 21.411916732788086,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": 1.6977355480194092,
      "logits/rejected": 1.7737756967544556,
      "logps/chosen": -291.9429931640625,
      "logps/ref_chosen": -262.6689147949219,
      "logps/ref_rejected": -260.8409118652344,
      "logps/rejected": -340.6807556152344,
      "loss": 4.6172,
      "margin_dpo/margin_mean": 50.56574249267578,
      "margin_dpo/margin_std": 75.60535430908203,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -31.136974334716797,
      "KL/mean": -53.828819274902344,
      "KL/rejected_KL_mean": -76.52064514160156,
      "KL/std": 66.8582763671875,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.006154588423669338,
      "fcm_dpo/delta": 0.02573973499238491,
      "fcm_dpo/margin": 45.383689880371094,
      "fcm_dpo/q_t": 0.4344669282436371,
      "grad_norm": 28.40320587158203,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": 1.5501267910003662,
      "logits/rejected": 1.5697038173675537,
      "logps/chosen": -330.7629089355469,
      "logps/ref_chosen": -299.6259765625,
      "logps/ref_rejected": -261.2701416015625,
      "logps/rejected": -337.790771484375,
      "loss": 4.746,
      "margin_dpo/margin_mean": 45.383689880371094,
      "margin_dpo/margin_std": 79.18882751464844,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -32.355499267578125,
      "KL/mean": -60.02436828613281,
      "KL/rejected_KL_mean": -87.69325256347656,
      "KL/std": 68.60318756103516,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.006361205130815506,
      "fcm_dpo/delta": -0.0029856301844120026,
      "fcm_dpo/margin": 55.33774185180664,
      "fcm_dpo/q_t": 0.41760241985321045,
      "grad_norm": 32.2179069519043,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": 1.4321014881134033,
      "logits/rejected": 1.4910461902618408,
      "logps/chosen": -323.305908203125,
      "logps/ref_chosen": -290.950439453125,
      "logps/ref_rejected": -255.15374755859375,
      "logps/rejected": -342.84698486328125,
      "loss": 4.5241,
      "margin_dpo/margin_mean": 55.337745666503906,
      "margin_dpo/margin_std": 79.06625366210938,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -28.295642852783203,
      "KL/mean": -50.253807067871094,
      "KL/rejected_KL_mean": -72.21197509765625,
      "KL/std": 65.3382797241211,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.006228390149772167,
      "fcm_dpo/delta": 0.00397074967622757,
      "fcm_dpo/margin": 43.91633605957031,
      "fcm_dpo/q_t": 0.4353671967983246,
      "grad_norm": 36.16842269897461,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": 1.3588234186172485,
      "logits/rejected": 1.4382396936416626,
      "logps/chosen": -301.5615234375,
      "logps/ref_chosen": -273.2658996582031,
      "logps/ref_rejected": -263.9664306640625,
      "logps/rejected": -336.1783752441406,
      "loss": 4.7251,
      "margin_dpo/margin_mean": 43.91633605957031,
      "margin_dpo/margin_std": 68.65762329101562,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -42.217369079589844,
      "KL/mean": -68.89308166503906,
      "KL/rejected_KL_mean": -95.56880187988281,
      "KL/std": 68.1356201171875,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.006025138311088085,
      "fcm_dpo/delta": -0.06754852086305618,
      "fcm_dpo/margin": 53.351436614990234,
      "fcm_dpo/q_t": 0.4235137701034546,
      "grad_norm": 33.29505920410156,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": 1.374111294746399,
      "logits/rejected": 1.443422555923462,
      "logps/chosen": -304.8995361328125,
      "logps/ref_chosen": -262.6821594238281,
      "logps/ref_rejected": -226.46859741210938,
      "logps/rejected": -322.03741455078125,
      "loss": 4.606,
      "margin_dpo/margin_mean": 53.35143280029297,
      "margin_dpo/margin_std": 75.81166076660156,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -38.947181701660156,
      "KL/mean": -61.898475646972656,
      "KL/rejected_KL_mean": -84.84976196289062,
      "KL/std": 67.70819854736328,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.006057591177523136,
      "fcm_dpo/delta": 0.04737425968050957,
      "fcm_dpo/margin": 45.902584075927734,
      "fcm_dpo/q_t": 0.4344373941421509,
      "grad_norm": 30.649253845214844,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": 1.1430245637893677,
      "logits/rejected": 1.1532371044158936,
      "logps/chosen": -302.77178955078125,
      "logps/ref_chosen": -263.8246154785156,
      "logps/ref_rejected": -237.9121551513672,
      "logps/rejected": -322.76190185546875,
      "loss": 4.735,
      "margin_dpo/margin_mean": 45.90258026123047,
      "margin_dpo/margin_std": 77.68055725097656,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -30.660860061645508,
      "KL/mean": -51.23648452758789,
      "KL/rejected_KL_mean": -71.81210327148438,
      "KL/std": 66.6136703491211,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.006172667723149061,
      "fcm_dpo/delta": 0.015860529616475105,
      "fcm_dpo/margin": 41.15125274658203,
      "fcm_dpo/q_t": 0.44026264548301697,
      "grad_norm": 32.11396026611328,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 1.3261613845825195,
      "logits/rejected": 1.3776123523712158,
      "logps/chosen": -308.814208984375,
      "logps/ref_chosen": -278.1533203125,
      "logps/ref_rejected": -263.39691162109375,
      "logps/rejected": -335.20904541015625,
      "loss": 4.7998,
      "margin_dpo/margin_mean": 41.1512451171875,
      "margin_dpo/margin_std": 71.23860168457031,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -34.34510803222656,
      "KL/mean": -62.36103057861328,
      "KL/rejected_KL_mean": -90.376953125,
      "KL/std": 70.77616119384766,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.0061196754686534405,
      "fcm_dpo/delta": -0.031492140144109726,
      "fcm_dpo/margin": 56.03184509277344,
      "fcm_dpo/q_t": 0.4195508360862732,
      "grad_norm": 24.01586151123047,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": 1.453566074371338,
      "logits/rejected": 1.4929893016815186,
      "logps/chosen": -352.8001403808594,
      "logps/ref_chosen": -318.455078125,
      "logps/ref_rejected": -288.5691833496094,
      "logps/rejected": -378.9461364746094,
      "loss": 4.5416,
      "margin_dpo/margin_mean": 56.03184509277344,
      "margin_dpo/margin_std": 79.4560546875,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -38.55744934082031,
      "KL/mean": -63.15978240966797,
      "KL/rejected_KL_mean": -87.7621078491211,
      "KL/std": 69.82572174072266,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.006102526094764471,
      "fcm_dpo/delta": -0.021357610821723938,
      "fcm_dpo/margin": 49.204673767089844,
      "fcm_dpo/q_t": 0.42944592237472534,
      "grad_norm": 32.46762466430664,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 1.3354533910751343,
      "logits/rejected": 1.3381510972976685,
      "logps/chosen": -322.74664306640625,
      "logps/ref_chosen": -284.1891784667969,
      "logps/ref_rejected": -247.94589233398438,
      "logps/rejected": -335.7079772949219,
      "loss": 4.7064,
      "margin_dpo/margin_mean": 49.20467758178711,
      "margin_dpo/margin_std": 82.47054290771484,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -36.69792175292969,
      "KL/mean": -53.93428421020508,
      "KL/rejected_KL_mean": -71.170654296875,
      "KL/std": 67.64187622070312,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.00599689083173871,
      "fcm_dpo/delta": -0.008337859064340591,
      "fcm_dpo/margin": 34.47273635864258,
      "fcm_dpo/q_t": 0.4519064128398895,
      "grad_norm": 26.907054901123047,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 1.4677790403366089,
      "logits/rejected": 1.4495065212249756,
      "logps/chosen": -340.00604248046875,
      "logps/ref_chosen": -303.30810546875,
      "logps/ref_rejected": -260.1866455078125,
      "logps/rejected": -331.3572998046875,
      "loss": 5.0103,
      "margin_dpo/margin_mean": 34.47273635864258,
      "margin_dpo/margin_std": 81.37649536132812,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -31.814943313598633,
      "KL/mean": -59.82748031616211,
      "KL/rejected_KL_mean": -87.84001159667969,
      "KL/std": 69.33028411865234,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.005866600200533867,
      "fcm_dpo/delta": -0.03140439838171005,
      "fcm_dpo/margin": 56.025062561035156,
      "fcm_dpo/q_t": 0.42222365736961365,
      "grad_norm": 27.763811111450195,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": 1.2200134992599487,
      "logits/rejected": 1.3957799673080444,
      "logps/chosen": -278.6220397949219,
      "logps/ref_chosen": -246.80709838867188,
      "logps/ref_rejected": -253.13223266601562,
      "logps/rejected": -340.97222900390625,
      "loss": 4.5636,
      "margin_dpo/margin_mean": 56.025062561035156,
      "margin_dpo/margin_std": 77.49406433105469,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -36.272701263427734,
      "KL/mean": -64.18000030517578,
      "KL/rejected_KL_mean": -92.0873031616211,
      "KL/std": 68.53519439697266,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.005725352093577385,
      "fcm_dpo/delta": -0.024335075169801712,
      "fcm_dpo/margin": 55.81460189819336,
      "fcm_dpo/q_t": 0.4242836833000183,
      "grad_norm": 27.597810745239258,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": 1.4572525024414062,
      "logits/rejected": 1.5415884256362915,
      "logps/chosen": -297.8741760253906,
      "logps/ref_chosen": -261.6014709472656,
      "logps/ref_rejected": -262.74761962890625,
      "logps/rejected": -354.83489990234375,
      "loss": 4.5889,
      "margin_dpo/margin_mean": 55.81460189819336,
      "margin_dpo/margin_std": 79.4286117553711,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -36.159507751464844,
      "KL/mean": -53.714969635009766,
      "KL/rejected_KL_mean": -71.27041625976562,
      "KL/std": 68.50431823730469,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.005764411762356758,
      "fcm_dpo/delta": 0.03222891315817833,
      "fcm_dpo/margin": 35.110904693603516,
      "fcm_dpo/q_t": 0.45275700092315674,
      "grad_norm": 25.47645378112793,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": 1.2966935634613037,
      "logits/rejected": 1.4130799770355225,
      "logps/chosen": -288.30792236328125,
      "logps/ref_chosen": -252.14840698242188,
      "logps/ref_rejected": -244.1017608642578,
      "logps/rejected": -315.3721618652344,
      "loss": 5.0314,
      "margin_dpo/margin_mean": 35.11090850830078,
      "margin_dpo/margin_std": 85.98765563964844,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -38.84111785888672,
      "KL/mean": -65.78304290771484,
      "KL/rejected_KL_mean": -92.7249526977539,
      "KL/std": 69.92668914794922,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.0057943640276789665,
      "fcm_dpo/delta": 0.006942464038729668,
      "fcm_dpo/margin": 53.883846282958984,
      "fcm_dpo/q_t": 0.426135390996933,
      "grad_norm": 38.07033920288086,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": 1.3758281469345093,
      "logits/rejected": 1.4903090000152588,
      "logps/chosen": -325.003662109375,
      "logps/ref_chosen": -286.16253662109375,
      "logps/ref_rejected": -272.3275146484375,
      "logps/rejected": -365.0524597167969,
      "loss": 4.6199,
      "margin_dpo/margin_mean": 53.88384246826172,
      "margin_dpo/margin_std": 80.14960479736328,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -30.34202003479004,
      "KL/mean": -53.189552307128906,
      "KL/rejected_KL_mean": -76.03707885742188,
      "KL/std": 65.03862762451172,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.005903454031795263,
      "fcm_dpo/delta": 0.025535959750413895,
      "fcm_dpo/margin": 45.695064544677734,
      "fcm_dpo/q_t": 0.4359555244445801,
      "grad_norm": 28.858745574951172,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": 1.2369577884674072,
      "logits/rejected": 1.287266492843628,
      "logps/chosen": -228.49481201171875,
      "logps/ref_chosen": -198.1527862548828,
      "logps/ref_rejected": -209.80569458007812,
      "logps/rejected": -285.8427734375,
      "loss": 4.7289,
      "margin_dpo/margin_mean": 45.695064544677734,
      "margin_dpo/margin_std": 71.59342956542969,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -42.63557052612305,
      "KL/mean": -67.17172241210938,
      "KL/rejected_KL_mean": -91.7078857421875,
      "KL/std": 66.83798217773438,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.005978786386549473,
      "fcm_dpo/delta": 0.0018498841673135757,
      "fcm_dpo/margin": 49.07231140136719,
      "fcm_dpo/q_t": 0.4316985011100769,
      "grad_norm": 27.530447006225586,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": 1.340505599975586,
      "logits/rejected": 1.402026653289795,
      "logps/chosen": -299.1427307128906,
      "logps/ref_chosen": -256.50714111328125,
      "logps/ref_rejected": -237.40769958496094,
      "logps/rejected": -329.1156005859375,
      "loss": 4.6989,
      "margin_dpo/margin_mean": 49.07231521606445,
      "margin_dpo/margin_std": 79.93595123291016,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -28.7392578125,
      "KL/mean": -56.517494201660156,
      "KL/rejected_KL_mean": -84.29573059082031,
      "KL/std": 68.5987777709961,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.005982166156172752,
      "fcm_dpo/delta": -0.007302367128431797,
      "fcm_dpo/margin": 55.55646514892578,
      "fcm_dpo/q_t": 0.42207586765289307,
      "grad_norm": 27.4040470123291,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": 1.2892777919769287,
      "logits/rejected": 1.2202014923095703,
      "logps/chosen": -321.1865234375,
      "logps/ref_chosen": -292.447265625,
      "logps/ref_rejected": -255.47872924804688,
      "logps/rejected": -339.7744445800781,
      "loss": 4.5913,
      "margin_dpo/margin_mean": 55.556461334228516,
      "margin_dpo/margin_std": 84.89895629882812,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -29.934709548950195,
      "KL/mean": -56.38200378417969,
      "KL/rejected_KL_mean": -82.82931518554688,
      "KL/std": 69.22119903564453,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.0059212283231318,
      "fcm_dpo/delta": -0.002729952335357666,
      "fcm_dpo/margin": 52.89460754394531,
      "fcm_dpo/q_t": 0.4258502125740051,
      "grad_norm": 22.477375030517578,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": 1.417060136795044,
      "logits/rejected": 1.3806686401367188,
      "logps/chosen": -323.6947021484375,
      "logps/ref_chosen": -293.7599792480469,
      "logps/ref_rejected": -259.4142761230469,
      "logps/rejected": -342.24359130859375,
      "loss": 4.6108,
      "margin_dpo/margin_mean": 52.89459991455078,
      "margin_dpo/margin_std": 77.79662322998047,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -37.98042678833008,
      "KL/mean": -61.03052520751953,
      "KL/rejected_KL_mean": -84.08061218261719,
      "KL/std": 68.55587005615234,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.006043045315891504,
      "fcm_dpo/delta": 0.034724850207567215,
      "fcm_dpo/margin": 46.100196838378906,
      "fcm_dpo/q_t": 0.4340156614780426,
      "grad_norm": 26.65464973449707,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 1.1355483531951904,
      "logits/rejected": 1.037163496017456,
      "logps/chosen": -338.8941650390625,
      "logps/ref_chosen": -300.91375732421875,
      "logps/ref_rejected": -256.4080810546875,
      "logps/rejected": -340.4886474609375,
      "loss": 4.7057,
      "margin_dpo/margin_mean": 46.100189208984375,
      "margin_dpo/margin_std": 71.7346420288086,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -25.890743255615234,
      "KL/mean": -56.886558532714844,
      "KL/rejected_KL_mean": -87.88238525390625,
      "KL/std": 70.35189056396484,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.006052282173186541,
      "fcm_dpo/delta": -0.02626667357981205,
      "fcm_dpo/margin": 61.99162292480469,
      "fcm_dpo/q_t": 0.4119662642478943,
      "grad_norm": 36.55012130737305,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.9105010032653809,
      "logits/rejected": 0.9987403750419617,
      "logps/chosen": -282.4510498046875,
      "logps/ref_chosen": -256.5603332519531,
      "logps/ref_rejected": -271.043701171875,
      "logps/rejected": -358.92608642578125,
      "loss": 4.4433,
      "margin_dpo/margin_mean": 61.99161911010742,
      "margin_dpo/margin_std": 84.93429565429688,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -32.053199768066406,
      "KL/mean": -58.387428283691406,
      "KL/rejected_KL_mean": -84.72164916992188,
      "KL/std": 65.23992919921875,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.0059899515472352505,
      "fcm_dpo/delta": 0.0035959784872829914,
      "fcm_dpo/margin": 52.66845703125,
      "fcm_dpo/q_t": 0.42549967765808105,
      "grad_norm": 61.9832649230957,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": 1.2275861501693726,
      "logits/rejected": 1.1228752136230469,
      "logps/chosen": -326.753662109375,
      "logps/ref_chosen": -294.700439453125,
      "logps/ref_rejected": -239.67922973632812,
      "logps/rejected": -324.40087890625,
      "loss": 4.6104,
      "margin_dpo/margin_mean": 52.66845703125,
      "margin_dpo/margin_std": 78.6138916015625,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -31.91779327392578,
      "KL/mean": -61.879432678222656,
      "KL/rejected_KL_mean": -91.84107971191406,
      "KL/std": 71.7168960571289,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.005965659860521555,
      "fcm_dpo/delta": -0.008282652124762535,
      "fcm_dpo/margin": 59.92329406738281,
      "fcm_dpo/q_t": 0.41607868671417236,
      "grad_norm": 34.26577377319336,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": 1.3039804697036743,
      "logits/rejected": 1.3088399171829224,
      "logps/chosen": -325.5936584472656,
      "logps/ref_chosen": -293.67584228515625,
      "logps/ref_rejected": -260.85675048828125,
      "logps/rejected": -352.6978454589844,
      "loss": 4.4853,
      "margin_dpo/margin_mean": 59.92329406738281,
      "margin_dpo/margin_std": 82.19157409667969,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -49.67052459716797,
      "KL/mean": -71.8952865600586,
      "KL/rejected_KL_mean": -94.12004089355469,
      "KL/std": 68.65251922607422,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.006001371890306473,
      "fcm_dpo/delta": 0.027858393266797066,
      "fcm_dpo/margin": 44.44951629638672,
      "fcm_dpo/q_t": 0.43775641918182373,
      "grad_norm": 22.40559196472168,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": 1.3653537034988403,
      "logits/rejected": 1.4463683366775513,
      "logps/chosen": -328.56591796875,
      "logps/ref_chosen": -278.8953857421875,
      "logps/ref_rejected": -283.8898620605469,
      "logps/rejected": -378.0098571777344,
      "loss": 4.8026,
      "margin_dpo/margin_mean": 44.44951629638672,
      "margin_dpo/margin_std": 81.05717468261719,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -31.246198654174805,
      "KL/mean": -59.86199188232422,
      "KL/rejected_KL_mean": -88.477783203125,
      "KL/std": 70.76200103759766,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.006048540119081736,
      "fcm_dpo/delta": -0.014273004606366158,
      "fcm_dpo/margin": 57.231590270996094,
      "fcm_dpo/q_t": 0.41884005069732666,
      "grad_norm": 32.999080657958984,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": 1.1383098363876343,
      "logits/rejected": 1.1712007522583008,
      "logps/chosen": -296.89007568359375,
      "logps/ref_chosen": -265.6439208984375,
      "logps/ref_rejected": -255.40138244628906,
      "logps/rejected": -343.879150390625,
      "loss": 4.5197,
      "margin_dpo/margin_mean": 57.231590270996094,
      "margin_dpo/margin_std": 81.38832092285156,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -37.44205856323242,
      "KL/mean": -65.267822265625,
      "KL/rejected_KL_mean": -93.0936050415039,
      "KL/std": 70.40582275390625,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.00596621772274375,
      "fcm_dpo/delta": -0.0013486244715750217,
      "fcm_dpo/margin": 55.65153884887695,
      "fcm_dpo/q_t": 0.4216139614582062,
      "grad_norm": 31.90444564819336,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": 1.3528850078582764,
      "logits/rejected": 1.3629270792007446,
      "logps/chosen": -317.6665344238281,
      "logps/ref_chosen": -280.2244873046875,
      "logps/ref_rejected": -248.72366333007812,
      "logps/rejected": -341.8172607421875,
      "loss": 4.5558,
      "margin_dpo/margin_mean": 55.65153884887695,
      "margin_dpo/margin_std": 78.9404296875,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -33.96515655517578,
      "KL/mean": -63.613067626953125,
      "KL/rejected_KL_mean": -93.26097869873047,
      "KL/std": 69.13290405273438,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.005953819956630468,
      "fcm_dpo/delta": -0.0036165667697787285,
      "fcm_dpo/margin": 59.29582214355469,
      "fcm_dpo/q_t": 0.4167368412017822,
      "grad_norm": 33.8055419921875,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": 1.33232843875885,
      "logits/rejected": 1.343187928199768,
      "logps/chosen": -326.90185546875,
      "logps/ref_chosen": -292.9366760253906,
      "logps/ref_rejected": -261.88763427734375,
      "logps/rejected": -355.14862060546875,
      "loss": 4.4649,
      "margin_dpo/margin_mean": 59.29582214355469,
      "margin_dpo/margin_std": 76.18218994140625,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -28.536251068115234,
      "KL/mean": -61.08167266845703,
      "KL/rejected_KL_mean": -93.62710571289062,
      "KL/std": 71.43053436279297,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.005798657890409231,
      "fcm_dpo/delta": -0.06363808363676071,
      "fcm_dpo/margin": 65.0908432006836,
      "fcm_dpo/q_t": 0.4102574288845062,
      "grad_norm": 32.42355728149414,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": 1.2614243030548096,
      "logits/rejected": 1.3514339923858643,
      "logps/chosen": -313.4420166015625,
      "logps/ref_chosen": -284.90576171875,
      "logps/ref_rejected": -256.59197998046875,
      "logps/rejected": -350.2190856933594,
      "loss": 4.3886,
      "margin_dpo/margin_mean": 65.0908432006836,
      "margin_dpo/margin_std": 76.98663330078125,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -32.7630729675293,
      "KL/mean": -56.920555114746094,
      "KL/rejected_KL_mean": -81.07803344726562,
      "KL/std": 67.47114562988281,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.00567658431828022,
      "fcm_dpo/delta": -0.0043438272550702095,
      "fcm_dpo/margin": 48.31495666503906,
      "fcm_dpo/q_t": 0.4349139630794525,
      "grad_norm": 29.5855712890625,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": 1.3165500164031982,
      "logits/rejected": 1.3292453289031982,
      "logps/chosen": -308.6789855957031,
      "logps/ref_chosen": -275.91595458984375,
      "logps/ref_rejected": -264.79736328125,
      "logps/rejected": -345.8753967285156,
      "loss": 4.7408,
      "margin_dpo/margin_mean": 48.31495666503906,
      "margin_dpo/margin_std": 79.46040344238281,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -33.711856842041016,
      "KL/mean": -63.38924789428711,
      "KL/rejected_KL_mean": -93.06664276123047,
      "KL/std": 70.58273315429688,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.0055852653458714485,
      "fcm_dpo/delta": -0.02872593328356743,
      "fcm_dpo/margin": 59.35477828979492,
      "fcm_dpo/q_t": 0.42161792516708374,
      "grad_norm": 21.68897819519043,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 1.1237859725952148,
      "logits/rejected": 1.1854989528656006,
      "logps/chosen": -278.65386962890625,
      "logps/ref_chosen": -244.9420166015625,
      "logps/ref_rejected": -235.61978149414062,
      "logps/rejected": -328.6864013671875,
      "loss": 4.5583,
      "margin_dpo/margin_mean": 59.35477828979492,
      "margin_dpo/margin_std": 86.00459289550781,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -38.10612487792969,
      "KL/mean": -62.06977081298828,
      "KL/rejected_KL_mean": -86.03341674804688,
      "KL/std": 69.53043365478516,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.00571614783257246,
      "fcm_dpo/delta": 0.05143982917070389,
      "fcm_dpo/margin": 47.92729187011719,
      "fcm_dpo/q_t": 0.4348520338535309,
      "grad_norm": 45.504642486572266,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 1.2054046392440796,
      "logits/rejected": 1.1922298669815063,
      "logps/chosen": -327.4559631347656,
      "logps/ref_chosen": -289.3498229980469,
      "logps/ref_rejected": -243.70205688476562,
      "logps/rejected": -329.7354736328125,
      "loss": 4.7283,
      "margin_dpo/margin_mean": 47.92729187011719,
      "margin_dpo/margin_std": 77.53506469726562,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -34.030208587646484,
      "KL/mean": -66.72765350341797,
      "KL/rejected_KL_mean": -99.42509460449219,
      "KL/std": 72.66618347167969,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.005627959500998259,
      "fcm_dpo/delta": -0.04143577069044113,
      "fcm_dpo/margin": 65.39488220214844,
      "fcm_dpo/q_t": 0.41253551840782166,
      "grad_norm": 25.582096099853516,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": 1.5202109813690186,
      "logits/rejected": 1.5190849304199219,
      "logps/chosen": -355.6837463378906,
      "logps/ref_chosen": -321.65350341796875,
      "logps/ref_rejected": -282.73773193359375,
      "logps/rejected": -382.1628112792969,
      "loss": 4.4299,
      "margin_dpo/margin_mean": 65.39488220214844,
      "margin_dpo/margin_std": 81.41972351074219,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -34.77204132080078,
      "KL/mean": -65.05564880371094,
      "KL/rejected_KL_mean": -95.33924102783203,
      "KL/std": 73.93132781982422,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.005591258406639099,
      "fcm_dpo/delta": 0.011283792555332184,
      "fcm_dpo/margin": 60.56721115112305,
      "fcm_dpo/q_t": 0.42008036375045776,
      "grad_norm": 32.360260009765625,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": 1.3759394884109497,
      "logits/rejected": 1.3560971021652222,
      "logps/chosen": -344.52252197265625,
      "logps/ref_chosen": -309.75048828125,
      "logps/ref_rejected": -267.41925048828125,
      "logps/rejected": -362.7585144042969,
      "loss": 4.5258,
      "margin_dpo/margin_mean": 60.56721496582031,
      "margin_dpo/margin_std": 82.38816833496094,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -39.350337982177734,
      "KL/mean": -60.30271911621094,
      "KL/rejected_KL_mean": -81.25511169433594,
      "KL/std": 71.06976318359375,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.0055954898707568645,
      "fcm_dpo/delta": 0.006408221088349819,
      "fcm_dpo/margin": 41.904762268066406,
      "fcm_dpo/q_t": 0.4447755515575409,
      "grad_norm": 38.172176361083984,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 1.4654282331466675,
      "logits/rejected": 1.4464434385299683,
      "logps/chosen": -350.34442138671875,
      "logps/ref_chosen": -310.9941101074219,
      "logps/ref_rejected": -279.94915771484375,
      "logps/rejected": -361.20428466796875,
      "loss": 4.8772,
      "margin_dpo/margin_mean": 41.904762268066406,
      "margin_dpo/margin_std": 80.91134643554688,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -36.82838439941406,
      "KL/mean": -63.239044189453125,
      "KL/rejected_KL_mean": -89.64969635009766,
      "KL/std": 68.57288360595703,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.005790985655039549,
      "fcm_dpo/delta": 0.018819209188222885,
      "fcm_dpo/margin": 52.82130432128906,
      "fcm_dpo/q_t": 0.4281553626060486,
      "grad_norm": 26.66946792602539,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": 1.3604259490966797,
      "logits/rejected": 1.2707215547561646,
      "logps/chosen": -346.0118408203125,
      "logps/ref_chosen": -309.18341064453125,
      "logps/ref_rejected": -256.9963073730469,
      "logps/rejected": -346.64599609375,
      "loss": 4.65,
      "margin_dpo/margin_mean": 52.8213005065918,
      "margin_dpo/margin_std": 83.13489532470703,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -43.051082611083984,
      "KL/mean": -67.50782775878906,
      "KL/rejected_KL_mean": -91.96456909179688,
      "KL/std": 71.51901245117188,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.005666680168360472,
      "fcm_dpo/delta": -0.03883873671293259,
      "fcm_dpo/margin": 48.91350555419922,
      "fcm_dpo/q_t": 0.43454957008361816,
      "grad_norm": 31.340059280395508,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": 1.3264204263687134,
      "logits/rejected": 1.385409951210022,
      "logps/chosen": -325.32623291015625,
      "logps/ref_chosen": -282.275146484375,
      "logps/ref_rejected": -255.17941284179688,
      "logps/rejected": -347.14398193359375,
      "loss": 4.751,
      "margin_dpo/margin_mean": 48.91350555419922,
      "margin_dpo/margin_std": 81.8960189819336,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -37.96763610839844,
      "KL/mean": -61.36542892456055,
      "KL/rejected_KL_mean": -84.76321411132812,
      "KL/std": 67.28466796875,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.005636535119265318,
      "fcm_dpo/delta": 0.035529837012290955,
      "fcm_dpo/margin": 46.79558563232422,
      "fcm_dpo/q_t": 0.4380364418029785,
      "grad_norm": 22.993364334106445,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": 1.2686402797698975,
      "logits/rejected": 1.2992103099822998,
      "logps/chosen": -330.8448486328125,
      "logps/ref_chosen": -292.8772277832031,
      "logps/ref_rejected": -266.78509521484375,
      "logps/rejected": -351.5483093261719,
      "loss": 4.7665,
      "margin_dpo/margin_mean": 46.79558563232422,
      "margin_dpo/margin_std": 80.05794525146484,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -38.31741714477539,
      "KL/mean": -61.478702545166016,
      "KL/rejected_KL_mean": -84.63998413085938,
      "KL/std": 70.8282470703125,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.005793347489088774,
      "fcm_dpo/delta": 0.015086591243743896,
      "fcm_dpo/margin": 46.32257080078125,
      "fcm_dpo/q_t": 0.4368296265602112,
      "grad_norm": 29.06804084777832,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": 1.3224433660507202,
      "logits/rejected": 1.3840413093566895,
      "logps/chosen": -311.21112060546875,
      "logps/ref_chosen": -272.8937072753906,
      "logps/ref_rejected": -293.058349609375,
      "logps/rejected": -377.6983337402344,
      "loss": 4.7572,
      "margin_dpo/margin_mean": 46.322574615478516,
      "margin_dpo/margin_std": 78.14448547363281,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -38.240684509277344,
      "KL/mean": -62.154232025146484,
      "KL/rejected_KL_mean": -86.06777954101562,
      "KL/std": 67.51475524902344,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.005812470801174641,
      "fcm_dpo/delta": 0.018583018332719803,
      "fcm_dpo/margin": 47.82709884643555,
      "fcm_dpo/q_t": 0.43461349606513977,
      "grad_norm": 32.57887268066406,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": 1.4091360569000244,
      "logits/rejected": 1.4208471775054932,
      "logps/chosen": -297.3822937011719,
      "logps/ref_chosen": -259.1416015625,
      "logps/ref_rejected": -229.64996337890625,
      "logps/rejected": -315.7177429199219,
      "loss": 4.7616,
      "margin_dpo/margin_mean": 47.827091217041016,
      "margin_dpo/margin_std": 85.61062622070312,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -44.10288619995117,
      "KL/mean": -69.46588897705078,
      "KL/rejected_KL_mean": -94.82889556884766,
      "KL/std": 71.96964263916016,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.005936335772275925,
      "fcm_dpo/delta": 0.009715357795357704,
      "fcm_dpo/margin": 50.725990295410156,
      "fcm_dpo/q_t": 0.4299638569355011,
      "grad_norm": 54.40616989135742,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 1.2212016582489014,
      "logits/rejected": 1.1446468830108643,
      "logps/chosen": -340.44085693359375,
      "logps/ref_chosen": -296.33795166015625,
      "logps/ref_rejected": -283.7359924316406,
      "logps/rejected": -378.5648498535156,
      "loss": 4.6663,
      "margin_dpo/margin_mean": 50.72599411010742,
      "margin_dpo/margin_std": 80.77369689941406,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -41.09203338623047,
      "KL/mean": -69.4698715209961,
      "KL/rejected_KL_mean": -97.84771728515625,
      "KL/std": 70.52741241455078,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.005972070153802633,
      "fcm_dpo/delta": 0.011249186471104622,
      "fcm_dpo/margin": 56.75566864013672,
      "fcm_dpo/q_t": 0.42043325304985046,
      "grad_norm": 54.88345718383789,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 1.2752745151519775,
      "logits/rejected": 1.3094722032546997,
      "logps/chosen": -290.0652160644531,
      "logps/ref_chosen": -248.97320556640625,
      "logps/ref_rejected": -259.5140075683594,
      "logps/rejected": -357.3617248535156,
      "loss": 4.5217,
      "margin_dpo/margin_mean": 56.75567626953125,
      "margin_dpo/margin_std": 79.07141876220703,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -27.84122085571289,
      "KL/mean": -62.730926513671875,
      "KL/rejected_KL_mean": -97.62063598632812,
      "KL/std": 71.48585510253906,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.005793525837361813,
      "fcm_dpo/delta": -0.08617181330919266,
      "fcm_dpo/margin": 69.77941131591797,
      "fcm_dpo/q_t": 0.40413814783096313,
      "grad_norm": 27.87969398498535,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": 1.3380595445632935,
      "logits/rejected": 1.498769760131836,
      "logps/chosen": -308.51348876953125,
      "logps/ref_chosen": -280.67230224609375,
      "logps/ref_rejected": -272.4359130859375,
      "logps/rejected": -370.0565185546875,
      "loss": 4.3291,
      "margin_dpo/margin_mean": 69.7794189453125,
      "margin_dpo/margin_std": 82.411865234375,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -36.76167297363281,
      "KL/mean": -62.112998962402344,
      "KL/rejected_KL_mean": -87.46432495117188,
      "KL/std": 69.09986877441406,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.005663538351655006,
      "fcm_dpo/delta": 0.02398616261780262,
      "fcm_dpo/margin": 50.702659606933594,
      "fcm_dpo/q_t": 0.4323664605617523,
      "grad_norm": 38.05475616455078,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": 1.0497825145721436,
      "logits/rejected": 1.0988709926605225,
      "logps/chosen": -304.420166015625,
      "logps/ref_chosen": -267.658447265625,
      "logps/ref_rejected": -262.9810791015625,
      "logps/rejected": -350.44537353515625,
      "loss": 4.6929,
      "margin_dpo/margin_mean": 50.70266342163086,
      "margin_dpo/margin_std": 79.7056884765625,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -42.7378044128418,
      "KL/mean": -61.523807525634766,
      "KL/rejected_KL_mean": -80.30979919433594,
      "KL/std": 68.0191650390625,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.005853219423443079,
      "fcm_dpo/delta": 0.04424022138118744,
      "fcm_dpo/margin": 37.57197570800781,
      "fcm_dpo/q_t": 0.448636919260025,
      "grad_norm": 59.32732391357422,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": 1.3517682552337646,
      "logits/rejected": 1.533236026763916,
      "logps/chosen": -308.984375,
      "logps/ref_chosen": -266.2465515136719,
      "logps/ref_rejected": -281.1450500488281,
      "logps/rejected": -361.454833984375,
      "loss": 4.9694,
      "margin_dpo/margin_mean": 37.57197570800781,
      "margin_dpo/margin_std": 85.94178009033203,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -37.99429702758789,
      "KL/mean": -62.98221206665039,
      "KL/rejected_KL_mean": -87.9701156616211,
      "KL/std": 68.75572967529297,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.0060038152150809765,
      "fcm_dpo/delta": 0.023472925648093224,
      "fcm_dpo/margin": 49.97581481933594,
      "fcm_dpo/q_t": 0.42984187602996826,
      "grad_norm": 40.46797180175781,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": 1.3415828943252563,
      "logits/rejected": 1.3292114734649658,
      "logps/chosen": -312.30010986328125,
      "logps/ref_chosen": -274.30584716796875,
      "logps/ref_rejected": -242.94247436523438,
      "logps/rejected": -330.9125671386719,
      "loss": 4.6695,
      "margin_dpo/margin_mean": 49.97581481933594,
      "margin_dpo/margin_std": 79.84535217285156,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -30.443164825439453,
      "KL/mean": -62.472225189208984,
      "KL/rejected_KL_mean": -94.50128173828125,
      "KL/std": 74.74332427978516,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.005792764946818352,
      "fcm_dpo/delta": -0.0587339885532856,
      "fcm_dpo/margin": 64.05811309814453,
      "fcm_dpo/q_t": 0.4118225574493408,
      "grad_norm": 25.488739013671875,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": 1.4536406993865967,
      "logits/rejected": 1.53383469581604,
      "logps/chosen": -339.7096252441406,
      "logps/ref_chosen": -309.2664489746094,
      "logps/ref_rejected": -298.4182434082031,
      "logps/rejected": -392.9195251464844,
      "loss": 4.4449,
      "margin_dpo/margin_mean": 64.05811309814453,
      "margin_dpo/margin_std": 81.68062591552734,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -32.32465744018555,
      "KL/mean": -61.232757568359375,
      "KL/rejected_KL_mean": -90.14086151123047,
      "KL/std": 70.91661071777344,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.0056473808363080025,
      "fcm_dpo/delta": -0.026649873703718185,
      "fcm_dpo/margin": 57.816200256347656,
      "fcm_dpo/q_t": 0.4230951964855194,
      "grad_norm": 39.39658737182617,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 1.3197588920593262,
      "logits/rejected": 1.3651326894760132,
      "logps/chosen": -296.3455810546875,
      "logps/ref_chosen": -264.0209045410156,
      "logps/ref_rejected": -264.3447570800781,
      "logps/rejected": -354.485595703125,
      "loss": 4.586,
      "margin_dpo/margin_mean": 57.81619644165039,
      "margin_dpo/margin_std": 84.36251068115234,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -45.428924560546875,
      "KL/mean": -68.07017517089844,
      "KL/rejected_KL_mean": -90.71142578125,
      "KL/std": 68.8130111694336,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.005607601720839739,
      "fcm_dpo/delta": 0.007387533783912659,
      "fcm_dpo/margin": 45.28251647949219,
      "fcm_dpo/q_t": 0.44032007455825806,
      "grad_norm": 63.666709899902344,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": 1.211143970489502,
      "logits/rejected": 1.2470929622650146,
      "logps/chosen": -308.64776611328125,
      "logps/ref_chosen": -263.2188415527344,
      "logps/ref_rejected": -236.08546447753906,
      "logps/rejected": -326.796875,
      "loss": 4.8372,
      "margin_dpo/margin_mean": 45.28251647949219,
      "margin_dpo/margin_std": 85.51162719726562,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -35.28526306152344,
      "KL/mean": -64.99435424804688,
      "KL/rejected_KL_mean": -94.70346069335938,
      "KL/std": 68.11092376708984,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.0055763330310583115,
      "fcm_dpo/delta": -0.013755641877651215,
      "fcm_dpo/margin": 59.4182014465332,
      "fcm_dpo/q_t": 0.4213353991508484,
      "grad_norm": 30.820510864257812,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": 1.3558927774429321,
      "logits/rejected": 1.3374745845794678,
      "logps/chosen": -309.0831298828125,
      "logps/ref_chosen": -273.79791259765625,
      "logps/ref_rejected": -242.4090118408203,
      "logps/rejected": -337.11248779296875,
      "loss": 4.5563,
      "margin_dpo/margin_mean": 59.4182014465332,
      "margin_dpo/margin_std": 83.64270782470703,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -36.946449279785156,
      "KL/mean": -65.11579132080078,
      "KL/rejected_KL_mean": -93.28514099121094,
      "KL/std": 68.25302124023438,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.005586415994912386,
      "fcm_dpo/delta": -0.019411765038967133,
      "fcm_dpo/margin": 56.33869934082031,
      "fcm_dpo/q_t": 0.4254843592643738,
      "grad_norm": 37.85884475708008,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 1.2012146711349487,
      "logits/rejected": 1.2611706256866455,
      "logps/chosen": -312.2616271972656,
      "logps/ref_chosen": -275.3151550292969,
      "logps/ref_rejected": -278.1514587402344,
      "logps/rejected": -371.4366149902344,
      "loss": 4.6382,
      "margin_dpo/margin_mean": 56.33869934082031,
      "margin_dpo/margin_std": 88.35138702392578,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -43.120975494384766,
      "KL/mean": -70.33393096923828,
      "KL/rejected_KL_mean": -97.54689025878906,
      "KL/std": 72.00347137451172,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.005473090335726738,
      "fcm_dpo/delta": -0.0095901470631361,
      "fcm_dpo/margin": 54.42591857910156,
      "fcm_dpo/q_t": 0.42951124906539917,
      "grad_norm": 27.182844161987305,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.3778250217437744,
      "logits/rejected": 1.3679306507110596,
      "logps/chosen": -318.3203125,
      "logps/ref_chosen": -275.1993408203125,
      "logps/ref_rejected": -261.32122802734375,
      "logps/rejected": -358.86810302734375,
      "loss": 4.6234,
      "margin_dpo/margin_mean": 54.425926208496094,
      "margin_dpo/margin_std": 75.29243469238281,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -36.23802947998047,
      "KL/mean": -71.00995635986328,
      "KL/rejected_KL_mean": -105.78189086914062,
      "KL/std": 71.94034576416016,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.005281152203679085,
      "fcm_dpo/delta": -0.054653629660606384,
      "fcm_dpo/margin": 69.54388427734375,
      "fcm_dpo/q_t": 0.41238948702812195,
      "grad_norm": 26.99108123779297,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": 1.3982120752334595,
      "logits/rejected": 1.4245784282684326,
      "logps/chosen": -336.0657958984375,
      "logps/ref_chosen": -299.8277587890625,
      "logps/ref_rejected": -251.24798583984375,
      "logps/rejected": -357.0298767089844,
      "loss": 4.3805,
      "margin_dpo/margin_mean": 69.54388427734375,
      "margin_dpo/margin_std": 76.20675659179688,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -35.843910217285156,
      "KL/mean": -61.08030700683594,
      "KL/rejected_KL_mean": -86.31669616699219,
      "KL/std": 69.33416748046875,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.005308274645358324,
      "fcm_dpo/delta": 0.020334182307124138,
      "fcm_dpo/margin": 50.47279357910156,
      "fcm_dpo/q_t": 0.43645262718200684,
      "grad_norm": 26.386720657348633,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": 1.1161525249481201,
      "logits/rejected": 1.075589895248413,
      "logps/chosen": -316.3050231933594,
      "logps/ref_chosen": -280.4610900878906,
      "logps/ref_rejected": -228.34823608398438,
      "logps/rejected": -314.6649169921875,
      "loss": 4.7461,
      "margin_dpo/margin_mean": 50.47279357910156,
      "margin_dpo/margin_std": 83.97797393798828,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -30.913042068481445,
      "KL/mean": -55.839378356933594,
      "KL/rejected_KL_mean": -80.76571655273438,
      "KL/std": 68.41380310058594,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.005328113678842783,
      "fcm_dpo/delta": 0.005002221092581749,
      "fcm_dpo/margin": 49.8526611328125,
      "fcm_dpo/q_t": 0.4372060298919678,
      "grad_norm": 39.97221755981445,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": 1.2867475748062134,
      "logits/rejected": 1.389235258102417,
      "logps/chosen": -350.51312255859375,
      "logps/ref_chosen": -319.60009765625,
      "logps/ref_rejected": -316.42822265625,
      "logps/rejected": -397.1939697265625,
      "loss": 4.7397,
      "margin_dpo/margin_mean": 49.852657318115234,
      "margin_dpo/margin_std": 78.91390228271484,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -40.99460220336914,
      "KL/mean": -67.2310791015625,
      "KL/rejected_KL_mean": -93.46754455566406,
      "KL/std": 67.73291778564453,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.005368333775550127,
      "fcm_dpo/delta": 0.014917712658643723,
      "fcm_dpo/margin": 52.472930908203125,
      "fcm_dpo/q_t": 0.43327516317367554,
      "grad_norm": 26.797521591186523,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": 1.141771674156189,
      "logits/rejected": 1.266058087348938,
      "logps/chosen": -301.41241455078125,
      "logps/ref_chosen": -260.4178466796875,
      "logps/ref_rejected": -270.04498291015625,
      "logps/rejected": -363.5125732421875,
      "loss": 4.6809,
      "margin_dpo/margin_mean": 52.472938537597656,
      "margin_dpo/margin_std": 77.29425048828125,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -41.14960479736328,
      "KL/mean": -67.41556549072266,
      "KL/rejected_KL_mean": -93.6815185546875,
      "KL/std": 70.39624786376953,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.0054582152515649796,
      "fcm_dpo/delta": 0.014819440431892872,
      "fcm_dpo/margin": 52.53190994262695,
      "fcm_dpo/q_t": 0.4319712221622467,
      "grad_norm": 24.582075119018555,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": 1.350229024887085,
      "logits/rejected": 1.3434491157531738,
      "logps/chosen": -321.8526611328125,
      "logps/ref_chosen": -280.7030029296875,
      "logps/ref_rejected": -250.76385498046875,
      "logps/rejected": -344.44537353515625,
      "loss": 4.6953,
      "margin_dpo/margin_mean": 52.53190994262695,
      "margin_dpo/margin_std": 84.07726287841797,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -34.80934143066406,
      "KL/mean": -60.1241340637207,
      "KL/rejected_KL_mean": -85.43893432617188,
      "KL/std": 71.42765808105469,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.005498247686773539,
      "fcm_dpo/delta": -0.003842413891106844,
      "fcm_dpo/margin": 50.62960433959961,
      "fcm_dpo/q_t": 0.43376344442367554,
      "grad_norm": 22.205915451049805,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": 1.2686049938201904,
      "logits/rejected": 1.3858766555786133,
      "logps/chosen": -335.69580078125,
      "logps/ref_chosen": -300.8864440917969,
      "logps/ref_rejected": -288.5682067871094,
      "logps/rejected": -374.00714111328125,
      "loss": 4.7267,
      "margin_dpo/margin_mean": 50.62960433959961,
      "margin_dpo/margin_std": 80.09807586669922,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -49.70215606689453,
      "KL/mean": -69.22557067871094,
      "KL/rejected_KL_mean": -88.74899291992188,
      "KL/std": 71.97066497802734,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.005459655076265335,
      "fcm_dpo/delta": 0.009212011471390724,
      "fcm_dpo/margin": 39.04682922363281,
      "fcm_dpo/q_t": 0.44997137784957886,
      "grad_norm": 36.091156005859375,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": 1.334824800491333,
      "logits/rejected": 1.4034242630004883,
      "logps/chosen": -343.0045166015625,
      "logps/ref_chosen": -293.3023681640625,
      "logps/ref_rejected": -270.0457763671875,
      "logps/rejected": -358.7947692871094,
      "loss": 4.9892,
      "margin_dpo/margin_mean": 39.04682540893555,
      "margin_dpo/margin_std": 91.44847106933594,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -43.69807815551758,
      "KL/mean": -67.76571655273438,
      "KL/rejected_KL_mean": -91.8333740234375,
      "KL/std": 72.28323364257812,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.005612127482891083,
      "fcm_dpo/delta": 0.05199463665485382,
      "fcm_dpo/margin": 48.13529968261719,
      "fcm_dpo/q_t": 0.4363265335559845,
      "grad_norm": 34.23748779296875,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": 1.2788867950439453,
      "logits/rejected": 1.3483338356018066,
      "logps/chosen": -327.0973205566406,
      "logps/ref_chosen": -283.39923095703125,
      "logps/ref_rejected": -264.5092468261719,
      "logps/rejected": -356.3426208496094,
      "loss": 4.7445,
      "margin_dpo/margin_mean": 48.13529968261719,
      "margin_dpo/margin_std": 78.98506164550781,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -38.25552749633789,
      "KL/mean": -65.82939147949219,
      "KL/rejected_KL_mean": -93.40324401855469,
      "KL/std": 68.00537872314453,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.00563463382422924,
      "fcm_dpo/delta": -0.03360137343406677,
      "fcm_dpo/margin": 55.14772033691406,
      "fcm_dpo/q_t": 0.426321417093277,
      "grad_norm": 34.30907440185547,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 1.3098801374435425,
      "logits/rejected": 1.3792104721069336,
      "logps/chosen": -292.7233581542969,
      "logps/ref_chosen": -254.4678192138672,
      "logps/ref_rejected": -252.00326538085938,
      "logps/rejected": -345.4065246582031,
      "loss": 4.617,
      "margin_dpo/margin_mean": 55.14772033691406,
      "margin_dpo/margin_std": 80.21300506591797,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_KL/chosen_KL_mean": -42.7293815612793,
      "eval_KL/mean": -68.3998031616211,
      "eval_KL/rejected_KL_mean": -94.07023620605469,
      "eval_KL/std": 69.59201049804688,
      "eval_fcm_dpo/beta": 0.005573027301579714,
      "eval_logits/chosen": 1.3840314149856567,
      "eval_logits/rejected": 1.4393243789672852,
      "eval_logps/chosen": -323.14605712890625,
      "eval_logps/ref_chosen": -280.41668701171875,
      "eval_logps/ref_rejected": -264.6455383300781,
      "eval_logps/rejected": -358.7157897949219,
      "eval_loss": 0.5890279412269592,
      "eval_margin_dpo/margin_mean": 51.34084701538086,
      "eval_margin_dpo/margin_std": 80.25874328613281,
      "eval_runtime": 88.6098,
      "eval_samples_per_second": 22.571,
      "eval_steps_per_second": 2.821,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -35.498653411865234,
      "KL/mean": -68.64216613769531,
      "KL/rejected_KL_mean": -101.78569030761719,
      "KL/std": 72.96342468261719,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.005540897138416767,
      "fcm_dpo/delta": -0.018198324367403984,
      "fcm_dpo/margin": 66.28703308105469,
      "fcm_dpo/q_t": 0.41273239254951477,
      "grad_norm": 29.90237808227539,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 1.0909414291381836,
      "logits/rejected": 1.1164941787719727,
      "logps/chosen": -346.09820556640625,
      "logps/ref_chosen": -310.5995788574219,
      "logps/ref_rejected": -292.7754821777344,
      "logps/rejected": -394.5611572265625,
      "loss": 4.4073,
      "margin_dpo/margin_mean": 66.28702545166016,
      "margin_dpo/margin_std": 79.53611755371094,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -37.211639404296875,
      "KL/mean": -63.70612335205078,
      "KL/rejected_KL_mean": -90.20060729980469,
      "KL/std": 71.20449829101562,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.005504201166331768,
      "fcm_dpo/delta": 0.012362987734377384,
      "fcm_dpo/margin": 52.98896026611328,
      "fcm_dpo/q_t": 0.43093177676200867,
      "grad_norm": 40.60675811767578,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": 1.4223815202713013,
      "logits/rejected": 1.351109266281128,
      "logps/chosen": -337.0630187988281,
      "logps/ref_chosen": -299.8513488769531,
      "logps/ref_rejected": -278.5664367675781,
      "logps/rejected": -368.76702880859375,
      "loss": 4.6789,
      "margin_dpo/margin_mean": 52.98896026611328,
      "margin_dpo/margin_std": 83.13359832763672,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -31.814498901367188,
      "KL/mean": -68.37123107910156,
      "KL/rejected_KL_mean": -104.9279556274414,
      "KL/std": 73.60391235351562,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.0054693943820893764,
      "fcm_dpo/delta": -0.05192917585372925,
      "fcm_dpo/margin": 73.11347198486328,
      "fcm_dpo/q_t": 0.4047307074069977,
      "grad_norm": 24.207406997680664,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": 1.2606215476989746,
      "logits/rejected": 1.2944611310958862,
      "logps/chosen": -334.2461242675781,
      "logps/ref_chosen": -302.4316101074219,
      "logps/ref_rejected": -270.951171875,
      "logps/rejected": -375.8791198730469,
      "loss": 4.2713,
      "margin_dpo/margin_mean": 73.11347198486328,
      "margin_dpo/margin_std": 73.01643371582031,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -46.126129150390625,
      "KL/mean": -71.71871185302734,
      "KL/rejected_KL_mean": -97.31130981445312,
      "KL/std": 73.84769439697266,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.005328364670276642,
      "fcm_dpo/delta": 0.01846805214881897,
      "fcm_dpo/margin": 51.18518829345703,
      "fcm_dpo/q_t": 0.43553632497787476,
      "grad_norm": 26.047636032104492,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": 1.2064199447631836,
      "logits/rejected": 1.2344309091567993,
      "logps/chosen": -336.02923583984375,
      "logps/ref_chosen": -289.9031066894531,
      "logps/ref_rejected": -252.84918212890625,
      "logps/rejected": -350.16046142578125,
      "loss": 4.7517,
      "margin_dpo/margin_mean": 51.18518829345703,
      "margin_dpo/margin_std": 85.69363403320312,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -36.63838195800781,
      "KL/mean": -66.2357177734375,
      "KL/rejected_KL_mean": -95.83305358886719,
      "KL/std": 72.18719482421875,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.005388339981436729,
      "fcm_dpo/delta": -0.006856870371848345,
      "fcm_dpo/margin": 59.194664001464844,
      "fcm_dpo/q_t": 0.42445388436317444,
      "grad_norm": 29.564132690429688,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": 1.3807421922683716,
      "logits/rejected": 1.5665624141693115,
      "logps/chosen": -284.81243896484375,
      "logps/ref_chosen": -248.17404174804688,
      "logps/ref_rejected": -252.23764038085938,
      "logps/rejected": -348.0707092285156,
      "loss": 4.5559,
      "margin_dpo/margin_mean": 59.194664001464844,
      "margin_dpo/margin_std": 78.19025421142578,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -42.89390563964844,
      "KL/mean": -67.67750549316406,
      "KL/rejected_KL_mean": -92.46110534667969,
      "KL/std": 72.16992950439453,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.005335635971277952,
      "fcm_dpo/delta": -0.008822361007332802,
      "fcm_dpo/margin": 49.56720733642578,
      "fcm_dpo/q_t": 0.43720918893814087,
      "grad_norm": 35.899635314941406,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": 1.3088067770004272,
      "logits/rejected": 1.3818573951721191,
      "logps/chosen": -319.31005859375,
      "logps/ref_chosen": -276.4161376953125,
      "logps/ref_rejected": -264.0082702636719,
      "logps/rejected": -356.4693908691406,
      "loss": 4.7742,
      "margin_dpo/margin_mean": 49.56720733642578,
      "margin_dpo/margin_std": 84.17759704589844,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -51.935054779052734,
      "KL/mean": -79.2212142944336,
      "KL/rejected_KL_mean": -106.50735473632812,
      "KL/std": 74.94820404052734,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.005469411611557007,
      "fcm_dpo/delta": 0.05280398204922676,
      "fcm_dpo/margin": 54.57229995727539,
      "fcm_dpo/q_t": 0.4299343526363373,
      "grad_norm": 31.544599533081055,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": 1.2408838272094727,
      "logits/rejected": 1.2357468605041504,
      "logps/chosen": -309.3724365234375,
      "logps/ref_chosen": -257.4373474121094,
      "logps/ref_rejected": -219.66390991210938,
      "logps/rejected": -326.1712646484375,
      "loss": 4.6483,
      "margin_dpo/margin_mean": 54.57229995727539,
      "margin_dpo/margin_std": 83.55366516113281,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -38.429691314697266,
      "KL/mean": -66.73556518554688,
      "KL/rejected_KL_mean": -95.04145050048828,
      "KL/std": 70.68415069580078,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.0055139982141554356,
      "fcm_dpo/delta": -0.03153260424733162,
      "fcm_dpo/margin": 56.611759185791016,
      "fcm_dpo/q_t": 0.42566677927970886,
      "grad_norm": 30.387678146362305,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": 1.312660574913025,
      "logits/rejected": 1.4376811981201172,
      "logps/chosen": -327.08306884765625,
      "logps/ref_chosen": -288.6533508300781,
      "logps/ref_rejected": -291.54107666015625,
      "logps/rejected": -386.5825500488281,
      "loss": 4.6009,
      "margin_dpo/margin_mean": 56.61176300048828,
      "margin_dpo/margin_std": 78.86919403076172,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -42.250762939453125,
      "KL/mean": -71.6051025390625,
      "KL/rejected_KL_mean": -100.95942687988281,
      "KL/std": 73.20411682128906,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.005356629844754934,
      "fcm_dpo/delta": -0.01752406544983387,
      "fcm_dpo/margin": 58.708656311035156,
      "fcm_dpo/q_t": 0.4257467985153198,
      "grad_norm": 32.193687438964844,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": 1.4274879693984985,
      "logits/rejected": 1.539833664894104,
      "logps/chosen": -319.9881591796875,
      "logps/ref_chosen": -277.7373962402344,
      "logps/ref_rejected": -272.988525390625,
      "logps/rejected": -373.9479064941406,
      "loss": 4.6173,
      "margin_dpo/margin_mean": 58.708656311035156,
      "margin_dpo/margin_std": 88.89273071289062,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -41.67294692993164,
      "KL/mean": -67.48551177978516,
      "KL/rejected_KL_mean": -93.29808044433594,
      "KL/std": 73.20637512207031,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.005431816913187504,
      "fcm_dpo/delta": 0.018092650920152664,
      "fcm_dpo/margin": 51.62512969970703,
      "fcm_dpo/q_t": 0.4338025152683258,
      "grad_norm": 41.33970642089844,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 1.5481975078582764,
      "logits/rejected": 1.528673529624939,
      "logps/chosen": -315.89404296875,
      "logps/ref_chosen": -274.2210998535156,
      "logps/ref_rejected": -264.33465576171875,
      "logps/rejected": -357.63275146484375,
      "loss": 4.7292,
      "margin_dpo/margin_mean": 51.62513732910156,
      "margin_dpo/margin_std": 86.80851745605469,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -35.78733444213867,
      "KL/mean": -65.7760009765625,
      "KL/rejected_KL_mean": -95.76467895507812,
      "KL/std": 72.82067108154297,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.0053490702994167805,
      "fcm_dpo/delta": -0.03525146469473839,
      "fcm_dpo/margin": 59.97734451293945,
      "fcm_dpo/q_t": 0.42362335324287415,
      "grad_norm": 37.033226013183594,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 1.3024818897247314,
      "logits/rejected": 1.3614189624786377,
      "logps/chosen": -308.6553039550781,
      "logps/ref_chosen": -272.86798095703125,
      "logps/ref_rejected": -250.35772705078125,
      "logps/rejected": -346.1224060058594,
      "loss": 4.5457,
      "margin_dpo/margin_mean": 59.97734451293945,
      "margin_dpo/margin_std": 74.9586410522461,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -50.56735610961914,
      "KL/mean": -75.27664947509766,
      "KL/rejected_KL_mean": -99.98595428466797,
      "KL/std": 73.76202392578125,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.00523147638887167,
      "fcm_dpo/delta": 0.008719928562641144,
      "fcm_dpo/margin": 49.418601989746094,
      "fcm_dpo/q_t": 0.4388090968132019,
      "grad_norm": 32.46539306640625,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": 1.1188397407531738,
      "logits/rejected": 1.2208689451217651,
      "logps/chosen": -318.4865417480469,
      "logps/ref_chosen": -267.919189453125,
      "logps/ref_rejected": -271.73162841796875,
      "logps/rejected": -371.71759033203125,
      "loss": 4.7935,
      "margin_dpo/margin_mean": 49.418601989746094,
      "margin_dpo/margin_std": 86.89283752441406,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -39.005008697509766,
      "KL/mean": -62.1069450378418,
      "KL/rejected_KL_mean": -85.2088851928711,
      "KL/std": 69.21144104003906,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.0053558265790343285,
      "fcm_dpo/delta": 0.03210192918777466,
      "fcm_dpo/margin": 46.20387268066406,
      "fcm_dpo/q_t": 0.44155871868133545,
      "grad_norm": 35.81843948364258,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 1.2362796068191528,
      "logits/rejected": 1.2924168109893799,
      "logps/chosen": -303.96307373046875,
      "logps/ref_chosen": -264.95806884765625,
      "logps/ref_rejected": -267.40093994140625,
      "logps/rejected": -352.6098327636719,
      "loss": 4.7946,
      "margin_dpo/margin_mean": 46.20386505126953,
      "margin_dpo/margin_std": 78.81336975097656,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -46.64021682739258,
      "KL/mean": -73.34260559082031,
      "KL/rejected_KL_mean": -100.04501342773438,
      "KL/std": 73.59876251220703,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.005493511445820332,
      "fcm_dpo/delta": 0.0005182831082493067,
      "fcm_dpo/margin": 53.40479278564453,
      "fcm_dpo/q_t": 0.4311297833919525,
      "grad_norm": 32.898860931396484,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": 1.4168510437011719,
      "logits/rejected": 1.4898626804351807,
      "logps/chosen": -336.6459655761719,
      "logps/ref_chosen": -290.00579833984375,
      "logps/ref_rejected": -266.8810119628906,
      "logps/rejected": -366.926025390625,
      "loss": 4.7106,
      "margin_dpo/margin_mean": 53.404788970947266,
      "margin_dpo/margin_std": 89.6422348022461,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -34.73671340942383,
      "KL/mean": -68.11457824707031,
      "KL/rejected_KL_mean": -101.49247741699219,
      "KL/std": 72.51370239257812,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.00547402910888195,
      "fcm_dpo/delta": -0.016184471547603607,
      "fcm_dpo/margin": 66.7557601928711,
      "fcm_dpo/q_t": 0.4137199819087982,
      "grad_norm": 34.1402473449707,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": 1.353202223777771,
      "logits/rejected": 1.446688175201416,
      "logps/chosen": -324.5477294921875,
      "logps/ref_chosen": -289.81103515625,
      "logps/ref_rejected": -278.2758483886719,
      "logps/rejected": -379.7683410644531,
      "loss": 4.433,
      "margin_dpo/margin_mean": 66.75576782226562,
      "margin_dpo/margin_std": 84.3497314453125,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -35.27904510498047,
      "KL/mean": -66.72291564941406,
      "KL/rejected_KL_mean": -98.16680908203125,
      "KL/std": 71.68118286132812,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.005272617097944021,
      "fcm_dpo/delta": -0.013310473412275314,
      "fcm_dpo/margin": 62.88777542114258,
      "fcm_dpo/q_t": 0.42120295763015747,
      "grad_norm": 40.73047637939453,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": 1.5274901390075684,
      "logits/rejected": 1.5188060998916626,
      "logps/chosen": -339.9278564453125,
      "logps/ref_chosen": -304.6488037109375,
      "logps/ref_rejected": -264.2703857421875,
      "logps/rejected": -362.43719482421875,
      "loss": 4.5408,
      "margin_dpo/margin_mean": 62.887779235839844,
      "margin_dpo/margin_std": 85.11808013916016,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -41.2015266418457,
      "KL/mean": -68.10173797607422,
      "KL/rejected_KL_mean": -95.001953125,
      "KL/std": 70.61921691894531,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.00529951136559248,
      "fcm_dpo/delta": -0.018041031435132027,
      "fcm_dpo/margin": 53.80043029785156,
      "fcm_dpo/q_t": 0.4322735071182251,
      "grad_norm": 48.594482421875,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": 1.4089438915252686,
      "logits/rejected": 1.4998588562011719,
      "logps/chosen": -343.0904235839844,
      "logps/ref_chosen": -301.8889465332031,
      "logps/ref_rejected": -275.0347595214844,
      "logps/rejected": -370.0367126464844,
      "loss": 4.7004,
      "margin_dpo/margin_mean": 53.80043029785156,
      "margin_dpo/margin_std": 83.8715591430664,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -51.90007781982422,
      "KL/mean": -74.42992401123047,
      "KL/rejected_KL_mean": -96.95977020263672,
      "KL/std": 71.92178344726562,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.005287775304168463,
      "fcm_dpo/delta": 0.002522658556699753,
      "fcm_dpo/margin": 45.0596923828125,
      "fcm_dpo/q_t": 0.4432629644870758,
      "grad_norm": 47.923439025878906,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": 1.0399930477142334,
      "logits/rejected": 1.0536653995513916,
      "logps/chosen": -321.1474304199219,
      "logps/ref_chosen": -269.2473449707031,
      "logps/ref_rejected": -275.5192565917969,
      "logps/rejected": -372.47900390625,
      "loss": 4.8414,
      "margin_dpo/margin_mean": 45.0596923828125,
      "margin_dpo/margin_std": 77.64122772216797,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -47.26727294921875,
      "KL/mean": -79.30509948730469,
      "KL/rejected_KL_mean": -111.34295654296875,
      "KL/std": 77.29688262939453,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.005187645088881254,
      "fcm_dpo/delta": -0.029406944289803505,
      "fcm_dpo/margin": 64.07568359375,
      "fcm_dpo/q_t": 0.42116132378578186,
      "grad_norm": 35.79600524902344,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": 1.2975807189941406,
      "logits/rejected": 1.2558884620666504,
      "logps/chosen": -338.42767333984375,
      "logps/ref_chosen": -291.1604309082031,
      "logps/ref_rejected": -245.18325805664062,
      "logps/rejected": -356.52618408203125,
      "loss": 4.5582,
      "margin_dpo/margin_mean": 64.07569122314453,
      "margin_dpo/margin_std": 91.10173797607422,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -43.4483757019043,
      "KL/mean": -74.30535125732422,
      "KL/rejected_KL_mean": -105.16233825683594,
      "KL/std": 74.90989685058594,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.005147272255271673,
      "fcm_dpo/delta": 0.010556086897850037,
      "fcm_dpo/margin": 61.71396255493164,
      "fcm_dpo/q_t": 0.4242892563343048,
      "grad_norm": 25.23750877380371,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 1.325177550315857,
      "logits/rejected": 1.299337387084961,
      "logps/chosen": -301.918212890625,
      "logps/ref_chosen": -258.4698181152344,
      "logps/ref_rejected": -242.916015625,
      "logps/rejected": -348.078369140625,
      "loss": 4.5725,
      "margin_dpo/margin_mean": 61.713958740234375,
      "margin_dpo/margin_std": 85.43513488769531,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -34.4514274597168,
      "KL/mean": -66.5307846069336,
      "KL/rejected_KL_mean": -98.61013793945312,
      "KL/std": 70.63162994384766,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.005150447599589825,
      "fcm_dpo/delta": -0.006944411899894476,
      "fcm_dpo/margin": 64.15872192382812,
      "fcm_dpo/q_t": 0.42111730575561523,
      "grad_norm": 36.054229736328125,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.0927780866622925,
      "logits/rejected": 1.065101146697998,
      "logps/chosen": -324.5921936035156,
      "logps/ref_chosen": -290.1407775878906,
      "logps/ref_rejected": -239.65676879882812,
      "logps/rejected": -338.2668762207031,
      "loss": 4.5096,
      "margin_dpo/margin_mean": 64.15872192382812,
      "margin_dpo/margin_std": 80.43748474121094,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -45.29853057861328,
      "KL/mean": -68.9530029296875,
      "KL/rejected_KL_mean": -92.60749053955078,
      "KL/std": 72.46685028076172,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.005250962451100349,
      "fcm_dpo/delta": 0.033232733607292175,
      "fcm_dpo/margin": 47.30894470214844,
      "fcm_dpo/q_t": 0.441007137298584,
      "grad_norm": 38.256168365478516,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": 1.1087309122085571,
      "logits/rejected": 1.1704410314559937,
      "logps/chosen": -322.23455810546875,
      "logps/ref_chosen": -276.93603515625,
      "logps/ref_rejected": -250.398681640625,
      "logps/rejected": -343.0061340332031,
      "loss": 4.8394,
      "margin_dpo/margin_mean": 47.30894470214844,
      "margin_dpo/margin_std": 88.87457275390625,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -38.38960647583008,
      "KL/mean": -70.86959838867188,
      "KL/rejected_KL_mean": -103.3495864868164,
      "KL/std": 74.49049377441406,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.005291205830872059,
      "fcm_dpo/delta": 0.006248156540095806,
      "fcm_dpo/margin": 64.9599838256836,
      "fcm_dpo/q_t": 0.41880571842193604,
      "grad_norm": 35.17270278930664,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": 1.355472445487976,
      "logits/rejected": 1.4541540145874023,
      "logps/chosen": -323.8197021484375,
      "logps/ref_chosen": -285.43011474609375,
      "logps/ref_rejected": -258.0743408203125,
      "logps/rejected": -361.4239501953125,
      "loss": 4.5173,
      "margin_dpo/margin_mean": 64.9599838256836,
      "margin_dpo/margin_std": 89.53885650634766,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -33.75310516357422,
      "KL/mean": -64.98143005371094,
      "KL/rejected_KL_mean": -96.20975494384766,
      "KL/std": 70.27592468261719,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.00529376370832324,
      "fcm_dpo/delta": -0.006104571744799614,
      "fcm_dpo/margin": 62.45664596557617,
      "fcm_dpo/q_t": 0.4211070239543915,
      "grad_norm": 26.905784606933594,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": 1.188397765159607,
      "logits/rejected": 1.1710264682769775,
      "logps/chosen": -333.0981140136719,
      "logps/ref_chosen": -299.3450012207031,
      "logps/ref_rejected": -244.60169982910156,
      "logps/rejected": -340.81146240234375,
      "loss": 4.5073,
      "margin_dpo/margin_mean": 62.45664596557617,
      "margin_dpo/margin_std": 77.17847442626953,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -40.495521545410156,
      "KL/mean": -67.9002914428711,
      "KL/rejected_KL_mean": -95.30506134033203,
      "KL/std": 74.1609878540039,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.005227731540799141,
      "fcm_dpo/delta": -0.03199569135904312,
      "fcm_dpo/margin": 54.809547424316406,
      "fcm_dpo/q_t": 0.43170714378356934,
      "grad_norm": 39.35697937011719,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": 1.3326420783996582,
      "logits/rejected": 1.4264705181121826,
      "logps/chosen": -313.44525146484375,
      "logps/ref_chosen": -272.9497375488281,
      "logps/ref_rejected": -274.6335754394531,
      "logps/rejected": -369.93865966796875,
      "loss": 4.6789,
      "margin_dpo/margin_mean": 54.80955123901367,
      "margin_dpo/margin_std": 80.52661895751953,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -35.716102600097656,
      "KL/mean": -64.99486541748047,
      "KL/rejected_KL_mean": -94.27362060546875,
      "KL/std": 71.96977233886719,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.005180190317332745,
      "fcm_dpo/delta": 0.0010611237958073616,
      "fcm_dpo/margin": 58.557525634765625,
      "fcm_dpo/q_t": 0.4276956021785736,
      "grad_norm": 25.219337463378906,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 1.218462347984314,
      "logits/rejected": 1.2576265335083008,
      "logps/chosen": -317.65765380859375,
      "logps/ref_chosen": -281.9415588378906,
      "logps/ref_rejected": -253.20486450195312,
      "logps/rejected": -347.4784851074219,
      "loss": 4.5894,
      "margin_dpo/margin_mean": 58.557525634765625,
      "margin_dpo/margin_std": 77.62861633300781,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -32.8505859375,
      "KL/mean": -71.90770721435547,
      "KL/rejected_KL_mean": -110.96482849121094,
      "KL/std": 74.88786315917969,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.005070388782769442,
      "fcm_dpo/delta": -0.07562313973903656,
      "fcm_dpo/margin": 78.11422729492188,
      "fcm_dpo/q_t": 0.4057159125804901,
      "grad_norm": 33.35033416748047,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": 1.318666934967041,
      "logits/rejected": 1.3167715072631836,
      "logps/chosen": -313.4671936035156,
      "logps/ref_chosen": -280.6166076660156,
      "logps/ref_rejected": -263.7532958984375,
      "logps/rejected": -374.7181396484375,
      "loss": 4.2949,
      "margin_dpo/margin_mean": 78.11421966552734,
      "margin_dpo/margin_std": 79.25650024414062,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -40.636322021484375,
      "KL/mean": -77.56893920898438,
      "KL/rejected_KL_mean": -114.50157165527344,
      "KL/std": 74.91705322265625,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.004884055815637112,
      "fcm_dpo/delta": -0.011564400047063828,
      "fcm_dpo/margin": 73.86524963378906,
      "fcm_dpo/q_t": 0.4136756956577301,
      "grad_norm": 36.071861267089844,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": 1.4763782024383545,
      "logits/rejected": 1.609140396118164,
      "logps/chosen": -326.798583984375,
      "logps/ref_chosen": -286.1622314453125,
      "logps/ref_rejected": -273.3803405761719,
      "logps/rejected": -387.8819274902344,
      "loss": 4.3889,
      "margin_dpo/margin_mean": 73.86524963378906,
      "margin_dpo/margin_std": 78.16394805908203,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -41.29792404174805,
      "KL/mean": -70.08042907714844,
      "KL/rejected_KL_mean": -98.86293029785156,
      "KL/std": 72.54641723632812,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.004791309125721455,
      "fcm_dpo/delta": 0.004546597599983215,
      "fcm_dpo/margin": 57.56501770019531,
      "fcm_dpo/q_t": 0.4345035254955292,
      "grad_norm": 29.696998596191406,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": 1.1684305667877197,
      "logits/rejected": 1.1691198348999023,
      "logps/chosen": -307.6344909667969,
      "logps/ref_chosen": -266.3365783691406,
      "logps/ref_rejected": -255.01956176757812,
      "logps/rejected": -353.88250732421875,
      "loss": 4.7395,
      "margin_dpo/margin_mean": 57.56501770019531,
      "margin_dpo/margin_std": 94.97341918945312,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -41.969024658203125,
      "KL/mean": -67.02253723144531,
      "KL/rejected_KL_mean": -92.0760498046875,
      "KL/std": 72.3785171508789,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.0048641664907336235,
      "fcm_dpo/delta": 0.0007863901555538177,
      "fcm_dpo/margin": 50.10704040527344,
      "fcm_dpo/q_t": 0.44190341234207153,
      "grad_norm": 44.90932083129883,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 1.269547700881958,
      "logits/rejected": 1.2997703552246094,
      "logps/chosen": -324.12042236328125,
      "logps/ref_chosen": -282.1513977050781,
      "logps/ref_rejected": -263.4792175292969,
      "logps/rejected": -355.5553283691406,
      "loss": 4.8322,
      "margin_dpo/margin_mean": 50.10703659057617,
      "margin_dpo/margin_std": 88.4993667602539,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -45.346435546875,
      "KL/mean": -68.88236236572266,
      "KL/rejected_KL_mean": -92.41828155517578,
      "KL/std": 72.60649108886719,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.004954863805323839,
      "fcm_dpo/delta": 0.04032646119594574,
      "fcm_dpo/margin": 47.07183837890625,
      "fcm_dpo/q_t": 0.44470804929733276,
      "grad_norm": 31.76032829284668,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 1.2593644857406616,
      "logits/rejected": 1.4216034412384033,
      "logps/chosen": -314.0595703125,
      "logps/ref_chosen": -268.713134765625,
      "logps/ref_rejected": -266.13446044921875,
      "logps/rejected": -358.5527648925781,
      "loss": 4.8402,
      "margin_dpo/margin_mean": 47.07183837890625,
      "margin_dpo/margin_std": 82.95166015625,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -33.48515701293945,
      "KL/mean": -64.91841888427734,
      "KL/rejected_KL_mean": -96.35165405273438,
      "KL/std": 72.66751861572266,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.004972536116838455,
      "fcm_dpo/delta": 0.008515238761901855,
      "fcm_dpo/margin": 62.86650848388672,
      "fcm_dpo/q_t": 0.42573660612106323,
      "grad_norm": 22.97835922241211,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": 1.3626418113708496,
      "logits/rejected": 1.3987596035003662,
      "logps/chosen": -317.4693908691406,
      "logps/ref_chosen": -283.9842224121094,
      "logps/ref_rejected": -253.9568634033203,
      "logps/rejected": -350.3085632324219,
      "loss": 4.5743,
      "margin_dpo/margin_mean": 62.866512298583984,
      "margin_dpo/margin_std": 84.69027709960938,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -51.54993438720703,
      "KL/mean": -76.5881118774414,
      "KL/rejected_KL_mean": -101.62628173828125,
      "KL/std": 72.17973327636719,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.005196927115321159,
      "fcm_dpo/delta": 0.07396793365478516,
      "fcm_dpo/margin": 50.07634353637695,
      "fcm_dpo/q_t": 0.43805941939353943,
      "grad_norm": 30.271202087402344,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": 1.0348814725875854,
      "logits/rejected": 1.0589193105697632,
      "logps/chosen": -293.9931335449219,
      "logps/ref_chosen": -242.4431915283203,
      "logps/ref_rejected": -252.87979125976562,
      "logps/rejected": -354.50604248046875,
      "loss": 4.7705,
      "margin_dpo/margin_mean": 50.07634353637695,
      "margin_dpo/margin_std": 84.07510375976562,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -44.484336853027344,
      "KL/mean": -73.5757064819336,
      "KL/rejected_KL_mean": -102.66706848144531,
      "KL/std": 73.25513458251953,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.00539375888183713,
      "fcm_dpo/delta": 0.005847714375704527,
      "fcm_dpo/margin": 58.18274688720703,
      "fcm_dpo/q_t": 0.42588692903518677,
      "grad_norm": 24.53589630126953,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": 1.1634289026260376,
      "logits/rejected": 1.073542833328247,
      "logps/chosen": -332.76849365234375,
      "logps/ref_chosen": -288.2841796875,
      "logps/ref_rejected": -252.9123992919922,
      "logps/rejected": -355.5794677734375,
      "loss": 4.6021,
      "margin_dpo/margin_mean": 58.1827392578125,
      "margin_dpo/margin_std": 83.94161987304688,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -38.03575134277344,
      "KL/mean": -64.1673355102539,
      "KL/rejected_KL_mean": -90.29890441894531,
      "KL/std": 72.33904266357422,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.005361389834433794,
      "fcm_dpo/delta": -0.014949459582567215,
      "fcm_dpo/margin": 52.26315689086914,
      "fcm_dpo/q_t": 0.4333416819572449,
      "grad_norm": 51.09342575073242,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": 1.3808811902999878,
      "logits/rejected": 1.4800599813461304,
      "logps/chosen": -287.0505676269531,
      "logps/ref_chosen": -249.01483154296875,
      "logps/ref_rejected": -243.81192016601562,
      "logps/rejected": -334.1108093261719,
      "loss": 4.7024,
      "margin_dpo/margin_mean": 52.26315689086914,
      "margin_dpo/margin_std": 81.6162338256836,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -44.308746337890625,
      "KL/mean": -63.30150604248047,
      "KL/rejected_KL_mean": -82.29428100585938,
      "KL/std": 73.72386169433594,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.0053863562643527985,
      "fcm_dpo/delta": 0.04432380944490433,
      "fcm_dpo/margin": 37.98553466796875,
      "fcm_dpo/q_t": 0.4517405033111572,
      "grad_norm": 47.06981658935547,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": 1.3791677951812744,
      "logits/rejected": 1.488995909690857,
      "logps/chosen": -357.23883056640625,
      "logps/ref_chosen": -312.9300537109375,
      "logps/ref_rejected": -296.2721862792969,
      "logps/rejected": -378.56646728515625,
      "loss": 4.9623,
      "margin_dpo/margin_mean": 37.985530853271484,
      "margin_dpo/margin_std": 81.15144348144531,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -35.99077606201172,
      "KL/mean": -67.5530014038086,
      "KL/rejected_KL_mean": -99.11524200439453,
      "KL/std": 75.4515151977539,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.005400956608355045,
      "fcm_dpo/delta": -0.03862059861421585,
      "fcm_dpo/margin": 63.12445831298828,
      "fcm_dpo/q_t": 0.41916579008102417,
      "grad_norm": 19.53777313232422,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": 1.4183087348937988,
      "logits/rejected": 1.4662986993789673,
      "logps/chosen": -304.3046875,
      "logps/ref_chosen": -268.31390380859375,
      "logps/ref_rejected": -268.7603759765625,
      "logps/rejected": -367.8756408691406,
      "loss": 4.5253,
      "margin_dpo/margin_mean": 63.12446212768555,
      "margin_dpo/margin_std": 86.23684692382812,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -53.549034118652344,
      "KL/mean": -68.2750244140625,
      "KL/rejected_KL_mean": -83.00102233886719,
      "KL/std": 71.50314331054688,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.005443968344479799,
      "fcm_dpo/delta": 0.05487688630819321,
      "fcm_dpo/margin": 29.452003479003906,
      "fcm_dpo/q_t": 0.46242785453796387,
      "grad_norm": 86.34410858154297,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": 1.082327127456665,
      "logits/rejected": 1.1423521041870117,
      "logps/chosen": -308.78094482421875,
      "logps/ref_chosen": -255.23190307617188,
      "logps/ref_rejected": -241.1186981201172,
      "logps/rejected": -324.1197204589844,
      "loss": 5.1598,
      "margin_dpo/margin_mean": 29.452001571655273,
      "margin_dpo/margin_std": 87.36746978759766,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -44.49769592285156,
      "KL/mean": -70.22216796875,
      "KL/rejected_KL_mean": -95.9466323852539,
      "KL/std": 69.98675537109375,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.005677058361470699,
      "fcm_dpo/delta": 0.04207335785031319,
      "fcm_dpo/margin": 51.448936462402344,
      "fcm_dpo/q_t": 0.43138042092323303,
      "grad_norm": 42.68515396118164,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 1.0110702514648438,
      "logits/rejected": 1.0219783782958984,
      "logps/chosen": -295.47802734375,
      "logps/ref_chosen": -250.9803466796875,
      "logps/ref_rejected": -239.9314422607422,
      "logps/rejected": -335.8780822753906,
      "loss": 4.6888,
      "margin_dpo/margin_mean": 51.448936462402344,
      "margin_dpo/margin_std": 83.85609436035156,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -52.495601654052734,
      "KL/mean": -72.27164459228516,
      "KL/rejected_KL_mean": -92.04769134521484,
      "KL/std": 70.34510803222656,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.005777380429208279,
      "fcm_dpo/delta": -0.0026240162551403046,
      "fcm_dpo/margin": 39.552093505859375,
      "fcm_dpo/q_t": 0.44624385237693787,
      "grad_norm": 32.94314956665039,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 1.4604151248931885,
      "logits/rejected": 1.4071040153503418,
      "logps/chosen": -353.4983825683594,
      "logps/ref_chosen": -301.0027770996094,
      "logps/ref_rejected": -263.628662109375,
      "logps/rejected": -355.67633056640625,
      "loss": 4.9382,
      "margin_dpo/margin_mean": 39.552093505859375,
      "margin_dpo/margin_std": 82.16785430908203,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -38.32175827026367,
      "KL/mean": -69.79114532470703,
      "KL/rejected_KL_mean": -101.26053619384766,
      "KL/std": 73.03443145751953,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.005676936358213425,
      "fcm_dpo/delta": -0.04290005564689636,
      "fcm_dpo/margin": 62.93877410888672,
      "fcm_dpo/q_t": 0.4153502285480499,
      "grad_norm": 27.678688049316406,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 1.3709224462509155,
      "logits/rejected": 1.4674614667892456,
      "logps/chosen": -339.1705322265625,
      "logps/ref_chosen": -300.8487854003906,
      "logps/ref_rejected": -296.7118835449219,
      "logps/rejected": -397.9724426269531,
      "loss": 4.4494,
      "margin_dpo/margin_mean": 62.93877410888672,
      "margin_dpo/margin_std": 75.01806640625,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -53.2134895324707,
      "KL/mean": -77.04689025878906,
      "KL/rejected_KL_mean": -100.8802719116211,
      "KL/std": 71.0693359375,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.005558415316045284,
      "fcm_dpo/delta": 0.02312423661351204,
      "fcm_dpo/margin": 47.666778564453125,
      "fcm_dpo/q_t": 0.4376974105834961,
      "grad_norm": 32.223663330078125,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": 1.470483422279358,
      "logits/rejected": 1.5340890884399414,
      "logps/chosen": -324.86187744140625,
      "logps/ref_chosen": -271.6484069824219,
      "logps/ref_rejected": -238.21197509765625,
      "logps/rejected": -339.09222412109375,
      "loss": 4.8338,
      "margin_dpo/margin_mean": 47.666778564453125,
      "margin_dpo/margin_std": 94.15717315673828,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -46.016319274902344,
      "KL/mean": -70.57562255859375,
      "KL/rejected_KL_mean": -95.13494110107422,
      "KL/std": 70.61792755126953,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.00566036906093359,
      "fcm_dpo/delta": -0.02906755357980728,
      "fcm_dpo/margin": 49.118621826171875,
      "fcm_dpo/q_t": 0.43402519822120667,
      "grad_norm": 34.34373474121094,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": 1.24135422706604,
      "logits/rejected": 1.327921748161316,
      "logps/chosen": -312.3128662109375,
      "logps/ref_chosen": -266.2965087890625,
      "logps/ref_rejected": -263.7326965332031,
      "logps/rejected": -358.86761474609375,
      "loss": 4.7405,
      "margin_dpo/margin_mean": 49.11861801147461,
      "margin_dpo/margin_std": 79.74325561523438,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -42.03857421875,
      "KL/mean": -68.72586059570312,
      "KL/rejected_KL_mean": -95.41314697265625,
      "KL/std": 74.23870849609375,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.005573991686105728,
      "fcm_dpo/delta": 0.015454444102942944,
      "fcm_dpo/margin": 53.37458038330078,
      "fcm_dpo/q_t": 0.43048393726348877,
      "grad_norm": 24.156660079956055,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": 1.1402620077133179,
      "logits/rejected": 1.1424504518508911,
      "logps/chosen": -334.3993225097656,
      "logps/ref_chosen": -292.36077880859375,
      "logps/ref_rejected": -264.35662841796875,
      "logps/rejected": -359.7697448730469,
      "loss": 4.6858,
      "margin_dpo/margin_mean": 53.37458038330078,
      "margin_dpo/margin_std": 88.7768325805664,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -43.042240142822266,
      "KL/mean": -64.37056732177734,
      "KL/rejected_KL_mean": -85.69889068603516,
      "KL/std": 72.92974853515625,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.005591816268861294,
      "fcm_dpo/delta": -0.012566296383738518,
      "fcm_dpo/margin": 42.656654357910156,
      "fcm_dpo/q_t": 0.4436893165111542,
      "grad_norm": 35.16365051269531,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": 1.1875078678131104,
      "logits/rejected": 1.292189598083496,
      "logps/chosen": -289.76202392578125,
      "logps/ref_chosen": -246.71978759765625,
      "logps/ref_rejected": -261.339599609375,
      "logps/rejected": -347.03851318359375,
      "loss": 4.8845,
      "margin_dpo/margin_mean": 42.65665817260742,
      "margin_dpo/margin_std": 81.83189392089844,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -36.77720260620117,
      "KL/mean": -66.61295318603516,
      "KL/rejected_KL_mean": -96.44872283935547,
      "KL/std": 71.69690704345703,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.0054801590740680695,
      "fcm_dpo/delta": -0.025517648085951805,
      "fcm_dpo/margin": 59.671512603759766,
      "fcm_dpo/q_t": 0.4224410951137543,
      "grad_norm": 29.334272384643555,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": 1.2255319356918335,
      "logits/rejected": 1.2264814376831055,
      "logps/chosen": -335.3359069824219,
      "logps/ref_chosen": -298.5587158203125,
      "logps/ref_rejected": -264.5877685546875,
      "logps/rejected": -361.0364990234375,
      "loss": 4.5684,
      "margin_dpo/margin_mean": 59.671512603759766,
      "margin_dpo/margin_std": 84.24697875976562,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -29.810333251953125,
      "KL/mean": -61.73623275756836,
      "KL/rejected_KL_mean": -93.66212463378906,
      "KL/std": 70.1768798828125,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.005336294881999493,
      "fcm_dpo/delta": -0.018713153898715973,
      "fcm_dpo/margin": 63.85178756713867,
      "fcm_dpo/q_t": 0.41864752769470215,
      "grad_norm": 26.727537155151367,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": 1.0617012977600098,
      "logits/rejected": 1.0448088645935059,
      "logps/chosen": -279.66357421875,
      "logps/ref_chosen": -249.8532257080078,
      "logps/ref_rejected": -233.2117156982422,
      "logps/rejected": -326.87384033203125,
      "loss": 4.4727,
      "margin_dpo/margin_mean": 63.85178756713867,
      "margin_dpo/margin_std": 77.62440490722656,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -38.66363525390625,
      "KL/mean": -66.21481323242188,
      "KL/rejected_KL_mean": -93.76599884033203,
      "KL/std": 72.10069274902344,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.0053711081854999065,
      "fcm_dpo/delta": 0.019835403189063072,
      "fcm_dpo/margin": 55.102359771728516,
      "fcm_dpo/q_t": 0.42987602949142456,
      "grad_norm": 33.1839714050293,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": 1.3277084827423096,
      "logits/rejected": 1.4927070140838623,
      "logps/chosen": -297.4852600097656,
      "logps/ref_chosen": -258.8216247558594,
      "logps/ref_rejected": -260.4953308105469,
      "logps/rejected": -354.26129150390625,
      "loss": 4.6479,
      "margin_dpo/margin_mean": 55.102359771728516,
      "margin_dpo/margin_std": 81.00550842285156,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -35.057559967041016,
      "KL/mean": -66.63055419921875,
      "KL/rejected_KL_mean": -98.20355224609375,
      "KL/std": 70.59123992919922,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.005368894897401333,
      "fcm_dpo/delta": -0.035969942808151245,
      "fcm_dpo/margin": 63.14598846435547,
      "fcm_dpo/q_t": 0.4192720353603363,
      "grad_norm": 27.71535301208496,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": 1.5029133558273315,
      "logits/rejected": 1.4035024642944336,
      "logps/chosen": -359.2740783691406,
      "logps/ref_chosen": -324.2165222167969,
      "logps/ref_rejected": -237.05453491210938,
      "logps/rejected": -335.2580871582031,
      "loss": 4.5057,
      "margin_dpo/margin_mean": 63.145992279052734,
      "margin_dpo/margin_std": 82.18006134033203,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -49.494773864746094,
      "KL/mean": -73.14852905273438,
      "KL/rejected_KL_mean": -96.80229187011719,
      "KL/std": 74.30992126464844,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.005321500822901726,
      "fcm_dpo/delta": 0.03446311503648758,
      "fcm_dpo/margin": 47.307533264160156,
      "fcm_dpo/q_t": 0.44047337770462036,
      "grad_norm": 31.25129508972168,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 1.508404016494751,
      "logits/rejected": 1.5336716175079346,
      "logps/chosen": -327.8960266113281,
      "logps/ref_chosen": -278.4012451171875,
      "logps/ref_rejected": -247.72671508789062,
      "logps/rejected": -344.529052734375,
      "loss": 4.8042,
      "margin_dpo/margin_mean": 47.307533264160156,
      "margin_dpo/margin_std": 83.1152572631836,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -39.886959075927734,
      "KL/mean": -67.20853424072266,
      "KL/rejected_KL_mean": -94.53010559082031,
      "KL/std": 70.16693115234375,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.005344086792320013,
      "fcm_dpo/delta": -0.010779989883303642,
      "fcm_dpo/margin": 54.64316177368164,
      "fcm_dpo/q_t": 0.4307183027267456,
      "grad_norm": 38.96586608886719,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.1587868928909302,
      "logits/rejected": 1.166917324066162,
      "logps/chosen": -331.9815979003906,
      "logps/ref_chosen": -292.0946350097656,
      "logps/ref_rejected": -252.24237060546875,
      "logps/rejected": -346.7724914550781,
      "loss": 4.6721,
      "margin_dpo/margin_mean": 54.643165588378906,
      "margin_dpo/margin_std": 81.2701644897461,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -33.46754455566406,
      "KL/mean": -59.81129837036133,
      "KL/rejected_KL_mean": -86.15505981445312,
      "KL/std": 73.46400451660156,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.005382422357797623,
      "fcm_dpo/delta": 0.01935405656695366,
      "fcm_dpo/margin": 52.68751907348633,
      "fcm_dpo/q_t": 0.4331214427947998,
      "grad_norm": 35.388893127441406,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.2501858472824097,
      "logits/rejected": 1.3093328475952148,
      "logps/chosen": -324.0783386230469,
      "logps/ref_chosen": -290.61077880859375,
      "logps/ref_rejected": -279.8963928222656,
      "logps/rejected": -366.0514831542969,
      "loss": 4.6841,
      "margin_dpo/margin_mean": 52.68751907348633,
      "margin_dpo/margin_std": 79.93425750732422,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -40.87921905517578,
      "KL/mean": -70.88130187988281,
      "KL/rejected_KL_mean": -100.88337707519531,
      "KL/std": 72.89090728759766,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.005489566829055548,
      "fcm_dpo/delta": 0.021257812157273293,
      "fcm_dpo/margin": 60.004146575927734,
      "fcm_dpo/q_t": 0.4219299852848053,
      "grad_norm": 26.304824829101562,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": 1.3672140836715698,
      "logits/rejected": 1.3352155685424805,
      "logps/chosen": -357.1944885253906,
      "logps/ref_chosen": -316.3152770996094,
      "logps/ref_rejected": -300.5086975097656,
      "logps/rejected": -401.39208984375,
      "loss": 4.5305,
      "margin_dpo/margin_mean": 60.004146575927734,
      "margin_dpo/margin_std": 80.86235809326172,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -41.04463195800781,
      "KL/mean": -68.36833190917969,
      "KL/rejected_KL_mean": -95.6920166015625,
      "KL/std": 75.39700317382812,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.005542519502341747,
      "fcm_dpo/delta": 0.0035089505836367607,
      "fcm_dpo/margin": 54.647377014160156,
      "fcm_dpo/q_t": 0.4284469485282898,
      "grad_norm": 26.791460037231445,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": 1.4166622161865234,
      "logits/rejected": 1.5192638635635376,
      "logps/chosen": -331.58990478515625,
      "logps/ref_chosen": -290.5452575683594,
      "logps/ref_rejected": -297.2102355957031,
      "logps/rejected": -392.9022521972656,
      "loss": 4.6359,
      "margin_dpo/margin_mean": 54.647377014160156,
      "margin_dpo/margin_std": 81.0822525024414,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -35.69273376464844,
      "KL/mean": -58.000244140625,
      "KL/rejected_KL_mean": -80.3077392578125,
      "KL/std": 69.43379211425781,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.0056755682453513145,
      "fcm_dpo/delta": 0.03527464345097542,
      "fcm_dpo/margin": 44.61500549316406,
      "fcm_dpo/q_t": 0.4403161108493805,
      "grad_norm": 38.123409271240234,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": 1.1807224750518799,
      "logits/rejected": 1.2819303274154663,
      "logps/chosen": -299.11309814453125,
      "logps/ref_chosen": -263.4203186035156,
      "logps/ref_rejected": -270.30291748046875,
      "logps/rejected": -350.61065673828125,
      "loss": 4.8129,
      "margin_dpo/margin_mean": 44.61500549316406,
      "margin_dpo/margin_std": 81.23895263671875,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -42.461029052734375,
      "KL/mean": -75.73291015625,
      "KL/rejected_KL_mean": -109.00479888916016,
      "KL/std": 77.41284942626953,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.005598099902272224,
      "fcm_dpo/delta": -0.05306318402290344,
      "fcm_dpo/margin": 66.54375457763672,
      "fcm_dpo/q_t": 0.4129056930541992,
      "grad_norm": 21.154287338256836,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": 1.3244861364364624,
      "logits/rejected": 1.5014989376068115,
      "logps/chosen": -354.4983825683594,
      "logps/ref_chosen": -312.0373840332031,
      "logps/ref_rejected": -290.50946044921875,
      "logps/rejected": -399.51422119140625,
      "loss": 4.4607,
      "margin_dpo/margin_mean": 66.54375457763672,
      "margin_dpo/margin_std": 92.53014373779297,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -45.82258224487305,
      "KL/mean": -75.42310333251953,
      "KL/rejected_KL_mean": -105.02359008789062,
      "KL/std": 75.29507446289062,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.005507130175828934,
      "fcm_dpo/delta": 0.002714234869927168,
      "fcm_dpo/margin": 59.201019287109375,
      "fcm_dpo/q_t": 0.4231553375720978,
      "grad_norm": 28.028757095336914,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": 1.1882832050323486,
      "logits/rejected": 1.2208930253982544,
      "logps/chosen": -342.99456787109375,
      "logps/ref_chosen": -297.1719970703125,
      "logps/ref_rejected": -269.15557861328125,
      "logps/rejected": -374.17919921875,
      "loss": 4.5791,
      "margin_dpo/margin_mean": 59.201019287109375,
      "margin_dpo/margin_std": 88.15311431884766,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -40.63059997558594,
      "KL/mean": -66.69808197021484,
      "KL/rejected_KL_mean": -92.76556396484375,
      "KL/std": 76.0389633178711,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.005490375682711601,
      "fcm_dpo/delta": -0.008920337073504925,
      "fcm_dpo/margin": 52.13497543334961,
      "fcm_dpo/q_t": 0.4327000677585602,
      "grad_norm": 25.037092208862305,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": 1.2805054187774658,
      "logits/rejected": 1.2055299282073975,
      "logps/chosen": -327.9954833984375,
      "logps/ref_chosen": -287.3648986816406,
      "logps/ref_rejected": -271.1472473144531,
      "logps/rejected": -363.912841796875,
      "loss": 4.7004,
      "margin_dpo/margin_mean": 52.13497543334961,
      "margin_dpo/margin_std": 82.30056762695312,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -47.38853454589844,
      "KL/mean": -67.71875762939453,
      "KL/rejected_KL_mean": -88.0489730834961,
      "KL/std": 71.74655151367188,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.0055211447179317474,
      "fcm_dpo/delta": 0.015288956463336945,
      "fcm_dpo/margin": 40.66045379638672,
      "fcm_dpo/q_t": 0.44733941555023193,
      "grad_norm": 36.26959991455078,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": 1.241492509841919,
      "logits/rejected": 1.3265279531478882,
      "logps/chosen": -335.99462890625,
      "logps/ref_chosen": -288.6061096191406,
      "logps/ref_rejected": -285.6366271972656,
      "logps/rejected": -373.6855773925781,
      "loss": 4.9406,
      "margin_dpo/margin_mean": 40.66045379638672,
      "margin_dpo/margin_std": 86.61062622070312,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -45.23552322387695,
      "KL/mean": -65.57929992675781,
      "KL/rejected_KL_mean": -85.92306518554688,
      "KL/std": 71.34281158447266,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.0055954414419829845,
      "fcm_dpo/delta": 0.03227192908525467,
      "fcm_dpo/margin": 40.68751907348633,
      "fcm_dpo/q_t": 0.4468037486076355,
      "grad_norm": 55.71131134033203,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 1.0944981575012207,
      "logits/rejected": 1.0186482667922974,
      "logps/chosen": -355.48748779296875,
      "logps/ref_chosen": -310.251953125,
      "logps/ref_rejected": -252.962158203125,
      "logps/rejected": -338.88519287109375,
      "loss": 4.9565,
      "margin_dpo/margin_mean": 40.68751525878906,
      "margin_dpo/margin_std": 91.09400177001953,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -42.476139068603516,
      "KL/mean": -68.77080535888672,
      "KL/rejected_KL_mean": -95.06547546386719,
      "KL/std": 74.36164093017578,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.005685979966074228,
      "fcm_dpo/delta": 0.004405747167766094,
      "fcm_dpo/margin": 52.589324951171875,
      "fcm_dpo/q_t": 0.42967134714126587,
      "grad_norm": 50.89521026611328,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 1.1772315502166748,
      "logits/rejected": 1.3082554340362549,
      "logps/chosen": -279.2565612792969,
      "logps/ref_chosen": -236.7804412841797,
      "logps/ref_rejected": -229.67550659179688,
      "logps/rejected": -324.740966796875,
      "loss": 4.7088,
      "margin_dpo/margin_mean": 52.58932113647461,
      "margin_dpo/margin_std": 91.05215454101562,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -36.047828674316406,
      "KL/mean": -69.71669006347656,
      "KL/rejected_KL_mean": -103.38554382324219,
      "KL/std": 74.15625,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.0055740512907505035,
      "fcm_dpo/delta": -0.0268813818693161,
      "fcm_dpo/margin": 67.33772277832031,
      "fcm_dpo/q_t": 0.4115889370441437,
      "grad_norm": 33.36655807495117,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": 1.2704864740371704,
      "logits/rejected": 1.1673810482025146,
      "logps/chosen": -371.3513488769531,
      "logps/ref_chosen": -335.30352783203125,
      "logps/ref_rejected": -250.6796875,
      "logps/rejected": -354.06524658203125,
      "loss": 4.4436,
      "margin_dpo/margin_mean": 67.33771514892578,
      "margin_dpo/margin_std": 91.66010284423828,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -44.575767517089844,
      "KL/mean": -68.55844116210938,
      "KL/rejected_KL_mean": -92.54109954833984,
      "KL/std": 73.06449890136719,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.00568718696013093,
      "fcm_dpo/delta": 0.051790542900562286,
      "fcm_dpo/margin": 47.9653434753418,
      "fcm_dpo/q_t": 0.4358616769313812,
      "grad_norm": 35.642730712890625,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": 1.4016664028167725,
      "logits/rejected": 1.2425955533981323,
      "logps/chosen": -332.7959899902344,
      "logps/ref_chosen": -288.22021484375,
      "logps/ref_rejected": -227.96673583984375,
      "logps/rejected": -320.5078125,
      "loss": 4.7453,
      "margin_dpo/margin_mean": 47.96533966064453,
      "margin_dpo/margin_std": 81.01168823242188,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -46.872825622558594,
      "KL/mean": -72.06521606445312,
      "KL/rejected_KL_mean": -97.25762939453125,
      "KL/std": 72.35063171386719,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.005833591800183058,
      "fcm_dpo/delta": 0.021631265059113503,
      "fcm_dpo/margin": 50.38481140136719,
      "fcm_dpo/q_t": 0.4304274916648865,
      "grad_norm": 29.464969635009766,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": 1.4088448286056519,
      "logits/rejected": 1.4427754878997803,
      "logps/chosen": -307.20977783203125,
      "logps/ref_chosen": -260.3369140625,
      "logps/ref_rejected": -254.2083740234375,
      "logps/rejected": -351.46600341796875,
      "loss": 4.6996,
      "margin_dpo/margin_mean": 50.38481140136719,
      "margin_dpo/margin_std": 82.05101776123047,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -29.598995208740234,
      "KL/mean": -62.39131546020508,
      "KL/rejected_KL_mean": -95.18364715576172,
      "KL/std": 72.81871795654297,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.005651663988828659,
      "fcm_dpo/delta": -0.07521416246891022,
      "fcm_dpo/margin": 65.58464813232422,
      "fcm_dpo/q_t": 0.412536084651947,
      "grad_norm": 26.716684341430664,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.6335678100585938,
      "logits/rejected": 1.7000172138214111,
      "logps/chosen": -312.6684875488281,
      "logps/ref_chosen": -283.0694885253906,
      "logps/ref_rejected": -289.1370544433594,
      "logps/rejected": -384.32073974609375,
      "loss": 4.4477,
      "margin_dpo/margin_mean": 65.58464813232422,
      "margin_dpo/margin_std": 81.46099090576172,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -30.2250919342041,
      "KL/mean": -65.8253173828125,
      "KL/rejected_KL_mean": -101.42555236816406,
      "KL/std": 73.92179107666016,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.005379385314881802,
      "fcm_dpo/delta": -0.05509934574365616,
      "fcm_dpo/margin": 71.20044708251953,
      "fcm_dpo/q_t": 0.40846970677375793,
      "grad_norm": 26.44307518005371,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": 0.9850060343742371,
      "logits/rejected": 1.0453568696975708,
      "logps/chosen": -287.1101379394531,
      "logps/ref_chosen": -256.8850402832031,
      "logps/ref_rejected": -247.0802459716797,
      "logps/rejected": -348.50579833984375,
      "loss": 4.3422,
      "margin_dpo/margin_mean": 71.20044708251953,
      "margin_dpo/margin_std": 79.24498748779297,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -43.43883514404297,
      "KL/mean": -68.64253997802734,
      "KL/rejected_KL_mean": -93.84625244140625,
      "KL/std": 73.4421615600586,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.0052882349118590355,
      "fcm_dpo/delta": -0.007140140049159527,
      "fcm_dpo/margin": 50.40740203857422,
      "fcm_dpo/q_t": 0.4369214177131653,
      "grad_norm": 32.628700256347656,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": 0.9331328272819519,
      "logits/rejected": 1.084244966506958,
      "logps/chosen": -300.962890625,
      "logps/ref_chosen": -257.5240783691406,
      "logps/ref_rejected": -274.82122802734375,
      "logps/rejected": -368.6674499511719,
      "loss": 4.7726,
      "margin_dpo/margin_mean": 50.40739822387695,
      "margin_dpo/margin_std": 86.01508331298828,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -35.60917663574219,
      "KL/mean": -58.260528564453125,
      "KL/rejected_KL_mean": -80.9118881225586,
      "KL/std": 70.80348205566406,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.00528649752959609,
      "fcm_dpo/delta": -0.016461767256259918,
      "fcm_dpo/margin": 45.302711486816406,
      "fcm_dpo/q_t": 0.44325023889541626,
      "grad_norm": 28.296815872192383,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": 1.5051778554916382,
      "logits/rejected": 1.4057185649871826,
      "logps/chosen": -347.868896484375,
      "logps/ref_chosen": -312.2597351074219,
      "logps/ref_rejected": -259.74383544921875,
      "logps/rejected": -340.65570068359375,
      "loss": 4.8315,
      "margin_dpo/margin_mean": 45.30271530151367,
      "margin_dpo/margin_std": 78.42842864990234,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -30.7714786529541,
      "KL/mean": -60.63641357421875,
      "KL/rejected_KL_mean": -90.50135803222656,
      "KL/std": 75.17670440673828,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.005276237148791552,
      "fcm_dpo/delta": 0.012214533984661102,
      "fcm_dpo/margin": 59.729881286621094,
      "fcm_dpo/q_t": 0.4251258969306946,
      "grad_norm": 24.768232345581055,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": 1.301353931427002,
      "logits/rejected": 1.2282295227050781,
      "logps/chosen": -347.81121826171875,
      "logps/ref_chosen": -317.03973388671875,
      "logps/ref_rejected": -260.12628173828125,
      "logps/rejected": -350.6275939941406,
      "loss": 4.6156,
      "margin_dpo/margin_mean": 59.72987747192383,
      "margin_dpo/margin_std": 89.7888412475586,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -29.997974395751953,
      "KL/mean": -63.17391586303711,
      "KL/rejected_KL_mean": -96.349853515625,
      "KL/std": 74.16685485839844,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.005170515272766352,
      "fcm_dpo/delta": -0.06587792932987213,
      "fcm_dpo/margin": 66.35189056396484,
      "fcm_dpo/q_t": 0.41843393445014954,
      "grad_norm": 25.76171875,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 1.3190009593963623,
      "logits/rejected": 1.261504888534546,
      "logps/chosen": -334.3918762207031,
      "logps/ref_chosen": -304.3939514160156,
      "logps/ref_rejected": -257.6587219238281,
      "logps/rejected": -354.0085754394531,
      "loss": 4.5066,
      "margin_dpo/margin_mean": 66.35189056396484,
      "margin_dpo/margin_std": 85.45376586914062,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -43.13533020019531,
      "KL/mean": -66.53172302246094,
      "KL/rejected_KL_mean": -89.9281234741211,
      "KL/std": 74.23787689208984,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.005101694725453854,
      "fcm_dpo/delta": 0.03779192641377449,
      "fcm_dpo/margin": 46.79279327392578,
      "fcm_dpo/q_t": 0.44339287281036377,
      "grad_norm": 43.6533317565918,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 1.2252767086029053,
      "logits/rejected": 1.3026254177093506,
      "logps/chosen": -328.0791931152344,
      "logps/ref_chosen": -284.94384765625,
      "logps/ref_rejected": -297.4357604980469,
      "logps/rejected": -387.3638916015625,
      "loss": 4.8248,
      "margin_dpo/margin_mean": 46.79280090332031,
      "margin_dpo/margin_std": 81.63873291015625,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -32.658668518066406,
      "KL/mean": -64.18466186523438,
      "KL/rejected_KL_mean": -95.71063995361328,
      "KL/std": 72.1098861694336,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.005169004201889038,
      "fcm_dpo/delta": 0.024842947721481323,
      "fcm_dpo/margin": 63.051971435546875,
      "fcm_dpo/q_t": 0.42333072423934937,
      "grad_norm": 42.63836669921875,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": 1.3769326210021973,
      "logits/rejected": 1.5165824890136719,
      "logps/chosen": -265.012939453125,
      "logps/ref_chosen": -232.35427856445312,
      "logps/ref_rejected": -245.6057891845703,
      "logps/rejected": -341.31640625,
      "loss": 4.5481,
      "margin_dpo/margin_mean": 63.051971435546875,
      "margin_dpo/margin_std": 85.52713012695312,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -38.70287322998047,
      "KL/mean": -70.0283203125,
      "KL/rejected_KL_mean": -101.35376739501953,
      "KL/std": 72.92610931396484,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.0052912598475813866,
      "fcm_dpo/delta": -0.0018884409219026566,
      "fcm_dpo/margin": 62.650909423828125,
      "fcm_dpo/q_t": 0.4214804172515869,
      "grad_norm": 27.894216537475586,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": 1.1967358589172363,
      "logits/rejected": 1.2490659952163696,
      "logps/chosen": -316.709716796875,
      "logps/ref_chosen": -278.0068664550781,
      "logps/ref_rejected": -257.19256591796875,
      "logps/rejected": -358.5463562011719,
      "loss": 4.5381,
      "margin_dpo/margin_mean": 62.650901794433594,
      "margin_dpo/margin_std": 85.49172973632812,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -49.63013458251953,
      "KL/mean": -72.73638153076172,
      "KL/rejected_KL_mean": -95.84263610839844,
      "KL/std": 71.77497100830078,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.005185516085475683,
      "fcm_dpo/delta": -0.018905341625213623,
      "fcm_dpo/margin": 46.21249771118164,
      "fcm_dpo/q_t": 0.44252175092697144,
      "grad_norm": 64.73716735839844,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": 1.2113747596740723,
      "logits/rejected": 1.2698981761932373,
      "logps/chosen": -330.38763427734375,
      "logps/ref_chosen": -280.75750732421875,
      "logps/ref_rejected": -264.34295654296875,
      "logps/rejected": -360.18560791015625,
      "loss": 4.8217,
      "margin_dpo/margin_mean": 46.21249771118164,
      "margin_dpo/margin_std": 76.33674621582031,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -35.97125244140625,
      "KL/mean": -61.21657943725586,
      "KL/rejected_KL_mean": -86.4619140625,
      "KL/std": 69.33377838134766,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.0052066585049033165,
      "fcm_dpo/delta": 0.017166942358016968,
      "fcm_dpo/margin": 50.49065017700195,
      "fcm_dpo/q_t": 0.43751728534698486,
      "grad_norm": 32.864158630371094,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": 1.1829890012741089,
      "logits/rejected": 1.314626932144165,
      "logps/chosen": -277.442626953125,
      "logps/ref_chosen": -241.4713592529297,
      "logps/ref_rejected": -244.0782928466797,
      "logps/rejected": -330.54022216796875,
      "loss": 4.7452,
      "margin_dpo/margin_mean": 50.49064636230469,
      "margin_dpo/margin_std": 80.11993408203125,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -26.50925064086914,
      "KL/mean": -55.45246887207031,
      "KL/rejected_KL_mean": -84.39569854736328,
      "KL/std": 70.28715515136719,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.005218974314630032,
      "fcm_dpo/delta": -0.006187994498759508,
      "fcm_dpo/margin": 57.88644790649414,
      "fcm_dpo/q_t": 0.42822182178497314,
      "grad_norm": 24.919050216674805,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": 1.4165868759155273,
      "logits/rejected": 1.430232286453247,
      "logps/chosen": -345.2835693359375,
      "logps/ref_chosen": -318.7742919921875,
      "logps/ref_rejected": -293.66595458984375,
      "logps/rejected": -378.0616149902344,
      "loss": 4.6283,
      "margin_dpo/margin_mean": 57.88644790649414,
      "margin_dpo/margin_std": 83.11366271972656,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -38.50298309326172,
      "KL/mean": -69.17434692382812,
      "KL/rejected_KL_mean": -99.84568786621094,
      "KL/std": 74.55204772949219,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.005337117705494165,
      "fcm_dpo/delta": 0.02273300662636757,
      "fcm_dpo/margin": 61.342708587646484,
      "fcm_dpo/q_t": 0.4228883385658264,
      "grad_norm": 36.270179748535156,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": 1.2776868343353271,
      "logits/rejected": 1.3203816413879395,
      "logps/chosen": -304.438720703125,
      "logps/ref_chosen": -265.9356994628906,
      "logps/ref_rejected": -243.6986083984375,
      "logps/rejected": -343.5442810058594,
      "loss": 4.5792,
      "margin_dpo/margin_mean": 61.342708587646484,
      "margin_dpo/margin_std": 90.21357727050781,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 4.881076782754382,
      "train_runtime": 7384.286,
      "train_samples_per_second": 8.279,
      "train_steps_per_second": 0.065
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}