{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999244142101285,
  "eval_steps": 200,
  "global_step": 661,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": 0.02867889404296875,
      "KL/mean": 0.029354453086853027,
      "KL/rejected_KL_mean": 0.030029296875,
      "KL/std": 0.2071000635623932,
      "epoch": 0.0015117157974300832,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0013532638549804688,
      "fcm_dpo/q_t": 0.5000336766242981,
      "grad_norm": 28.219078063964844,
      "learning_rate": 0.0,
      "logits/chosen": 0.13337239623069763,
      "logits/rejected": 0.12492949515581131,
      "logps/chosen": -64.5841293334961,
      "logps/ref_chosen": -64.61280822753906,
      "logps/ref_rejected": -64.17195129394531,
      "logps/rejected": -64.14192199707031,
      "loss": 1.3866,
      "margin_dpo/margin_mean": -0.0013527870178222656,
      "margin_dpo/margin_std": 0.2561596930027008,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": -0.00289154052734375,
      "KL/mean": -0.021616414189338684,
      "KL/rejected_KL_mean": -0.04033660888671875,
      "KL/std": 0.19624735414981842,
      "epoch": 0.0030234315948601664,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.037450045347213745,
      "fcm_dpo/q_t": 0.4990639090538025,
      "grad_norm": 27.82866096496582,
      "learning_rate": 7.462686567164179e-09,
      "logits/chosen": 0.09414851665496826,
      "logits/rejected": 0.07363267242908478,
      "logps/chosen": -56.101890563964844,
      "logps/ref_chosen": -56.0989990234375,
      "logps/ref_rejected": -66.59971618652344,
      "logps/rejected": -66.64006042480469,
      "loss": 1.3828,
      "margin_dpo/margin_mean": 0.03744968771934509,
      "margin_dpo/margin_std": 0.27811938524246216,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": 0.02931976318359375,
      "KL/mean": 0.034121572971343994,
      "KL/rejected_KL_mean": 0.038921356201171875,
      "KL/std": 0.24494421482086182,
      "epoch": 0.0045351473922902496,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.009602487087249756,
      "fcm_dpo/q_t": 0.500239908695221,
      "grad_norm": 31.2530460357666,
      "learning_rate": 1.4925373134328357e-08,
      "logits/chosen": 0.09943141043186188,
      "logits/rejected": 0.06147802621126175,
      "logps/chosen": -65.42794036865234,
      "logps/ref_chosen": -65.45726013183594,
      "logps/ref_rejected": -90.82853698730469,
      "logps/rejected": -90.78961181640625,
      "loss": 1.3875,
      "margin_dpo/margin_mean": -0.009602636098861694,
      "margin_dpo/margin_std": 0.31281691789627075,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.007144927978515625,
      "KL/mean": 0.002002894878387451,
      "KL/rejected_KL_mean": -0.003143310546875,
      "KL/std": 0.24436025321483612,
      "epoch": 0.006046863189720333,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.010289788246154785,
      "fcm_dpo/q_t": 0.4997430443763733,
      "grad_norm": 34.29277420043945,
      "learning_rate": 2.2388059701492534e-08,
      "logits/chosen": 0.11271625757217407,
      "logits/rejected": 0.09644747525453568,
      "logps/chosen": -76.85304260253906,
      "logps/ref_chosen": -76.86018371582031,
      "logps/ref_rejected": -79.91523742675781,
      "logps/rejected": -79.91838073730469,
      "loss": 1.3855,
      "margin_dpo/margin_mean": 0.010288774967193604,
      "margin_dpo/margin_std": 0.3010460138320923,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": -0.011060714721679688,
      "KL/mean": -0.0001738518476486206,
      "KL/rejected_KL_mean": 0.010715484619140625,
      "KL/std": 0.2380131483078003,
      "epoch": 0.007558578987150416,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.02177181839942932,
      "fcm_dpo/q_t": 0.5005446076393127,
      "grad_norm": 29.53087615966797,
      "learning_rate": 2.9850746268656714e-08,
      "logits/chosen": 0.08209095895290375,
      "logits/rejected": 0.04320637136697769,
      "logps/chosen": -62.982398986816406,
      "logps/ref_chosen": -62.97134017944336,
      "logps/ref_rejected": -79.9192123413086,
      "logps/rejected": -79.90849304199219,
      "loss": 1.3888,
      "margin_dpo/margin_mean": -0.021771669387817383,
      "margin_dpo/margin_std": 0.3412056565284729,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": -0.06389617919921875,
      "KL/mean": -0.022043123841285706,
      "KL/rejected_KL_mean": 0.019815444946289062,
      "KL/std": 0.22818465530872345,
      "epoch": 0.009070294784580499,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.08370622992515564,
      "fcm_dpo/q_t": 0.5020915269851685,
      "grad_norm": 29.720598220825195,
      "learning_rate": 3.731343283582089e-08,
      "logits/chosen": 0.12795253098011017,
      "logits/rejected": 0.08949151635169983,
      "logps/chosen": -51.37126159667969,
      "logps/ref_chosen": -51.30736541748047,
      "logps/ref_rejected": -82.77239227294922,
      "logps/rejected": -82.75257873535156,
      "loss": 1.3949,
      "margin_dpo/margin_mean": -0.08370649814605713,
      "margin_dpo/margin_std": 0.3061661124229431,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.000690460205078125,
      "KL/mean": 0.004148155450820923,
      "KL/rejected_KL_mean": 0.0076045989990234375,
      "KL/std": 0.2287873923778534,
      "epoch": 0.010582010582010581,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.006914794445037842,
      "fcm_dpo/q_t": 0.5001726150512695,
      "grad_norm": 27.180267333984375,
      "learning_rate": 4.477611940298507e-08,
      "logits/chosen": 0.01639726385474205,
      "logits/rejected": -0.02727954089641571,
      "logps/chosen": -51.45872116088867,
      "logps/ref_chosen": -51.45941162109375,
      "logps/ref_rejected": -66.3828125,
      "logps/rejected": -66.37520599365234,
      "loss": 1.3872,
      "margin_dpo/margin_mean": -0.00691559910774231,
      "margin_dpo/margin_std": 0.29766663908958435,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": 0.012800216674804688,
      "KL/mean": 0.0012196600437164307,
      "KL/rejected_KL_mean": -0.0103607177734375,
      "KL/std": 0.21195456385612488,
      "epoch": 0.012093726379440665,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02315998077392578,
      "fcm_dpo/q_t": 0.4994213283061981,
      "grad_norm": 28.690420150756836,
      "learning_rate": 5.223880597014925e-08,
      "logits/chosen": 0.06877216696739197,
      "logits/rejected": 0.046678461134433746,
      "logps/chosen": -62.18474578857422,
      "logps/ref_chosen": -62.197547912597656,
      "logps/ref_rejected": -74.66180419921875,
      "logps/rejected": -74.67216491699219,
      "loss": 1.3842,
      "margin_dpo/margin_mean": 0.023159563541412354,
      "margin_dpo/margin_std": 0.29333364963531494,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.0251922607421875,
      "KL/mean": -0.03252735733985901,
      "KL/rejected_KL_mean": -0.03986358642578125,
      "KL/std": 0.22694696485996246,
      "epoch": 0.013605442176870748,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.014674007892608643,
      "fcm_dpo/q_t": 0.49963313341140747,
      "grad_norm": 31.518144607543945,
      "learning_rate": 5.970149253731343e-08,
      "logits/chosen": 0.177996426820755,
      "logits/rejected": 0.1178831085562706,
      "logps/chosen": -55.65491485595703,
      "logps/ref_chosen": -55.629722595214844,
      "logps/ref_rejected": -86.21221923828125,
      "logps/rejected": -86.25209045410156,
      "loss": 1.3851,
      "margin_dpo/margin_mean": 0.014674067497253418,
      "margin_dpo/margin_std": 0.3014518618583679,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": 0.011089324951171875,
      "KL/mean": 0.04612685739994049,
      "KL/rejected_KL_mean": 0.08116531372070312,
      "KL/std": 0.2460353970527649,
      "epoch": 0.015117157974300832,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07007792592048645,
      "fcm_dpo/q_t": 0.5017515420913696,
      "grad_norm": 29.845827102661133,
      "learning_rate": 6.71641791044776e-08,
      "logits/chosen": 0.13188131153583527,
      "logits/rejected": 0.10091814398765564,
      "logps/chosen": -62.67951202392578,
      "logps/ref_chosen": -62.69060134887695,
      "logps/ref_rejected": -90.610107421875,
      "logps/rejected": -90.52894592285156,
      "loss": 1.3937,
      "margin_dpo/margin_mean": -0.0700773298740387,
      "margin_dpo/margin_std": 0.3673255443572998,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": -0.030055999755859375,
      "KL/mean": -0.00986570119857788,
      "KL/rejected_KL_mean": 0.01031494140625,
      "KL/std": 0.18542543053627014,
      "epoch": 0.016628873771730914,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.040368854999542236,
      "fcm_dpo/q_t": 0.5010088086128235,
      "grad_norm": 29.479524612426758,
      "learning_rate": 7.462686567164178e-08,
      "logits/chosen": 0.09895411133766174,
      "logits/rejected": 0.09225950390100479,
      "logps/chosen": -65.79717254638672,
      "logps/ref_chosen": -65.76712036132812,
      "logps/ref_rejected": -72.4764633178711,
      "logps/rejected": -72.46614837646484,
      "loss": 1.3905,
      "margin_dpo/margin_mean": -0.04036879539489746,
      "margin_dpo/margin_std": 0.2673962116241455,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": -0.02222442626953125,
      "KL/mean": -0.01459282636642456,
      "KL/rejected_KL_mean": -0.0069599151611328125,
      "KL/std": 0.2098838984966278,
      "epoch": 0.018140589569160998,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.015265405178070068,
      "fcm_dpo/q_t": 0.5003814697265625,
      "grad_norm": 28.26340103149414,
      "learning_rate": 8.208955223880596e-08,
      "logits/chosen": 0.022189727053046227,
      "logits/rejected": 0.00604275893419981,
      "logps/chosen": -60.727115631103516,
      "logps/ref_chosen": -60.704891204833984,
      "logps/ref_rejected": -69.41564178466797,
      "logps/rejected": -69.42259979248047,
      "loss": 1.388,
      "margin_dpo/margin_mean": -0.01526564359664917,
      "margin_dpo/margin_std": 0.2689279317855835,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": 0.0106048583984375,
      "KL/mean": 0.01754908263683319,
      "KL/rejected_KL_mean": 0.02449798583984375,
      "KL/std": 0.22062669694423676,
      "epoch": 0.019652305366591082,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.013890773057937622,
      "fcm_dpo/q_t": 0.5003470778465271,
      "grad_norm": 29.100948333740234,
      "learning_rate": 8.955223880597014e-08,
      "logits/chosen": 0.11829005181789398,
      "logits/rejected": 0.05488961189985275,
      "logps/chosen": -49.898651123046875,
      "logps/ref_chosen": -49.90925598144531,
      "logps/ref_rejected": -92.37818145751953,
      "logps/rejected": -92.35368347167969,
      "loss": 1.3879,
      "margin_dpo/margin_mean": -0.013890862464904785,
      "margin_dpo/margin_std": 0.30010896921157837,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.018400192260742188,
      "KL/mean": 0.0002371072769165039,
      "KL/rejected_KL_mean": -0.01792144775390625,
      "KL/std": 0.2105257362127304,
      "epoch": 0.021164021164021163,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03632456064224243,
      "fcm_dpo/q_t": 0.49909305572509766,
      "grad_norm": 29.442873001098633,
      "learning_rate": 9.701492537313432e-08,
      "logits/chosen": 0.10424400866031647,
      "logits/rejected": 0.08598647266626358,
      "logps/chosen": -60.60039520263672,
      "logps/ref_chosen": -60.61879348754883,
      "logps/ref_rejected": -71.79306030273438,
      "logps/rejected": -71.81098175048828,
      "loss": 1.3829,
      "margin_dpo/margin_mean": 0.03632441163063049,
      "margin_dpo/margin_std": 0.3239067792892456,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.05114936828613281,
      "KL/mean": -0.06186847388744354,
      "KL/rejected_KL_mean": -0.07258224487304688,
      "KL/std": 0.2505655288696289,
      "epoch": 0.022675736961451247,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.021430760622024536,
      "fcm_dpo/q_t": 0.4994640648365021,
      "grad_norm": 33.124202728271484,
      "learning_rate": 1.044776119402985e-07,
      "logits/chosen": 0.09332643449306488,
      "logits/rejected": 0.0486547127366066,
      "logps/chosen": -63.520687103271484,
      "logps/ref_chosen": -63.46953582763672,
      "logps/ref_rejected": -88.88951110839844,
      "logps/rejected": -88.96209716796875,
      "loss": 1.3844,
      "margin_dpo/margin_mean": 0.021430641412734985,
      "margin_dpo/margin_std": 0.33308592438697815,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": -0.020135879516601562,
      "KL/mean": -0.004521623253822327,
      "KL/rejected_KL_mean": 0.011089324951171875,
      "KL/std": 0.18780693411827087,
      "epoch": 0.02418745275888133,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.031230241060256958,
      "fcm_dpo/q_t": 0.5007807016372681,
      "grad_norm": 26.916568756103516,
      "learning_rate": 1.1194029850746268e-07,
      "logits/chosen": 0.12158643454313278,
      "logits/rejected": 0.08412902057170868,
      "logps/chosen": -46.55243682861328,
      "logps/ref_chosen": -46.53229904174805,
      "logps/ref_rejected": -74.27533721923828,
      "logps/rejected": -74.26425170898438,
      "loss": 1.3896,
      "margin_dpo/margin_mean": -0.03123021125793457,
      "margin_dpo/margin_std": 0.24277648329734802,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": -0.0008640289306640625,
      "KL/mean": -0.015623345971107483,
      "KL/rejected_KL_mean": -0.030376434326171875,
      "KL/std": 0.20342105627059937,
      "epoch": 0.025699168556311415,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.029514700174331665,
      "fcm_dpo/q_t": 0.4992625117301941,
      "grad_norm": 32.90407180786133,
      "learning_rate": 1.1940298507462686e-07,
      "logits/chosen": 0.0347953625023365,
      "logits/rejected": 0.016835104674100876,
      "logps/chosen": -64.07869720458984,
      "logps/ref_chosen": -64.07783508300781,
      "logps/ref_rejected": -86.40876770019531,
      "logps/rejected": -86.43914794921875,
      "loss": 1.3836,
      "margin_dpo/margin_mean": 0.02951410412788391,
      "margin_dpo/margin_std": 0.29045775532722473,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": 0.0038318634033203125,
      "KL/mean": -0.004959017038345337,
      "KL/rejected_KL_mean": -0.013753890991210938,
      "KL/std": 0.20491787791252136,
      "epoch": 0.027210884353741496,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0175778865814209,
      "fcm_dpo/q_t": 0.49956053495407104,
      "grad_norm": 27.838245391845703,
      "learning_rate": 1.2686567164179106e-07,
      "logits/chosen": 0.10396843403577805,
      "logits/rejected": 0.05753494054079056,
      "logps/chosen": -44.870506286621094,
      "logps/ref_chosen": -44.87433624267578,
      "logps/ref_rejected": -70.97604370117188,
      "logps/rejected": -70.98979949951172,
      "loss": 1.3847,
      "margin_dpo/margin_mean": 0.017578035593032837,
      "margin_dpo/margin_std": 0.2776282727718353,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": 0.013826370239257812,
      "KL/mean": -0.0132150799036026,
      "KL/rejected_KL_mean": -0.040256500244140625,
      "KL/std": 0.23131096363067627,
      "epoch": 0.02872260015117158,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.054083555936813354,
      "fcm_dpo/q_t": 0.49864810705184937,
      "grad_norm": 30.866737365722656,
      "learning_rate": 1.343283582089552e-07,
      "logits/chosen": 0.06870782375335693,
      "logits/rejected": 0.05521427094936371,
      "logps/chosen": -68.14598083496094,
      "logps/ref_chosen": -68.1598129272461,
      "logps/ref_rejected": -81.17138671875,
      "logps/rejected": -81.2116470336914,
      "loss": 1.3811,
      "margin_dpo/margin_mean": 0.054083943367004395,
      "margin_dpo/margin_std": 0.29337140917778015,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.0003376007080078125,
      "KL/mean": -0.0011960119009017944,
      "KL/rejected_KL_mean": -0.002727508544921875,
      "KL/std": 0.2168964147567749,
      "epoch": 0.030234315948601664,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.003067702054977417,
      "fcm_dpo/q_t": 0.49992379546165466,
      "grad_norm": 29.303083419799805,
      "learning_rate": 1.4179104477611938e-07,
      "logits/chosen": 0.13337117433547974,
      "logits/rejected": 0.11041676253080368,
      "logps/chosen": -53.67822265625,
      "logps/ref_chosen": -53.67856216430664,
      "logps/ref_rejected": -74.16911315917969,
      "logps/rejected": -74.17184448242188,
      "loss": 1.3862,
      "margin_dpo/margin_mean": 0.0030680596828460693,
      "margin_dpo/margin_std": 0.31527629494667053,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.017999649047851562,
      "KL/mean": 0.021966353058815002,
      "KL/rejected_KL_mean": 0.02593231201171875,
      "KL/std": 0.21482843160629272,
      "epoch": 0.031746031746031744,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.007932394742965698,
      "fcm_dpo/q_t": 0.5001983642578125,
      "grad_norm": 29.316267013549805,
      "learning_rate": 1.4925373134328355e-07,
      "logits/chosen": 0.11854798346757889,
      "logits/rejected": 0.09246804565191269,
      "logps/chosen": -64.68355560302734,
      "logps/ref_chosen": -64.70155334472656,
      "logps/ref_rejected": -81.02095031738281,
      "logps/rejected": -80.99501037597656,
      "loss": 1.3873,
      "margin_dpo/margin_mean": -0.007933080196380615,
      "margin_dpo/margin_std": 0.27441009879112244,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.013017654418945312,
      "KL/mean": -0.01622338593006134,
      "KL/rejected_KL_mean": -0.045467376708984375,
      "KL/std": 0.2124525010585785,
      "epoch": 0.03325774754346183,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05848535895347595,
      "fcm_dpo/q_t": 0.4985382556915283,
      "grad_norm": 28.89164161682129,
      "learning_rate": 1.5671641791044775e-07,
      "logits/chosen": 0.029597830027341843,
      "logits/rejected": 0.007340491749346256,
      "logps/chosen": -58.022972106933594,
      "logps/ref_chosen": -58.03599166870117,
      "logps/ref_rejected": -80.72721862792969,
      "logps/rejected": -80.77268981933594,
      "loss": 1.3806,
      "margin_dpo/margin_mean": 0.05848526954650879,
      "margin_dpo/margin_std": 0.2741105556488037,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.018955230712890625,
      "KL/mean": 0.005454123020172119,
      "KL/rejected_KL_mean": -0.00804901123046875,
      "KL/std": 0.2213050127029419,
      "epoch": 0.03476946334089191,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02700936794281006,
      "fcm_dpo/q_t": 0.4993247985839844,
      "grad_norm": 32.89314270019531,
      "learning_rate": 1.6417910447761193e-07,
      "logits/chosen": 0.14521625638008118,
      "logits/rejected": 0.11956644058227539,
      "logps/chosen": -66.33712768554688,
      "logps/ref_chosen": -66.35608673095703,
      "logps/ref_rejected": -93.02769470214844,
      "logps/rejected": -93.03573608398438,
      "loss": 1.3839,
      "margin_dpo/margin_mean": 0.02700933814048767,
      "margin_dpo/margin_std": 0.33931535482406616,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": -0.038440704345703125,
      "KL/mean": -0.027752220630645752,
      "KL/rejected_KL_mean": -0.017061233520507812,
      "KL/std": 0.20728710293769836,
      "epoch": 0.036281179138321996,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.021373331546783447,
      "fcm_dpo/q_t": 0.500534176826477,
      "grad_norm": 26.315832138061523,
      "learning_rate": 1.716417910447761e-07,
      "logits/chosen": 0.11648119986057281,
      "logits/rejected": 0.08412452787160873,
      "logps/chosen": -54.49967956542969,
      "logps/ref_chosen": -54.461238861083984,
      "logps/ref_rejected": -68.33817291259766,
      "logps/rejected": -68.35523986816406,
      "loss": 1.3886,
      "margin_dpo/margin_mean": -0.021373480558395386,
      "margin_dpo/margin_std": 0.26126983761787415,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": -0.0121917724609375,
      "KL/mean": -0.02050521969795227,
      "KL/rejected_KL_mean": -0.02881622314453125,
      "KL/std": 0.21590664982795715,
      "epoch": 0.03779289493575208,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.016626060009002686,
      "fcm_dpo/q_t": 0.4995846748352051,
      "grad_norm": 29.75655746459961,
      "learning_rate": 1.7910447761194027e-07,
      "logits/chosen": 0.11330267786979675,
      "logits/rejected": 0.061182692646980286,
      "logps/chosen": -60.016395568847656,
      "logps/ref_chosen": -60.00420379638672,
      "logps/ref_rejected": -90.47376251220703,
      "logps/rejected": -90.50257873535156,
      "loss": 1.385,
      "margin_dpo/margin_mean": 0.016626060009002686,
      "margin_dpo/margin_std": 0.3512018322944641,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": -0.010725021362304688,
      "KL/mean": -0.02841213345527649,
      "KL/rejected_KL_mean": -0.046092987060546875,
      "KL/std": 0.19508026540279388,
      "epoch": 0.039304610733182165,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03537088632583618,
      "fcm_dpo/q_t": 0.4991161823272705,
      "grad_norm": 29.64750099182129,
      "learning_rate": 1.8656716417910447e-07,
      "logits/chosen": 0.12426209449768066,
      "logits/rejected": 0.10544230788946152,
      "logps/chosen": -56.82987976074219,
      "logps/ref_chosen": -56.81915283203125,
      "logps/ref_rejected": -77.84333038330078,
      "logps/rejected": -77.88941955566406,
      "loss": 1.383,
      "margin_dpo/margin_mean": 0.03537055850028992,
      "margin_dpo/margin_std": 0.29276707768440247,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": -0.0206756591796875,
      "KL/mean": -0.012468293309211731,
      "KL/rejected_KL_mean": -0.0042629241943359375,
      "KL/std": 0.2252947986125946,
      "epoch": 0.04081632653061224,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.016415029764175415,
      "fcm_dpo/q_t": 0.5004103183746338,
      "grad_norm": 29.00526237487793,
      "learning_rate": 1.9402985074626865e-07,
      "logits/chosen": 0.09494505822658539,
      "logits/rejected": 0.0703374445438385,
      "logps/chosen": -62.897701263427734,
      "logps/ref_chosen": -62.87702560424805,
      "logps/ref_rejected": -71.34437561035156,
      "logps/rejected": -71.34864044189453,
      "loss": 1.3881,
      "margin_dpo/margin_mean": -0.016414940357208252,
      "margin_dpo/margin_std": 0.2852821946144104,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": -0.023799896240234375,
      "KL/mean": -0.015411525964736938,
      "KL/rejected_KL_mean": -0.007022857666015625,
      "KL/std": 0.2348794937133789,
      "epoch": 0.042328042328042326,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.016774117946624756,
      "fcm_dpo/q_t": 0.5004186034202576,
      "grad_norm": 27.660003662109375,
      "learning_rate": 2.0149253731343282e-07,
      "logits/chosen": 0.047501303255558014,
      "logits/rejected": 0.03891991078853607,
      "logps/chosen": -59.857177734375,
      "logps/ref_chosen": -59.8333740234375,
      "logps/ref_rejected": -70.39804077148438,
      "logps/rejected": -70.40505981445312,
      "loss": 1.3882,
      "margin_dpo/margin_mean": -0.01677393913269043,
      "margin_dpo/margin_std": 0.30225175619125366,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": -0.011280059814453125,
      "KL/mean": -0.03545750677585602,
      "KL/rejected_KL_mean": -0.059627532958984375,
      "KL/std": 0.2457764446735382,
      "epoch": 0.04383975812547241,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04835239052772522,
      "fcm_dpo/q_t": 0.4987908601760864,
      "grad_norm": 32.58844757080078,
      "learning_rate": 2.08955223880597e-07,
      "logits/chosen": 0.15888172388076782,
      "logits/rejected": 0.14050382375717163,
      "logps/chosen": -74.13148498535156,
      "logps/ref_chosen": -74.12020111083984,
      "logps/ref_rejected": -83.33099365234375,
      "logps/rejected": -83.39061737060547,
      "loss": 1.3818,
      "margin_dpo/margin_mean": 0.04835203289985657,
      "margin_dpo/margin_std": 0.355918824672699,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": -0.04425048828125,
      "KL/mean": -0.014838695526123047,
      "KL/rejected_KL_mean": 0.0145721435546875,
      "KL/std": 0.23300054669380188,
      "epoch": 0.045351473922902494,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.05882430076599121,
      "fcm_dpo/q_t": 0.5014699697494507,
      "grad_norm": 30.512819290161133,
      "learning_rate": 2.1641791044776117e-07,
      "logits/chosen": 0.13098673522472382,
      "logits/rejected": 0.07602270692586899,
      "logps/chosen": -50.79553985595703,
      "logps/ref_chosen": -50.75128936767578,
      "logps/ref_rejected": -89.29063415527344,
      "logps/rejected": -89.27606201171875,
      "loss": 1.3924,
      "margin_dpo/margin_mean": -0.058824241161346436,
      "margin_dpo/margin_std": 0.31058943271636963,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": -0.023962020874023438,
      "KL/mean": -0.048681557178497314,
      "KL/rejected_KL_mean": -0.07340240478515625,
      "KL/std": 0.23842594027519226,
      "epoch": 0.04686318972033258,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.049435973167419434,
      "fcm_dpo/q_t": 0.498765230178833,
      "grad_norm": 34.26254653930664,
      "learning_rate": 2.2388059701492537e-07,
      "logits/chosen": 0.124081090092659,
      "logits/rejected": 0.07678476721048355,
      "logps/chosen": -65.3607177734375,
      "logps/ref_chosen": -65.33675384521484,
      "logps/ref_rejected": -100.76666259765625,
      "logps/rejected": -100.8400650024414,
      "loss": 1.3817,
      "margin_dpo/margin_mean": 0.049436479806900024,
      "margin_dpo/margin_std": 0.35601481795310974,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": -0.029903411865234375,
      "KL/mean": -0.018042847514152527,
      "KL/rejected_KL_mean": -0.006183624267578125,
      "KL/std": 0.268637478351593,
      "epoch": 0.04837490551776266,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.023714810609817505,
      "fcm_dpo/q_t": 0.5005922913551331,
      "grad_norm": 30.1397762298584,
      "learning_rate": 2.3134328358208954e-07,
      "logits/chosen": 0.0673169419169426,
      "logits/rejected": 0.05956514924764633,
      "logps/chosen": -67.2132339477539,
      "logps/ref_chosen": -67.18333435058594,
      "logps/ref_rejected": -82.80763244628906,
      "logps/rejected": -82.8138198852539,
      "loss": 1.389,
      "margin_dpo/margin_mean": -0.023714929819107056,
      "margin_dpo/margin_std": 0.356290727853775,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": -0.008056640625,
      "KL/mean": -0.037323206663131714,
      "KL/rejected_KL_mean": -0.06658554077148438,
      "KL/std": 0.24775487184524536,
      "epoch": 0.049886621315192746,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0585293173789978,
      "fcm_dpo/q_t": 0.4985373020172119,
      "grad_norm": 30.928356170654297,
      "learning_rate": 2.388059701492537e-07,
      "logits/chosen": 0.020615022629499435,
      "logits/rejected": -0.004851259291172028,
      "logps/chosen": -64.04753875732422,
      "logps/ref_chosen": -64.03948211669922,
      "logps/ref_rejected": -75.68357849121094,
      "logps/rejected": -75.75016784667969,
      "loss": 1.3807,
      "margin_dpo/margin_mean": 0.058529555797576904,
      "margin_dpo/margin_std": 0.32526156306266785,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": -0.04400825500488281,
      "KL/mean": -0.07188649475574493,
      "KL/rejected_KL_mean": -0.09976577758789062,
      "KL/std": 0.22814133763313293,
      "epoch": 0.05139833711262283,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.055761903524398804,
      "fcm_dpo/q_t": 0.49860724806785583,
      "grad_norm": 28.403575897216797,
      "learning_rate": 2.4626865671641786e-07,
      "logits/chosen": 0.09220936894416809,
      "logits/rejected": 0.06258954852819443,
      "logps/chosen": -53.70830535888672,
      "logps/ref_chosen": -53.6642951965332,
      "logps/ref_rejected": -65.77989959716797,
      "logps/rejected": -65.87966918945312,
      "loss": 1.381,
      "margin_dpo/margin_mean": 0.0557616651058197,
      "margin_dpo/margin_std": 0.34898078441619873,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": -0.08493995666503906,
      "KL/mean": -0.09149569272994995,
      "KL/rejected_KL_mean": -0.098052978515625,
      "KL/std": 0.239829421043396,
      "epoch": 0.05291005291005291,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.013110756874084473,
      "fcm_dpo/q_t": 0.49967312812805176,
      "grad_norm": 27.82366180419922,
      "learning_rate": 2.537313432835821e-07,
      "logits/chosen": 0.07824540138244629,
      "logits/rejected": 0.055066656321287155,
      "logps/chosen": -61.101802825927734,
      "logps/ref_chosen": -61.01686096191406,
      "logps/ref_rejected": -72.78598022460938,
      "logps/rejected": -72.884033203125,
      "loss": 1.3853,
      "margin_dpo/margin_mean": 0.013110697269439697,
      "margin_dpo/margin_std": 0.33318257331848145,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": -0.10409927368164062,
      "KL/mean": -0.07765229046344757,
      "KL/rejected_KL_mean": -0.051204681396484375,
      "KL/std": 0.22821326553821564,
      "epoch": 0.05442176870748299,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.05289342999458313,
      "fcm_dpo/q_t": 0.5013221502304077,
      "grad_norm": 28.933420181274414,
      "learning_rate": 2.611940298507462e-07,
      "logits/chosen": 0.09718886762857437,
      "logits/rejected": 0.04460256174206734,
      "logps/chosen": -50.64146041870117,
      "logps/ref_chosen": -50.53736114501953,
      "logps/ref_rejected": -78.11678314208984,
      "logps/rejected": -78.16798400878906,
      "loss": 1.3918,
      "margin_dpo/margin_mean": -0.052893489599227905,
      "margin_dpo/margin_std": 0.3156048059463501,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": -0.05438232421875,
      "KL/mean": -0.10749967396259308,
      "KL/rejected_KL_mean": -0.16062164306640625,
      "KL/std": 0.2757084369659424,
      "epoch": 0.055933484504913075,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10623124241828918,
      "fcm_dpo/q_t": 0.4973453879356384,
      "grad_norm": 37.78874969482422,
      "learning_rate": 2.686567164179104e-07,
      "logits/chosen": 0.07935284823179245,
      "logits/rejected": 0.0024427399039268494,
      "logps/chosen": -59.60832977294922,
      "logps/ref_chosen": -59.55394744873047,
      "logps/ref_rejected": -108.27702331542969,
      "logps/rejected": -108.43765258789062,
      "loss": 1.376,
      "margin_dpo/margin_mean": 0.10623079538345337,
      "margin_dpo/margin_std": 0.36104801297187805,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": -0.11186981201171875,
      "KL/mean": -0.10196225345134735,
      "KL/rejected_KL_mean": -0.0920562744140625,
      "KL/std": 0.2679488956928253,
      "epoch": 0.05744520030234316,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.019819647073745728,
      "fcm_dpo/q_t": 0.5004957318305969,
      "grad_norm": 29.816259384155273,
      "learning_rate": 2.761194029850746e-07,
      "logits/chosen": 0.051365844905376434,
      "logits/rejected": 0.03763524442911148,
      "logps/chosen": -65.90022277832031,
      "logps/ref_chosen": -65.78836059570312,
      "logps/ref_rejected": -76.1619873046875,
      "logps/rejected": -76.25404357910156,
      "loss": 1.3886,
      "margin_dpo/margin_mean": -0.019819557666778564,
      "margin_dpo/margin_std": 0.36583802103996277,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": -0.03894615173339844,
      "KL/mean": -0.09722255170345306,
      "KL/rejected_KL_mean": -0.15549468994140625,
      "KL/std": 0.2549492120742798,
      "epoch": 0.05895691609977324,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11654523015022278,
      "fcm_dpo/q_t": 0.49708667397499084,
      "grad_norm": 29.102598190307617,
      "learning_rate": 2.8358208955223876e-07,
      "logits/chosen": 0.1645927131175995,
      "logits/rejected": 0.13760069012641907,
      "logps/chosen": -57.215755462646484,
      "logps/ref_chosen": -57.17681121826172,
      "logps/ref_rejected": -79.486328125,
      "logps/rejected": -79.64183044433594,
      "loss": 1.375,
      "margin_dpo/margin_mean": 0.11654546856880188,
      "margin_dpo/margin_std": 0.3799525499343872,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": -0.10531044006347656,
      "KL/mean": -0.11316066980361938,
      "KL/rejected_KL_mean": -0.1210174560546875,
      "KL/std": 0.266708105802536,
      "epoch": 0.06046863189720333,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0157049298286438,
      "fcm_dpo/q_t": 0.4996066093444824,
      "grad_norm": 31.49516487121582,
      "learning_rate": 2.9104477611940296e-07,
      "logits/chosen": 0.09835559129714966,
      "logits/rejected": 0.04947054386138916,
      "logps/chosen": -61.439476013183594,
      "logps/ref_chosen": -61.33416748046875,
      "logps/ref_rejected": -79.10697174072266,
      "logps/rejected": -79.22798919677734,
      "loss": 1.3851,
      "margin_dpo/margin_mean": 0.01570478081703186,
      "margin_dpo/margin_std": 0.36846187710762024,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": -0.09158706665039062,
      "KL/mean": -0.13693425059318542,
      "KL/rejected_KL_mean": -0.18227386474609375,
      "KL/std": 0.2849547564983368,
      "epoch": 0.06198034769463341,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09068530797958374,
      "fcm_dpo/q_t": 0.49773430824279785,
      "grad_norm": 30.41977310180664,
      "learning_rate": 2.985074626865671e-07,
      "logits/chosen": 0.04748855158686638,
      "logits/rejected": 0.02758950926363468,
      "logps/chosen": -67.63832092285156,
      "logps/ref_chosen": -67.5467300415039,
      "logps/ref_rejected": -83.87788391113281,
      "logps/rejected": -84.06016540527344,
      "loss": 1.3777,
      "margin_dpo/margin_mean": 0.09068578481674194,
      "margin_dpo/margin_std": 0.40315473079681396,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": -0.1395435333251953,
      "KL/mean": -0.13160492479801178,
      "KL/rejected_KL_mean": -0.12366485595703125,
      "KL/std": 0.3056698143482208,
      "epoch": 0.06349206349206349,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.01587921380996704,
      "fcm_dpo/q_t": 0.5003963112831116,
      "grad_norm": 29.38926124572754,
      "learning_rate": 3.059701492537313e-07,
      "logits/chosen": 0.031875789165496826,
      "logits/rejected": 0.010956110432744026,
      "logps/chosen": -61.40440368652344,
      "logps/ref_chosen": -61.26485824584961,
      "logps/ref_rejected": -76.3629150390625,
      "logps/rejected": -76.48657989501953,
      "loss": 1.3882,
      "margin_dpo/margin_mean": -0.01587921380996704,
      "margin_dpo/margin_std": 0.37683185935020447,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": -0.14949798583984375,
      "KL/mean": -0.16279058158397675,
      "KL/rejected_KL_mean": -0.17608070373535156,
      "KL/std": 0.32842856645584106,
      "epoch": 0.06500377928949358,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.026586264371871948,
      "fcm_dpo/q_t": 0.4993341565132141,
      "grad_norm": 34.7089729309082,
      "learning_rate": 3.134328358208955e-07,
      "logits/chosen": 0.09826282411813736,
      "logits/rejected": 0.08700218051671982,
      "logps/chosen": -71.95852661132812,
      "logps/ref_chosen": -71.80902862548828,
      "logps/ref_rejected": -81.12464141845703,
      "logps/rejected": -81.30072021484375,
      "loss": 1.3841,
      "margin_dpo/margin_mean": 0.026585698127746582,
      "margin_dpo/margin_std": 0.4369037449359894,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": -0.1836719512939453,
      "KL/mean": -0.18749667704105377,
      "KL/rejected_KL_mean": -0.19132232666015625,
      "KL/std": 0.3246310353279114,
      "epoch": 0.06651549508692366,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0076510608196258545,
      "fcm_dpo/q_t": 0.49981001019477844,
      "grad_norm": 32.50434875488281,
      "learning_rate": 3.2089552238805965e-07,
      "logits/chosen": 0.04866597801446915,
      "logits/rejected": 0.018206467851996422,
      "logps/chosen": -66.73410034179688,
      "logps/ref_chosen": -66.55043029785156,
      "logps/ref_rejected": -85.06198120117188,
      "logps/rejected": -85.25330352783203,
      "loss": 1.3861,
      "margin_dpo/margin_mean": 0.007650971412658691,
      "margin_dpo/margin_std": 0.4642822742462158,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": -0.14289283752441406,
      "KL/mean": -0.18554267287254333,
      "KL/rejected_KL_mean": -0.22819900512695312,
      "KL/std": 0.2894941568374634,
      "epoch": 0.06802721088435375,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08530868589878082,
      "fcm_dpo/q_t": 0.49786853790283203,
      "grad_norm": 31.843597412109375,
      "learning_rate": 3.2835820895522385e-07,
      "logits/chosen": 0.12124882638454437,
      "logits/rejected": 0.06842705607414246,
      "logps/chosen": -62.386749267578125,
      "logps/ref_chosen": -62.24385452270508,
      "logps/ref_rejected": -92.96665954589844,
      "logps/rejected": -93.19486236572266,
      "loss": 1.3782,
      "margin_dpo/margin_mean": 0.0853097140789032,
      "margin_dpo/margin_std": 0.4236711859703064,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": -0.11667251586914062,
      "KL/mean": -0.1959172636270523,
      "KL/rejected_KL_mean": -0.2751655578613281,
      "KL/std": 0.3037590980529785,
      "epoch": 0.06953892668178382,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15848949551582336,
      "fcm_dpo/q_t": 0.49603962898254395,
      "grad_norm": 31.1035099029541,
      "learning_rate": 3.3582089552238805e-07,
      "logits/chosen": 0.12088718265295029,
      "logits/rejected": 0.07512138783931732,
      "logps/chosen": -61.615577697753906,
      "logps/ref_chosen": -61.498905181884766,
      "logps/ref_rejected": -78.91172790527344,
      "logps/rejected": -79.1868896484375,
      "loss": 1.3709,
      "margin_dpo/margin_mean": 0.15848910808563232,
      "margin_dpo/margin_std": 0.39531779289245605,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": -0.12522506713867188,
      "KL/mean": -0.21782416105270386,
      "KL/rejected_KL_mean": -0.3104248046875,
      "KL/std": 0.27646124362945557,
      "epoch": 0.0710506424792139,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.18519717454910278,
      "fcm_dpo/q_t": 0.49537193775177,
      "grad_norm": 28.432392120361328,
      "learning_rate": 3.432835820895522e-07,
      "logits/chosen": 0.05430842190980911,
      "logits/rejected": 0.01028415560722351,
      "logps/chosen": -51.70357131958008,
      "logps/ref_chosen": -51.578346252441406,
      "logps/ref_rejected": -68.2215576171875,
      "logps/rejected": -68.531982421875,
      "loss": 1.3682,
      "margin_dpo/margin_mean": 0.18519741296768188,
      "margin_dpo/margin_std": 0.378314733505249,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": -0.2560863494873047,
      "KL/mean": -0.2473316341638565,
      "KL/rejected_KL_mean": -0.23857879638671875,
      "KL/std": 0.33754950761795044,
      "epoch": 0.07256235827664399,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.017511337995529175,
      "fcm_dpo/q_t": 0.5004381537437439,
      "grad_norm": 26.973928451538086,
      "learning_rate": 3.507462686567164e-07,
      "logits/chosen": 0.15267328917980194,
      "logits/rejected": 0.12278998643159866,
      "logps/chosen": -52.04973602294922,
      "logps/ref_chosen": -51.79365158081055,
      "logps/ref_rejected": -64.22503662109375,
      "logps/rejected": -64.46361541748047,
      "loss": 1.3886,
      "margin_dpo/margin_mean": -0.017511069774627686,
      "margin_dpo/margin_std": 0.47600388526916504,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": -0.1961212158203125,
      "KL/mean": -0.2512929439544678,
      "KL/rejected_KL_mean": -0.30646514892578125,
      "KL/std": 0.3455272912979126,
      "epoch": 0.07407407407407407,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11034372448921204,
      "fcm_dpo/q_t": 0.49724340438842773,
      "grad_norm": 27.25345230102539,
      "learning_rate": 3.5820895522388055e-07,
      "logits/chosen": 0.013772407546639442,
      "logits/rejected": -0.007179133594036102,
      "logps/chosen": -58.33072280883789,
      "logps/ref_chosen": -58.13460159301758,
      "logps/ref_rejected": -64.63206481933594,
      "logps/rejected": -64.93853759765625,
      "loss": 1.3758,
      "margin_dpo/margin_mean": 0.11034420132637024,
      "margin_dpo/margin_std": 0.468539834022522,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": -0.3067760467529297,
      "KL/mean": -0.3461652398109436,
      "KL/rejected_KL_mean": -0.3855552673339844,
      "KL/std": 0.3353579044342041,
      "epoch": 0.07558578987150416,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07877588272094727,
      "fcm_dpo/q_t": 0.4980308413505554,
      "grad_norm": 27.978439331054688,
      "learning_rate": 3.6567164179104475e-07,
      "logits/chosen": 0.10822086036205292,
      "logits/rejected": 0.07860735058784485,
      "logps/chosen": -53.16321563720703,
      "logps/ref_chosen": -52.85643768310547,
      "logps/ref_rejected": -72.17460632324219,
      "logps/rejected": -72.5601577758789,
      "loss": 1.3788,
      "margin_dpo/margin_mean": 0.07877624034881592,
      "margin_dpo/margin_std": 0.39916181564331055,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": -0.2651844024658203,
      "KL/mean": -0.37544912099838257,
      "KL/rejected_KL_mean": -0.4857177734375,
      "KL/std": 0.3876107335090637,
      "epoch": 0.07709750566893424,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.22053340077400208,
      "fcm_dpo/q_t": 0.4944919943809509,
      "grad_norm": 31.027009963989258,
      "learning_rate": 3.7313432835820895e-07,
      "logits/chosen": 0.10386451333761215,
      "logits/rejected": 0.0755525678396225,
      "logps/chosen": -63.921627044677734,
      "logps/ref_chosen": -63.65644073486328,
      "logps/ref_rejected": -86.13229370117188,
      "logps/rejected": -86.61801147460938,
      "loss": 1.365,
      "margin_dpo/margin_mean": 0.22053375840187073,
      "margin_dpo/margin_std": 0.5061776041984558,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": -0.3102092742919922,
      "KL/mean": -0.3927253186702728,
      "KL/rejected_KL_mean": -0.47524261474609375,
      "KL/std": 0.3908153176307678,
      "epoch": 0.07860922146636433,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16503168642520905,
      "fcm_dpo/q_t": 0.49587881565093994,
      "grad_norm": 32.90274429321289,
      "learning_rate": 3.805970149253731e-07,
      "logits/chosen": 0.10359272360801697,
      "logits/rejected": 0.05269782245159149,
      "logps/chosen": -68.15042114257812,
      "logps/ref_chosen": -67.8402099609375,
      "logps/ref_rejected": -96.97090911865234,
      "logps/rejected": -97.44615173339844,
      "loss": 1.3705,
      "margin_dpo/margin_mean": 0.16503138840198517,
      "margin_dpo/margin_std": 0.5094035863876343,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": -0.35711097717285156,
      "KL/mean": -0.4368184208869934,
      "KL/rejected_KL_mean": -0.5165271759033203,
      "KL/std": 0.3795577883720398,
      "epoch": 0.0801209372637944,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15941789746284485,
      "fcm_dpo/q_t": 0.49601683020591736,
      "grad_norm": 27.472475051879883,
      "learning_rate": 3.880597014925373e-07,
      "logits/chosen": 0.08907319605350494,
      "logits/rejected": 0.07837893813848495,
      "logps/chosen": -57.235252380371094,
      "logps/ref_chosen": -56.87813949584961,
      "logps/ref_rejected": -60.75569152832031,
      "logps/rejected": -61.272220611572266,
      "loss": 1.371,
      "margin_dpo/margin_mean": 0.1594180166721344,
      "margin_dpo/margin_std": 0.46473363041877747,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": -0.36806488037109375,
      "KL/mean": -0.4340946674346924,
      "KL/rejected_KL_mean": -0.5001239776611328,
      "KL/std": 0.42446714639663696,
      "epoch": 0.08163265306122448,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.13206136226654053,
      "fcm_dpo/q_t": 0.49670255184173584,
      "grad_norm": 26.566865921020508,
      "learning_rate": 3.9552238805970144e-07,
      "logits/chosen": 0.0764525830745697,
      "logits/rejected": 0.060887373983860016,
      "logps/chosen": -47.634986877441406,
      "logps/ref_chosen": -47.26692199707031,
      "logps/ref_rejected": -62.19426727294922,
      "logps/rejected": -62.69438934326172,
      "loss": 1.3739,
      "margin_dpo/margin_mean": 0.13206100463867188,
      "margin_dpo/margin_std": 0.5414613485336304,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": -0.369964599609375,
      "KL/mean": -0.5161944031715393,
      "KL/rejected_KL_mean": -0.6624259948730469,
      "KL/std": 0.48815223574638367,
      "epoch": 0.08314436885865457,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2924594283103943,
      "fcm_dpo/q_t": 0.49270111322402954,
      "grad_norm": 31.41529083251953,
      "learning_rate": 4.0298507462686564e-07,
      "logits/chosen": 0.03972642123699188,
      "logits/rejected": -0.036940090358257294,
      "logps/chosen": -50.6961555480957,
      "logps/ref_chosen": -50.32619094848633,
      "logps/ref_rejected": -92.44389343261719,
      "logps/rejected": -93.10631561279297,
      "loss": 1.3584,
      "margin_dpo/margin_mean": 0.29245901107788086,
      "margin_dpo/margin_std": 0.6678496599197388,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": -0.38695526123046875,
      "KL/mean": -0.4743455946445465,
      "KL/rejected_KL_mean": -0.5617294311523438,
      "KL/std": 0.5216856002807617,
      "epoch": 0.08465608465608465,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.17477595806121826,
      "fcm_dpo/q_t": 0.49562889337539673,
      "grad_norm": 27.613101959228516,
      "learning_rate": 4.1044776119402984e-07,
      "logits/chosen": 0.13359126448631287,
      "logits/rejected": 0.11104871332645416,
      "logps/chosen": -57.153926849365234,
      "logps/ref_chosen": -56.766971588134766,
      "logps/ref_rejected": -66.30504608154297,
      "logps/rejected": -66.86677551269531,
      "loss": 1.37,
      "margin_dpo/margin_mean": 0.17477604746818542,
      "margin_dpo/margin_std": 0.6422098875045776,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": -0.5014972686767578,
      "KL/mean": -0.6661262512207031,
      "KL/rejected_KL_mean": -0.83074951171875,
      "KL/std": 0.6199729442596436,
      "epoch": 0.08616780045351474,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.32925212383270264,
      "fcm_dpo/q_t": 0.4917835295200348,
      "grad_norm": 30.380630493164062,
      "learning_rate": 4.17910447761194e-07,
      "logits/chosen": 0.09156804531812668,
      "logits/rejected": 0.027995580807328224,
      "logps/chosen": -58.26924133300781,
      "logps/ref_chosen": -57.76774597167969,
      "logps/ref_rejected": -82.75698852539062,
      "logps/rejected": -83.58773803710938,
      "loss": 1.3549,
      "margin_dpo/margin_mean": 0.3292523920536041,
      "margin_dpo/margin_std": 0.6963714361190796,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": -0.5995521545410156,
      "KL/mean": -0.7129923105239868,
      "KL/rejected_KL_mean": -0.8264312744140625,
      "KL/std": 0.6734466552734375,
      "epoch": 0.08767951625094482,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.22688013315200806,
      "fcm_dpo/q_t": 0.49438679218292236,
      "grad_norm": 30.21674346923828,
      "learning_rate": 4.253731343283582e-07,
      "logits/chosen": 0.06498520076274872,
      "logits/rejected": 0.04932039976119995,
      "logps/chosen": -73.36363983154297,
      "logps/ref_chosen": -72.76408386230469,
      "logps/ref_rejected": -84.49275207519531,
      "logps/rejected": -85.31918334960938,
      "loss": 1.3665,
      "margin_dpo/margin_mean": 0.22687986493110657,
      "margin_dpo/margin_std": 0.9897109270095825,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": -0.5455207824707031,
      "KL/mean": -0.7040902972221375,
      "KL/rejected_KL_mean": -0.8626651763916016,
      "KL/std": 0.6501806974411011,
      "epoch": 0.08919123204837491,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3171398937702179,
      "fcm_dpo/q_t": 0.49210113286972046,
      "grad_norm": 26.566312789916992,
      "learning_rate": 4.3283582089552234e-07,
      "logits/chosen": 0.10520491003990173,
      "logits/rejected": 0.0415894091129303,
      "logps/chosen": -50.366294860839844,
      "logps/ref_chosen": -49.820777893066406,
      "logps/ref_rejected": -77.14368438720703,
      "logps/rejected": -78.00634765625,
      "loss": 1.3567,
      "margin_dpo/margin_mean": 0.31714025139808655,
      "margin_dpo/margin_std": 0.8397043943405151,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": -0.6186370849609375,
      "KL/mean": -0.6712080240249634,
      "KL/rejected_KL_mean": -0.7237796783447266,
      "KL/std": 0.5914332866668701,
      "epoch": 0.09070294784580499,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10513994097709656,
      "fcm_dpo/q_t": 0.4973700940608978,
      "grad_norm": 29.50324821472168,
      "learning_rate": 4.4029850746268654e-07,
      "logits/chosen": 0.09925903379917145,
      "logits/rejected": 0.09783255308866501,
      "logps/chosen": -63.84341049194336,
      "logps/ref_chosen": -63.22477340698242,
      "logps/ref_rejected": -61.360477447509766,
      "logps/rejected": -62.084259033203125,
      "loss": 1.3773,
      "margin_dpo/margin_mean": 0.10513973236083984,
      "margin_dpo/margin_std": 0.7537751197814941,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": -0.7250652313232422,
      "KL/mean": -0.7618651390075684,
      "KL/rejected_KL_mean": -0.7986640930175781,
      "KL/std": 0.730823278427124,
      "epoch": 0.09221466364323508,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07359689474105835,
      "fcm_dpo/q_t": 0.4981638789176941,
      "grad_norm": 28.105127334594727,
      "learning_rate": 4.4776119402985074e-07,
      "logits/chosen": 0.13474620878696442,
      "logits/rejected": 0.10221856087446213,
      "logps/chosen": -49.74186325073242,
      "logps/ref_chosen": -49.01679992675781,
      "logps/ref_rejected": -74.90817260742188,
      "logps/rejected": -75.70683288574219,
      "loss": 1.3817,
      "margin_dpo/margin_mean": 0.07359689474105835,
      "margin_dpo/margin_std": 1.0419827699661255,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": -0.7413101196289062,
      "KL/mean": -0.8587090969085693,
      "KL/rejected_KL_mean": -0.9761085510253906,
      "KL/std": 0.6620085835456848,
      "epoch": 0.09372637944066516,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.23479923605918884,
      "fcm_dpo/q_t": 0.49413466453552246,
      "grad_norm": 28.633806228637695,
      "learning_rate": 4.552238805970149e-07,
      "logits/chosen": 0.08576183021068573,
      "logits/rejected": 0.04774682596325874,
      "logps/chosen": -63.49317932128906,
      "logps/ref_chosen": -62.751869201660156,
      "logps/ref_rejected": -78.93360900878906,
      "logps/rejected": -79.90971374511719,
      "loss": 1.3652,
      "margin_dpo/margin_mean": 0.23479923605918884,
      "margin_dpo/margin_std": 0.927647054195404,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": -0.5629634857177734,
      "KL/mean": -0.8584684133529663,
      "KL/rejected_KL_mean": -1.1539764404296875,
      "KL/std": 0.6872652173042297,
      "epoch": 0.09523809523809523,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5910177230834961,
      "fcm_dpo/q_t": 0.4852627217769623,
      "grad_norm": 31.272174835205078,
      "learning_rate": 4.626865671641791e-07,
      "logits/chosen": 0.18413883447647095,
      "logits/rejected": 0.15941709280014038,
      "logps/chosen": -61.07821273803711,
      "logps/ref_chosen": -60.51525115966797,
      "logps/ref_rejected": -85.11021423339844,
      "logps/rejected": -86.26419067382812,
      "loss": 1.3298,
      "margin_dpo/margin_mean": 0.591017484664917,
      "margin_dpo/margin_std": 0.8419014811515808,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": -0.8456764221191406,
      "KL/mean": -0.8847048282623291,
      "KL/rejected_KL_mean": -0.923736572265625,
      "KL/std": 0.7426958084106445,
      "epoch": 0.09674981103552532,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0780559778213501,
      "fcm_dpo/q_t": 0.4980408251285553,
      "grad_norm": 26.20693016052246,
      "learning_rate": 4.701492537313433e-07,
      "logits/chosen": 0.09699878096580505,
      "logits/rejected": 0.07188250124454498,
      "logps/chosen": -52.052520751953125,
      "logps/ref_chosen": -51.20684814453125,
      "logps/ref_rejected": -66.93081665039062,
      "logps/rejected": -67.85455322265625,
      "loss": 1.3807,
      "margin_dpo/margin_mean": 0.07805639505386353,
      "margin_dpo/margin_std": 0.9259697794914246,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": -0.8305568695068359,
      "KL/mean": -1.0773502588272095,
      "KL/rejected_KL_mean": -1.3241462707519531,
      "KL/std": 0.9229645729064941,
      "epoch": 0.0982615268329554,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.49359071254730225,
      "fcm_dpo/q_t": 0.48773229122161865,
      "grad_norm": 30.839191436767578,
      "learning_rate": 4.776119402985074e-07,
      "logits/chosen": 0.1971920132637024,
      "logits/rejected": 0.16757403314113617,
      "logps/chosen": -68.11924743652344,
      "logps/ref_chosen": -67.2886962890625,
      "logps/ref_rejected": -74.44281005859375,
      "logps/rejected": -75.76695251464844,
      "loss": 1.3412,
      "margin_dpo/margin_mean": 0.49358999729156494,
      "margin_dpo/margin_std": 1.2063360214233398,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": -0.9015731811523438,
      "KL/mean": -1.0950579643249512,
      "KL/rejected_KL_mean": -1.2885398864746094,
      "KL/std": 0.8506461381912231,
      "epoch": 0.09977324263038549,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3869714140892029,
      "fcm_dpo/q_t": 0.4903630018234253,
      "grad_norm": 29.211280822753906,
      "learning_rate": 4.850746268656717e-07,
      "logits/chosen": 0.07336064428091049,
      "logits/rejected": 0.05018358305096626,
      "logps/chosen": -71.64498901367188,
      "logps/ref_chosen": -70.743408203125,
      "logps/ref_rejected": -77.26499938964844,
      "logps/rejected": -78.55354309082031,
      "loss": 1.3513,
      "margin_dpo/margin_mean": 0.3869715631008148,
      "margin_dpo/margin_std": 1.139390230178833,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": -0.7808132171630859,
      "KL/mean": -0.936085045337677,
      "KL/rejected_KL_mean": -1.0913543701171875,
      "KL/std": 0.7684589624404907,
      "epoch": 0.10128495842781557,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3105343282222748,
      "fcm_dpo/q_t": 0.49226200580596924,
      "grad_norm": 29.08330726623535,
      "learning_rate": 4.925373134328357e-07,
      "logits/chosen": 0.11251701414585114,
      "logits/rejected": 0.05569060891866684,
      "logps/chosen": -61.38341522216797,
      "logps/ref_chosen": -60.60260009765625,
      "logps/ref_rejected": -75.22235870361328,
      "logps/rejected": -76.31371307373047,
      "loss": 1.3578,
      "margin_dpo/margin_mean": 0.3105342984199524,
      "margin_dpo/margin_std": 0.95625901222229,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": -1.2724533081054688,
      "KL/mean": -1.4205259084701538,
      "KL/rejected_KL_mean": -1.5685997009277344,
      "KL/std": 0.9099494218826294,
      "epoch": 0.10279667422524566,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.29615065455436707,
      "fcm_dpo/q_t": 0.4926130473613739,
      "grad_norm": 31.419614791870117,
      "learning_rate": 5e-07,
      "logits/chosen": 0.04703470319509506,
      "logits/rejected": 0.01785670965909958,
      "logps/chosen": -78.80081939697266,
      "logps/ref_chosen": -77.52836608886719,
      "logps/ref_rejected": -93.17778015136719,
      "logps/rejected": -94.74637603759766,
      "loss": 1.3604,
      "margin_dpo/margin_mean": 0.2961508333683014,
      "margin_dpo/margin_std": 1.1492292881011963,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": -1.152212142944336,
      "KL/mean": -1.4678752422332764,
      "KL/rejected_KL_mean": -1.7835311889648438,
      "KL/std": 0.9967954158782959,
      "epoch": 0.10430839002267574,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6313230991363525,
      "fcm_dpo/q_t": 0.48431456089019775,
      "grad_norm": 30.98222541809082,
      "learning_rate": 4.999965034812934e-07,
      "logits/chosen": 0.08557026088237762,
      "logits/rejected": 0.04331623762845993,
      "logps/chosen": -67.09526062011719,
      "logps/ref_chosen": -65.94305419921875,
      "logps/ref_rejected": -89.7735595703125,
      "logps/rejected": -91.55709075927734,
      "loss": 1.3278,
      "margin_dpo/margin_mean": 0.6313223838806152,
      "margin_dpo/margin_std": 1.183309555053711,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": -1.2136993408203125,
      "KL/mean": -1.3960089683532715,
      "KL/rejected_KL_mean": -1.5783195495605469,
      "KL/std": 0.8884181976318359,
      "epoch": 0.10582010582010581,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3646145164966583,
      "fcm_dpo/q_t": 0.4909105896949768,
      "grad_norm": 28.629304885864258,
      "learning_rate": 4.999860140229787e-07,
      "logits/chosen": 0.1712493747472763,
      "logits/rejected": 0.1474042534828186,
      "logps/chosen": -63.17161178588867,
      "logps/ref_chosen": -61.95791244506836,
      "logps/ref_rejected": -75.80945587158203,
      "logps/rejected": -77.38777923583984,
      "loss": 1.3539,
      "margin_dpo/margin_mean": 0.3646150529384613,
      "margin_dpo/margin_std": 1.2180376052856445,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": -1.4519882202148438,
      "KL/mean": -1.560695767402649,
      "KL/rejected_KL_mean": -1.6694049835205078,
      "KL/std": 1.0225567817687988,
      "epoch": 0.1073318216175359,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.21741512417793274,
      "fcm_dpo/q_t": 0.4945552945137024,
      "grad_norm": 28.560457229614258,
      "learning_rate": 4.999685319184688e-07,
      "logits/chosen": 0.10548342764377594,
      "logits/rejected": 0.08939765393733978,
      "logps/chosen": -64.79956817626953,
      "logps/ref_chosen": -63.34757995605469,
      "logps/ref_rejected": -67.49658203125,
      "logps/rejected": -69.16598510742188,
      "loss": 1.37,
      "margin_dpo/margin_mean": 0.2174154818058014,
      "margin_dpo/margin_std": 1.438173770904541,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": -1.1966476440429688,
      "KL/mean": -1.5650217533111572,
      "KL/rejected_KL_mean": -1.9333915710449219,
      "KL/std": 1.066718339920044,
      "epoch": 0.10884353741496598,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7367445826530457,
      "fcm_dpo/q_t": 0.4816743731498718,
      "grad_norm": 30.220441818237305,
      "learning_rate": 4.999440576567755e-07,
      "logits/chosen": 0.1182754710316658,
      "logits/rejected": 0.05511663109064102,
      "logps/chosen": -57.055946350097656,
      "logps/ref_chosen": -55.85929870605469,
      "logps/ref_rejected": -68.45423889160156,
      "logps/rejected": -70.38763427734375,
      "loss": 1.3184,
      "margin_dpo/margin_mean": 0.7367447018623352,
      "margin_dpo/margin_std": 1.2988572120666504,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": -1.8454322814941406,
      "KL/mean": -1.8656189441680908,
      "KL/rejected_KL_mean": -1.8858089447021484,
      "KL/std": 1.2100117206573486,
      "epoch": 0.11035525321239607,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.040384888648986816,
      "fcm_dpo/q_t": 0.49911391735076904,
      "grad_norm": 32.26237106323242,
      "learning_rate": 4.999125919224965e-07,
      "logits/chosen": 0.09246772527694702,
      "logits/rejected": 0.07867833971977234,
      "logps/chosen": -70.98423767089844,
      "logps/ref_chosen": -69.13880920410156,
      "logps/ref_rejected": -79.04586791992188,
      "logps/rejected": -80.93167877197266,
      "loss": 1.3884,
      "margin_dpo/margin_mean": 0.04038524627685547,
      "margin_dpo/margin_std": 1.5114188194274902,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": -1.2555103302001953,
      "KL/mean": -1.619905710220337,
      "KL/rejected_KL_mean": -1.9843025207519531,
      "KL/std": 1.012344479560852,
      "epoch": 0.11186696900982615,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7287911176681519,
      "fcm_dpo/q_t": 0.48189249634742737,
      "grad_norm": 27.792308807373047,
      "learning_rate": 4.998741355957963e-07,
      "logits/chosen": 0.14601758122444153,
      "logits/rejected": 0.09223125129938126,
      "logps/chosen": -51.17924499511719,
      "logps/ref_chosen": -49.923736572265625,
      "logps/ref_rejected": -81.73213958740234,
      "logps/rejected": -83.71644592285156,
      "loss": 1.3192,
      "margin_dpo/margin_mean": 0.7287905812263489,
      "margin_dpo/margin_std": 1.300370216369629,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": -1.3832435607910156,
      "KL/mean": -1.7584812641143799,
      "KL/rejected_KL_mean": -2.1337146759033203,
      "KL/std": 1.0733695030212402,
      "epoch": 0.11337868480725624,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7504755258560181,
      "fcm_dpo/q_t": 0.48133933544158936,
      "grad_norm": 26.247146606445312,
      "learning_rate": 4.998286897523808e-07,
      "logits/chosen": 0.11124646663665771,
      "logits/rejected": 0.07929684221744537,
      "logps/chosen": -47.451995849609375,
      "logps/ref_chosen": -46.06875228881836,
      "logps/ref_rejected": -66.1181411743164,
      "logps/rejected": -68.25186157226562,
      "loss": 1.3169,
      "margin_dpo/margin_mean": 0.7504750490188599,
      "margin_dpo/margin_std": 1.2960433959960938,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": -1.6049728393554688,
      "KL/mean": -1.7576977014541626,
      "KL/rejected_KL_mean": -1.9104232788085938,
      "KL/std": 1.2057301998138428,
      "epoch": 0.11489040060468632,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.30545294284820557,
      "fcm_dpo/q_t": 0.49238139390945435,
      "grad_norm": 29.37653160095215,
      "learning_rate": 4.997762556634679e-07,
      "logits/chosen": 0.1120544895529747,
      "logits/rejected": 0.06925463676452637,
      "logps/chosen": -55.667724609375,
      "logps/ref_chosen": -54.06275177001953,
      "logps/ref_rejected": -74.87464141845703,
      "logps/rejected": -76.78506469726562,
      "loss": 1.3618,
      "margin_dpo/margin_mean": 0.305452823638916,
      "margin_dpo/margin_std": 1.5330884456634521,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": -1.575113296508789,
      "KL/mean": -1.9794244766235352,
      "KL/rejected_KL_mean": -2.3837318420410156,
      "KL/std": 1.2247073650360107,
      "epoch": 0.1164021164021164,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.808618426322937,
      "fcm_dpo/q_t": 0.47992613911628723,
      "grad_norm": 29.050642013549805,
      "learning_rate": 4.99716834795752e-07,
      "logits/chosen": 0.16106051206588745,
      "logits/rejected": 0.11938437819480896,
      "logps/chosen": -54.651206970214844,
      "logps/ref_chosen": -53.07609176635742,
      "logps/ref_rejected": -74.45601654052734,
      "logps/rejected": -76.83975219726562,
      "loss": 1.3124,
      "margin_dpo/margin_mean": 0.8086180686950684,
      "margin_dpo/margin_std": 1.4267933368682861,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": -1.8742866516113281,
      "KL/mean": -2.0521607398986816,
      "KL/rejected_KL_mean": -2.2300338745117188,
      "KL/std": 1.3071465492248535,
      "epoch": 0.11791383219954649,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3557405471801758,
      "fcm_dpo/q_t": 0.4911442995071411,
      "grad_norm": 29.603862762451172,
      "learning_rate": 4.996504288113623e-07,
      "logits/chosen": 0.09281802177429199,
      "logits/rejected": 0.07283802330493927,
      "logps/chosen": -69.5997085571289,
      "logps/ref_chosen": -67.72541809082031,
      "logps/ref_rejected": -79.03926849365234,
      "logps/rejected": -81.26930236816406,
      "loss": 1.3563,
      "margin_dpo/margin_mean": 0.35574063658714294,
      "margin_dpo/margin_std": 1.4488558769226074,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": -1.71051025390625,
      "KL/mean": -2.288090467453003,
      "KL/rejected_KL_mean": -2.8656692504882812,
      "KL/std": 1.5266368389129639,
      "epoch": 0.11942554799697656,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1551591157913208,
      "fcm_dpo/q_t": 0.4716222286224365,
      "grad_norm": 30.841747283935547,
      "learning_rate": 4.995770395678171e-07,
      "logits/chosen": 0.1539367437362671,
      "logits/rejected": 0.09523071348667145,
      "logps/chosen": -53.87115478515625,
      "logps/ref_chosen": -52.16064453125,
      "logps/ref_rejected": -83.31062316894531,
      "logps/rejected": -86.17628479003906,
      "loss": 1.2838,
      "margin_dpo/margin_mean": 1.1551592350006104,
      "margin_dpo/margin_std": 1.974243402481079,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": -2.048025131225586,
      "KL/mean": -2.3210818767547607,
      "KL/rejected_KL_mean": -2.594135284423828,
      "KL/std": 1.3664963245391846,
      "epoch": 0.12093726379440665,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5461106300354004,
      "fcm_dpo/q_t": 0.486517071723938,
      "grad_norm": 28.41274070739746,
      "learning_rate": 4.994966691179711e-07,
      "logits/chosen": 0.16208377480506897,
      "logits/rejected": 0.10084976255893707,
      "logps/chosen": -63.458587646484375,
      "logps/ref_chosen": -61.410560607910156,
      "logps/ref_rejected": -78.66004943847656,
      "logps/rejected": -81.25418090820312,
      "loss": 1.3421,
      "margin_dpo/margin_mean": 0.546110987663269,
      "margin_dpo/margin_std": 1.9074268341064453,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": -2.1474876403808594,
      "KL/mean": -2.5709149837493896,
      "KL/rejected_KL_mean": -2.9943389892578125,
      "KL/std": 1.4404411315917969,
      "epoch": 0.12244897959183673,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8468587398529053,
      "fcm_dpo/q_t": 0.47913920879364014,
      "grad_norm": 29.03059196472168,
      "learning_rate": 4.994093197099587e-07,
      "logits/chosen": 0.1493208110332489,
      "logits/rejected": 0.11450086534023285,
      "logps/chosen": -65.95185852050781,
      "logps/ref_chosen": -63.80437088012695,
      "logps/ref_rejected": -79.3484115600586,
      "logps/rejected": -82.3427505493164,
      "loss": 1.3115,
      "margin_dpo/margin_mean": 0.8468592166900635,
      "margin_dpo/margin_std": 1.740824580192566,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": -1.7643756866455078,
      "KL/mean": -2.288670539855957,
      "KL/rejected_KL_mean": -2.812967300415039,
      "KL/std": 1.3698699474334717,
      "epoch": 0.12396069538926682,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0485919713974,
      "fcm_dpo/q_t": 0.47396597266197205,
      "grad_norm": 26.609189987182617,
      "learning_rate": 4.993149937871306e-07,
      "logits/chosen": 0.12430812418460846,
      "logits/rejected": 0.06034460663795471,
      "logps/chosen": -50.58226776123047,
      "logps/ref_chosen": -48.817893981933594,
      "logps/ref_rejected": -70.31497955322266,
      "logps/rejected": -73.12794494628906,
      "loss": 1.2906,
      "margin_dpo/margin_mean": 1.0485917329788208,
      "margin_dpo/margin_std": 1.5989954471588135,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": -2.0936527252197266,
      "KL/mean": -2.651911497116089,
      "KL/rejected_KL_mean": -3.210174560546875,
      "KL/std": 1.4586286544799805,
      "epoch": 0.1254724111866969,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1165223121643066,
      "fcm_dpo/q_t": 0.47243359684944153,
      "grad_norm": 29.370174407958984,
      "learning_rate": 4.992136939879856e-07,
      "logits/chosen": 0.203558087348938,
      "logits/rejected": 0.15304410457611084,
      "logps/chosen": -59.24442672729492,
      "logps/ref_chosen": -57.15077209472656,
      "logps/ref_rejected": -75.1710205078125,
      "logps/rejected": -78.38119506835938,
      "loss": 1.286,
      "margin_dpo/margin_mean": 1.1165223121643066,
      "margin_dpo/margin_std": 1.8079650402069092,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": -2.5359344482421875,
      "KL/mean": -2.9698383808135986,
      "KL/rejected_KL_mean": -3.40374755859375,
      "KL/std": 1.8658125400543213,
      "epoch": 0.12698412698412698,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8678133487701416,
      "fcm_dpo/q_t": 0.47854459285736084,
      "grad_norm": 30.701486587524414,
      "learning_rate": 4.991054231460969e-07,
      "logits/chosen": 0.1644321233034134,
      "logits/rejected": 0.12267635762691498,
      "logps/chosen": -67.313232421875,
      "logps/ref_chosen": -64.77729797363281,
      "logps/ref_rejected": -84.71949768066406,
      "logps/rejected": -88.12324523925781,
      "loss": 1.3107,
      "margin_dpo/margin_mean": 0.8678131103515625,
      "margin_dpo/margin_std": 1.9344353675842285,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": -2.246265411376953,
      "KL/mean": -2.887638568878174,
      "KL/rejected_KL_mean": -3.5290069580078125,
      "KL/std": 1.7709710597991943,
      "epoch": 0.12849584278155707,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2827439308166504,
      "fcm_dpo/q_t": 0.46833422780036926,
      "grad_norm": 26.880765914916992,
      "learning_rate": 4.989901842900325e-07,
      "logits/chosen": 0.15874062478542328,
      "logits/rejected": 0.11513642966747284,
      "logps/chosen": -52.497962951660156,
      "logps/ref_chosen": -50.25169372558594,
      "logps/ref_rejected": -66.55439758300781,
      "logps/rejected": -70.08340454101562,
      "loss": 1.2723,
      "margin_dpo/margin_mean": 1.2827439308166504,
      "margin_dpo/margin_std": 1.971299409866333,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": -2.840503692626953,
      "KL/mean": -3.3124403953552246,
      "KL/rejected_KL_mean": -3.784381866455078,
      "KL/std": 1.908517837524414,
      "epoch": 0.13000755857898716,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9438769817352295,
      "fcm_dpo/q_t": 0.4766838848590851,
      "grad_norm": 27.22740364074707,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 0.20484548807144165,
      "logits/rejected": 0.18582268059253693,
      "logps/chosen": -63.56968307495117,
      "logps/ref_chosen": -60.72917938232422,
      "logps/ref_rejected": -72.30961608886719,
      "logps/rejected": -76.093994140625,
      "loss": 1.305,
      "margin_dpo/margin_mean": 0.9438771605491638,
      "margin_dpo/margin_std": 2.0875279903411865,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": -2.993976593017578,
      "KL/mean": -3.515448808670044,
      "KL/rejected_KL_mean": -4.036918640136719,
      "KL/std": 2.058159589767456,
      "epoch": 0.13151927437641722,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0429438352584839,
      "fcm_dpo/q_t": 0.4742722809314728,
      "grad_norm": 30.60996437072754,
      "learning_rate": 4.987388156241114e-07,
      "logits/chosen": 0.1737605780363083,
      "logits/rejected": 0.1141999289393425,
      "logps/chosen": -68.75194549560547,
      "logps/ref_chosen": -65.75796508789062,
      "logps/ref_rejected": -84.81159973144531,
      "logps/rejected": -88.8485107421875,
      "loss": 1.3,
      "margin_dpo/margin_mean": 1.0429439544677734,
      "margin_dpo/margin_std": 2.4612228870391846,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": -3.053285598754883,
      "KL/mean": -3.4472389221191406,
      "KL/rejected_KL_mean": -3.8411865234375,
      "KL/std": 2.2897181510925293,
      "epoch": 0.1330309901738473,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7879018783569336,
      "fcm_dpo/q_t": 0.48072606325149536,
      "grad_norm": 30.390886306762695,
      "learning_rate": 4.986026928455767e-07,
      "logits/chosen": 0.22705325484275818,
      "logits/rejected": 0.20026172697544098,
      "logps/chosen": -65.87731170654297,
      "logps/ref_chosen": -62.82402801513672,
      "logps/ref_rejected": -74.9607162475586,
      "logps/rejected": -78.8019027709961,
      "loss": 1.3306,
      "margin_dpo/margin_mean": 0.7879022359848022,
      "margin_dpo/margin_std": 2.8485002517700195,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": -2.558351516723633,
      "KL/mean": -3.2431600093841553,
      "KL/rejected_KL_mean": -3.927967071533203,
      "KL/std": 2.133073329925537,
      "epoch": 0.1345427059712774,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3696104288101196,
      "fcm_dpo/q_t": 0.46659523248672485,
      "grad_norm": 29.33757209777832,
      "learning_rate": 4.984596161153135e-07,
      "logits/chosen": 0.24068006873130798,
      "logits/rejected": 0.15901678800582886,
      "logps/chosen": -43.749786376953125,
      "logps/ref_chosen": -41.191436767578125,
      "logps/ref_rejected": -85.44769287109375,
      "logps/rejected": -89.37566375732422,
      "loss": 1.2704,
      "margin_dpo/margin_mean": 1.369611382484436,
      "margin_dpo/margin_std": 2.50301194190979,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": -3.100625991821289,
      "KL/mean": -3.7633066177368164,
      "KL/rejected_KL_mean": -4.425987243652344,
      "KL/std": 2.4566781520843506,
      "epoch": 0.1360544217687075,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3253576755523682,
      "fcm_dpo/q_t": 0.4674978256225586,
      "grad_norm": 29.031574249267578,
      "learning_rate": 4.983095894354857e-07,
      "logits/chosen": 0.14617952704429626,
      "logits/rejected": 0.09175971895456314,
      "logps/chosen": -59.684532165527344,
      "logps/ref_chosen": -56.58390808105469,
      "logps/ref_rejected": -86.86978149414062,
      "logps/rejected": -91.29576873779297,
      "loss": 1.2746,
      "margin_dpo/margin_mean": 1.3253577947616577,
      "margin_dpo/margin_std": 2.5310139656066895,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": -3.18841552734375,
      "KL/mean": -3.7420177459716797,
      "KL/rejected_KL_mean": -4.295619964599609,
      "KL/std": 2.2847001552581787,
      "epoch": 0.13756613756613756,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1072009801864624,
      "fcm_dpo/q_t": 0.473149836063385,
      "grad_norm": 25.827465057373047,
      "learning_rate": 4.98152617002662e-07,
      "logits/chosen": 0.15624286234378815,
      "logits/rejected": 0.11258872598409653,
      "logps/chosen": -55.57075881958008,
      "logps/ref_chosen": -52.38234329223633,
      "logps/ref_rejected": -72.17642211914062,
      "logps/rejected": -76.4720458984375,
      "loss": 1.2981,
      "margin_dpo/margin_mean": 1.1072008609771729,
      "margin_dpo/margin_std": 2.7962405681610107,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": -3.2339859008789062,
      "KL/mean": -4.029601097106934,
      "KL/rejected_KL_mean": -4.825218200683594,
      "KL/std": 2.677659511566162,
      "epoch": 0.13907785336356765,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.591225266456604,
      "fcm_dpo/q_t": 0.46159979701042175,
      "grad_norm": 26.94463348388672,
      "learning_rate": 4.979887032076988e-07,
      "logits/chosen": 0.19445021450519562,
      "logits/rejected": 0.15572763979434967,
      "logps/chosen": -56.2426872253418,
      "logps/ref_chosen": -53.00870132446289,
      "logps/ref_rejected": -79.77812957763672,
      "logps/rejected": -84.60334777832031,
      "loss": 1.2577,
      "margin_dpo/margin_mean": 1.591225504875183,
      "margin_dpo/margin_std": 3.1309449672698975,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": -3.5741615295410156,
      "KL/mean": -4.070510387420654,
      "KL/rejected_KL_mean": -4.566856384277344,
      "KL/std": 2.6192855834960938,
      "epoch": 0.14058956916099774,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.992700457572937,
      "fcm_dpo/q_t": 0.47632110118865967,
      "grad_norm": 24.588478088378906,
      "learning_rate": 4.978178526356172e-07,
      "logits/chosen": 0.19541963934898376,
      "logits/rejected": 0.16661858558654785,
      "logps/chosen": -48.48121643066406,
      "logps/ref_chosen": -44.90705108642578,
      "logps/ref_rejected": -58.7879524230957,
      "logps/rejected": -63.35480880737305,
      "loss": 1.315,
      "margin_dpo/margin_mean": 0.9927000999450684,
      "margin_dpo/margin_std": 3.234583854675293,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": -3.2676773071289062,
      "KL/mean": -4.234769821166992,
      "KL/rejected_KL_mean": -5.201866149902344,
      "KL/std": 3.246453285217285,
      "epoch": 0.1421012849584278,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9341883659362793,
      "fcm_dpo/q_t": 0.4535469710826874,
      "grad_norm": 27.196422576904297,
      "learning_rate": 4.976400700654751e-07,
      "logits/chosen": 0.2802616059780121,
      "logits/rejected": 0.23843827843666077,
      "logps/chosen": -63.205448150634766,
      "logps/ref_chosen": -59.93777084350586,
      "logps/ref_rejected": -79.3138427734375,
      "logps/rejected": -84.51570129394531,
      "loss": 1.2407,
      "margin_dpo/margin_mean": 1.9341883659362793,
      "margin_dpo/margin_std": 3.9628539085388184,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": -4.273235321044922,
      "KL/mean": -5.078390121459961,
      "KL/rejected_KL_mean": -5.883544921875,
      "KL/std": 2.7410950660705566,
      "epoch": 0.1436130007558579,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6103031635284424,
      "fcm_dpo/q_t": 0.4611051082611084,
      "grad_norm": 29.146900177001953,
      "learning_rate": 4.974553604702332e-07,
      "logits/chosen": 0.1459863930940628,
      "logits/rejected": 0.08234073221683502,
      "logps/chosen": -64.44172668457031,
      "logps/ref_chosen": -60.168487548828125,
      "logps/ref_rejected": -90.73665618896484,
      "logps/rejected": -96.62020111083984,
      "loss": 1.2611,
      "margin_dpo/margin_mean": 1.6103029251098633,
      "margin_dpo/margin_std": 3.4583120346069336,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": -4.176933288574219,
      "KL/mean": -5.115758895874023,
      "KL/rejected_KL_mean": -6.054584503173828,
      "KL/std": 3.4817161560058594,
      "epoch": 0.14512471655328799,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.877647876739502,
      "fcm_dpo/q_t": 0.456631064414978,
      "grad_norm": 27.309680938720703,
      "learning_rate": 4.972637290166157e-07,
      "logits/chosen": 0.20910996198654175,
      "logits/rejected": 0.16543054580688477,
      "logps/chosen": -64.84571075439453,
      "logps/ref_chosen": -60.66877746582031,
      "logps/ref_rejected": -88.30673217773438,
      "logps/rejected": -94.36131286621094,
      "loss": 1.2584,
      "margin_dpo/margin_mean": 1.877647876739502,
      "margin_dpo/margin_std": 4.564449787139893,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": -5.000583648681641,
      "KL/mean": -5.523819923400879,
      "KL/rejected_KL_mean": -6.047061920166016,
      "KL/std": 3.636831760406494,
      "epoch": 0.14663643235071808,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0464853048324585,
      "fcm_dpo/q_t": 0.4750252962112427,
      "grad_norm": 36.2060546875,
      "learning_rate": 4.970651810649666e-07,
      "logits/chosen": 0.1086052805185318,
      "logits/rejected": 0.06623318046331406,
      "logps/chosen": -70.04470825195312,
      "logps/ref_chosen": -65.04412078857422,
      "logps/ref_rejected": -78.42092895507812,
      "logps/rejected": -84.46798706054688,
      "loss": 1.3438,
      "margin_dpo/margin_mean": 1.0464853048324585,
      "margin_dpo/margin_std": 4.956970691680908,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": -4.634487152099609,
      "KL/mean": -4.977179527282715,
      "KL/rejected_KL_mean": -5.319873809814453,
      "KL/std": 3.2767844200134277,
      "epoch": 0.14814814814814814,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6853851079940796,
      "fcm_dpo/q_t": 0.48287999629974365,
      "grad_norm": 30.321731567382812,
      "learning_rate": 4.968597221690985e-07,
      "logits/chosen": 0.2463841438293457,
      "logits/rejected": 0.21862992644309998,
      "logps/chosen": -60.137718200683594,
      "logps/ref_chosen": -55.503231048583984,
      "logps/ref_rejected": -72.81553649902344,
      "logps/rejected": -78.13541412353516,
      "loss": 1.3458,
      "margin_dpo/margin_mean": 0.6853851079940796,
      "margin_dpo/margin_std": 3.264176368713379,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": -4.686042785644531,
      "KL/mean": -5.184030532836914,
      "KL/rejected_KL_mean": -5.6820220947265625,
      "KL/std": 3.4981861114501953,
      "epoch": 0.14965986394557823,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.9959783554077148,
      "fcm_dpo/q_t": 0.4764708876609802,
      "grad_norm": 32.71485900878906,
      "learning_rate": 4.966473580761389e-07,
      "logits/chosen": 0.2241283655166626,
      "logits/rejected": 0.18563303351402283,
      "logps/chosen": -63.261680603027344,
      "logps/ref_chosen": -58.57563781738281,
      "logps/ref_rejected": -78.693603515625,
      "logps/rejected": -84.37562561035156,
      "loss": 1.3406,
      "margin_dpo/margin_mean": 0.9959783554077148,
      "margin_dpo/margin_std": 4.575559139251709,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": -5.299495697021484,
      "KL/mean": -6.027351379394531,
      "KL/rejected_KL_mean": -6.755210876464844,
      "KL/std": 4.143482208251953,
      "epoch": 0.15117157974300832,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4557148218154907,
      "fcm_dpo/q_t": 0.4673404097557068,
      "grad_norm": 33.2791862487793,
      "learning_rate": 4.964280947263676e-07,
      "logits/chosen": 0.22981172800064087,
      "logits/rejected": 0.22199641168117523,
      "logps/chosen": -84.8829345703125,
      "logps/ref_chosen": -79.58343505859375,
      "logps/ref_rejected": -92.152587890625,
      "logps/rejected": -98.90779113769531,
      "loss": 1.3324,
      "margin_dpo/margin_mean": 1.4557149410247803,
      "margin_dpo/margin_std": 5.845252513885498,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": -4.274101257324219,
      "KL/mean": -5.311479568481445,
      "KL/rejected_KL_mean": -6.348859786987305,
      "KL/std": 3.6799697875976562,
      "epoch": 0.15268329554043839,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0747623443603516,
      "fcm_dpo/q_t": 0.44993698596954346,
      "grad_norm": 25.746591567993164,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": 0.2311178743839264,
      "logits/rejected": 0.19193893671035767,
      "logps/chosen": -56.60688781738281,
      "logps/ref_chosen": -52.332786560058594,
      "logps/ref_rejected": -69.55589294433594,
      "logps/rejected": -75.90475463867188,
      "loss": 1.2324,
      "margin_dpo/margin_mean": 2.0747623443603516,
      "margin_dpo/margin_std": 4.121917724609375,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": -5.331483840942383,
      "KL/mean": -5.742671012878418,
      "KL/rejected_KL_mean": -6.153861999511719,
      "KL/std": 3.7148096561431885,
      "epoch": 0.15419501133786848,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8223755359649658,
      "fcm_dpo/q_t": 0.4832330346107483,
      "grad_norm": 32.66339874267578,
      "learning_rate": 4.959688949822748e-07,
      "logits/chosen": 0.18088501691818237,
      "logits/rejected": 0.14139467477798462,
      "logps/chosen": -70.07496643066406,
      "logps/ref_chosen": -64.74348449707031,
      "logps/ref_rejected": -69.06132507324219,
      "logps/rejected": -75.21519470214844,
      "loss": 1.376,
      "margin_dpo/margin_mean": 0.8223754167556763,
      "margin_dpo/margin_std": 5.470452308654785,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": -4.802188873291016,
      "KL/mean": -5.606751441955566,
      "KL/rejected_KL_mean": -6.41131591796875,
      "KL/std": 3.937784194946289,
      "epoch": 0.15570672713529857,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6091289520263672,
      "fcm_dpo/q_t": 0.4614095985889435,
      "grad_norm": 29.439403533935547,
      "learning_rate": 4.957289714327572e-07,
      "logits/chosen": 0.2667997479438782,
      "logits/rejected": 0.23269619047641754,
      "logps/chosen": -68.63883209228516,
      "logps/ref_chosen": -63.83664321899414,
      "logps/ref_rejected": -79.32362365722656,
      "logps/rejected": -85.73493957519531,
      "loss": 1.2651,
      "margin_dpo/margin_mean": 1.6091279983520508,
      "margin_dpo/margin_std": 3.6503872871398926,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": -4.722269058227539,
      "KL/mean": -5.856806755065918,
      "KL/rejected_KL_mean": -6.991344451904297,
      "KL/std": 4.956378936767578,
      "epoch": 0.15721844293272866,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.269068956375122,
      "fcm_dpo/q_t": 0.4513574540615082,
      "grad_norm": 30.835031509399414,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 0.24004673957824707,
      "logits/rejected": 0.15426743030548096,
      "logps/chosen": -65.72148132324219,
      "logps/ref_chosen": -60.99920654296875,
      "logps/ref_rejected": -98.84645080566406,
      "logps/rejected": -105.83779907226562,
      "loss": 1.2645,
      "margin_dpo/margin_mean": 2.2690696716308594,
      "margin_dpo/margin_std": 6.295289993286133,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": -5.31561279296875,
      "KL/mean": -6.242671966552734,
      "KL/rejected_KL_mean": -7.169731140136719,
      "KL/std": 4.008975028991699,
      "epoch": 0.15873015873015872,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.854116678237915,
      "fcm_dpo/q_t": 0.45764368772506714,
      "grad_norm": 30.735721588134766,
      "learning_rate": 4.952285105344791e-07,
      "logits/chosen": 0.21237248182296753,
      "logits/rejected": 0.15671700239181519,
      "logps/chosen": -76.26588439941406,
      "logps/ref_chosen": -70.95027160644531,
      "logps/ref_rejected": -87.88340759277344,
      "logps/rejected": -95.05314636230469,
      "loss": 1.2691,
      "margin_dpo/margin_mean": 1.8541167974472046,
      "margin_dpo/margin_std": 4.961075782775879,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": -5.028408050537109,
      "KL/mean": -5.82650089263916,
      "KL/rejected_KL_mean": -6.624591827392578,
      "KL/std": 3.8884782791137695,
      "epoch": 0.1602418745275888,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5961841344833374,
      "fcm_dpo/q_t": 0.4617699980735779,
      "grad_norm": 30.808412551879883,
      "learning_rate": 4.949679871846857e-07,
      "logits/chosen": 0.24007391929626465,
      "logits/rejected": 0.22604778409004211,
      "logps/chosen": -67.48773956298828,
      "logps/ref_chosen": -62.45933151245117,
      "logps/ref_rejected": -67.00595092773438,
      "logps/rejected": -73.63053894042969,
      "loss": 1.2995,
      "margin_dpo/margin_mean": 1.5961840152740479,
      "margin_dpo/margin_std": 5.234855651855469,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": -6.487876892089844,
      "KL/mean": -6.899287223815918,
      "KL/rejected_KL_mean": -7.310695648193359,
      "KL/std": 4.909206390380859,
      "epoch": 0.1617535903250189,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.8228192329406738,
      "fcm_dpo/q_t": 0.4787726402282715,
      "grad_norm": 41.80364227294922,
      "learning_rate": 4.947006115536947e-07,
      "logits/chosen": 0.15545688569545746,
      "logits/rejected": 0.13251692056655884,
      "logps/chosen": -82.32584381103516,
      "logps/ref_chosen": -75.83796691894531,
      "logps/ref_rejected": -87.74038696289062,
      "logps/rejected": -95.05107879638672,
      "loss": 1.3979,
      "margin_dpo/margin_mean": 0.8228189945220947,
      "margin_dpo/margin_std": 6.183783531188965,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": -5.504695892333984,
      "KL/mean": -6.444911956787109,
      "KL/rejected_KL_mean": -7.385128021240234,
      "KL/std": 4.147845268249512,
      "epoch": 0.16326530612244897,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8804302215576172,
      "fcm_dpo/q_t": 0.45709145069122314,
      "grad_norm": 28.610286712646484,
      "learning_rate": 4.944263911205772e-07,
      "logits/chosen": 0.19874165952205658,
      "logits/rejected": 0.16665475070476532,
      "logps/chosen": -73.89793395996094,
      "logps/ref_chosen": -68.39323425292969,
      "logps/ref_rejected": -83.24267578125,
      "logps/rejected": -90.62779998779297,
      "loss": 1.2779,
      "margin_dpo/margin_mean": 1.8804303407669067,
      "margin_dpo/margin_std": 5.413944244384766,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": -4.886608123779297,
      "KL/mean": -6.389529228210449,
      "KL/rejected_KL_mean": -7.892448425292969,
      "KL/std": 5.053153991699219,
      "epoch": 0.16477702191987906,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.0058376789093018,
      "fcm_dpo/q_t": 0.4337880611419678,
      "grad_norm": 28.171001434326172,
      "learning_rate": 4.941453335558681e-07,
      "logits/chosen": 0.17483273148536682,
      "logits/rejected": 0.12170157581567764,
      "logps/chosen": -60.41408920288086,
      "logps/ref_chosen": -55.52748107910156,
      "logps/ref_rejected": -83.55218505859375,
      "logps/rejected": -91.44464111328125,
      "loss": 1.1749,
      "margin_dpo/margin_mean": 3.0058372020721436,
      "margin_dpo/margin_std": 5.493310928344727,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": -6.248790740966797,
      "KL/mean": -6.400554656982422,
      "KL/rejected_KL_mean": -6.552318572998047,
      "KL/std": 4.0687785148620605,
      "epoch": 0.16628873771730915,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3035222589969635,
      "fcm_dpo/q_t": 0.49050843715667725,
      "grad_norm": 44.2472038269043,
      "learning_rate": 4.938574467213517e-07,
      "logits/chosen": 0.18585237860679626,
      "logits/rejected": 0.1952962875366211,
      "logps/chosen": -87.40753173828125,
      "logps/ref_chosen": -81.15874481201172,
      "logps/ref_rejected": -72.56021118164062,
      "logps/rejected": -79.11253356933594,
      "loss": 1.4264,
      "margin_dpo/margin_mean": 0.3035214841365814,
      "margin_dpo/margin_std": 5.468291282653809,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": -5.70982551574707,
      "KL/mean": -6.685848236083984,
      "KL/rejected_KL_mean": -7.661876678466797,
      "KL/std": 4.2794389724731445,
      "epoch": 0.16780045351473924,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.952048659324646,
      "fcm_dpo/q_t": 0.4560878276824951,
      "grad_norm": 25.860748291015625,
      "learning_rate": 4.935627386698418e-07,
      "logits/chosen": 0.29305583238601685,
      "logits/rejected": 0.2550733983516693,
      "logps/chosen": -58.068809509277344,
      "logps/ref_chosen": -52.358985900878906,
      "logps/ref_rejected": -77.06150817871094,
      "logps/rejected": -84.72338104248047,
      "loss": 1.266,
      "margin_dpo/margin_mean": 1.9520485401153564,
      "margin_dpo/margin_std": 5.244046688079834,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": -5.34930419921875,
      "KL/mean": -6.553627014160156,
      "KL/rejected_KL_mean": -7.7579498291015625,
      "KL/std": 4.263571739196777,
      "epoch": 0.1693121693121693,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.4086484909057617,
      "fcm_dpo/q_t": 0.4429079294204712,
      "grad_norm": 32.236202239990234,
      "learning_rate": 4.932612176449559e-07,
      "logits/chosen": 0.1755533367395401,
      "logits/rejected": 0.11200804263353348,
      "logps/chosen": -68.36936950683594,
      "logps/ref_chosen": -63.02006530761719,
      "logps/ref_rejected": -111.36941528320312,
      "logps/rejected": -119.12736511230469,
      "loss": 1.2155,
      "margin_dpo/margin_mean": 2.4086484909057617,
      "margin_dpo/margin_std": 4.813718795776367,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": -5.344881057739258,
      "KL/mean": -6.047405242919922,
      "KL/rejected_KL_mean": -6.749931335449219,
      "KL/std": 3.577111005783081,
      "epoch": 0.1708238851095994,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.4050495624542236,
      "fcm_dpo/q_t": 0.4661615192890167,
      "grad_norm": 35.494136810302734,
      "learning_rate": 4.929528920808854e-07,
      "logits/chosen": 0.20536774396896362,
      "logits/rejected": 0.16611523926258087,
      "logps/chosen": -61.15254211425781,
      "logps/ref_chosen": -55.80766296386719,
      "logps/ref_rejected": -69.84014129638672,
      "logps/rejected": -76.59007263183594,
      "loss": 1.3032,
      "margin_dpo/margin_mean": 1.405049204826355,
      "margin_dpo/margin_std": 4.589272499084473,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": -5.111001968383789,
      "KL/mean": -6.265921115875244,
      "KL/rejected_KL_mean": -7.420845031738281,
      "KL/std": 3.935863494873047,
      "epoch": 0.17233560090702948,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.309840202331543,
      "fcm_dpo/q_t": 0.4466482996940613,
      "grad_norm": 29.656740188598633,
      "learning_rate": 4.92637770602159e-07,
      "logits/chosen": 0.25359949469566345,
      "logits/rejected": 0.19307023286819458,
      "logps/chosen": -71.44377136230469,
      "logps/ref_chosen": -66.33277130126953,
      "logps/ref_rejected": -71.61489868164062,
      "logps/rejected": -79.03573608398438,
      "loss": 1.2227,
      "margin_dpo/margin_mean": 2.309840202331543,
      "margin_dpo/margin_std": 4.768823623657227,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": -5.491331100463867,
      "KL/mean": -6.419828414916992,
      "KL/rejected_KL_mean": -7.348325729370117,
      "KL/std": 3.607484817504883,
      "epoch": 0.17384731670445955,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8569921255111694,
      "fcm_dpo/q_t": 0.4564257860183716,
      "grad_norm": 28.554027557373047,
      "learning_rate": 4.923158620234019e-07,
      "logits/chosen": 0.23709729313850403,
      "logits/rejected": 0.1739954650402069,
      "logps/chosen": -61.24037170410156,
      "logps/ref_chosen": -55.74903869628906,
      "logps/ref_rejected": -79.59849548339844,
      "logps/rejected": -86.94682312011719,
      "loss": 1.2544,
      "margin_dpo/margin_mean": 1.8569918870925903,
      "margin_dpo/margin_std": 4.30812931060791,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": -5.2065277099609375,
      "KL/mean": -6.613717079162598,
      "KL/rejected_KL_mean": -8.02090835571289,
      "KL/std": 3.7173666954040527,
      "epoch": 0.17535903250188964,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.814383029937744,
      "fcm_dpo/q_t": 0.432966411113739,
      "grad_norm": 26.021554946899414,
      "learning_rate": 4.91987175349089e-07,
      "logits/chosen": 0.20693862438201904,
      "logits/rejected": 0.14074885845184326,
      "logps/chosen": -54.571693420410156,
      "logps/ref_chosen": -49.36516571044922,
      "logps/ref_rejected": -72.84671020507812,
      "logps/rejected": -80.86761474609375,
      "loss": 1.1659,
      "margin_dpo/margin_mean": 2.8143835067749023,
      "margin_dpo/margin_std": 4.077053070068359,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": -5.159467697143555,
      "KL/mean": -6.121030807495117,
      "KL/rejected_KL_mean": -7.0825958251953125,
      "KL/std": 3.491952896118164,
      "epoch": 0.17687074829931973,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.9231154918670654,
      "fcm_dpo/q_t": 0.45408564805984497,
      "grad_norm": 28.094541549682617,
      "learning_rate": 4.916517197732933e-07,
      "logits/chosen": 0.21251803636550903,
      "logits/rejected": 0.17677460610866547,
      "logps/chosen": -62.870365142822266,
      "logps/ref_chosen": -57.710899353027344,
      "logps/ref_rejected": -69.77253723144531,
      "logps/rejected": -76.85513305664062,
      "loss": 1.2545,
      "margin_dpo/margin_mean": 1.9231159687042236,
      "margin_dpo/margin_std": 4.358719825744629,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": -5.038366317749023,
      "KL/mean": -6.310278415679932,
      "KL/rejected_KL_mean": -7.582187652587891,
      "KL/std": 3.8248229026794434,
      "epoch": 0.17838246409674982,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.543827533721924,
      "fcm_dpo/q_t": 0.4391190707683563,
      "grad_norm": 28.034107208251953,
      "learning_rate": 4.913095046794281e-07,
      "logits/chosen": 0.294602632522583,
      "logits/rejected": 0.25325727462768555,
      "logps/chosen": -57.51826477050781,
      "logps/ref_chosen": -52.479896545410156,
      "logps/ref_rejected": -81.359130859375,
      "logps/rejected": -88.94131469726562,
      "loss": 1.1856,
      "margin_dpo/margin_mean": 2.5438270568847656,
      "margin_dpo/margin_std": 3.8979780673980713,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": -6.389644622802734,
      "KL/mean": -7.3072357177734375,
      "KL/rejected_KL_mean": -8.22482681274414,
      "KL/std": 4.4486799240112305,
      "epoch": 0.17989417989417988,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8351812362670898,
      "fcm_dpo/q_t": 0.4581907093524933,
      "grad_norm": 29.41862678527832,
      "learning_rate": 4.909605396399855e-07,
      "logits/chosen": 0.215057373046875,
      "logits/rejected": 0.17584341764450073,
      "logps/chosen": -67.747314453125,
      "logps/ref_chosen": -61.35767364501953,
      "logps/ref_rejected": -75.71510314941406,
      "logps/rejected": -83.93992614746094,
      "loss": 1.283,
      "margin_dpo/margin_mean": 1.8351819515228271,
      "margin_dpo/margin_std": 5.5315446853637695,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": -5.267726898193359,
      "KL/mean": -6.668299674987793,
      "KL/rejected_KL_mean": -8.06887435913086,
      "KL/std": 3.7570960521698,
      "epoch": 0.18140589569160998,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.801149845123291,
      "fcm_dpo/q_t": 0.4335363209247589,
      "grad_norm": 28.7716007232666,
      "learning_rate": 4.906048344162676e-07,
      "logits/chosen": 0.21911869943141937,
      "logits/rejected": 0.15837198495864868,
      "logps/chosen": -65.17529296875,
      "logps/ref_chosen": -59.907569885253906,
      "logps/ref_rejected": -79.6910629272461,
      "logps/rejected": -87.75993347167969,
      "loss": 1.1622,
      "margin_dpo/margin_mean": 2.80115008354187,
      "margin_dpo/margin_std": 3.8388729095458984,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": -5.821605682373047,
      "KL/mean": -7.307372570037842,
      "KL/rejected_KL_mean": -8.793140411376953,
      "KL/std": 4.403526306152344,
      "epoch": 0.18291761148904007,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.9715352058410645,
      "fcm_dpo/q_t": 0.43289828300476074,
      "grad_norm": 26.99895477294922,
      "learning_rate": 4.902423989581143e-07,
      "logits/chosen": 0.3170771598815918,
      "logits/rejected": 0.2303023487329483,
      "logps/chosen": -61.487648010253906,
      "logps/ref_chosen": -55.66604232788086,
      "logps/ref_rejected": -101.56233978271484,
      "logps/rejected": -110.35548400878906,
      "loss": 1.1654,
      "margin_dpo/margin_mean": 2.9715352058410645,
      "margin_dpo/margin_std": 4.898141860961914,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": -6.339637756347656,
      "KL/mean": -7.675061225891113,
      "KL/rejected_KL_mean": -9.010486602783203,
      "KL/std": 4.725244998931885,
      "epoch": 0.18442932728647016,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.6708438396453857,
      "fcm_dpo/q_t": 0.44164925813674927,
      "grad_norm": 30.728546142578125,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 0.24164050817489624,
      "logits/rejected": 0.20562344789505005,
      "logps/chosen": -69.67401123046875,
      "logps/ref_chosen": -63.334373474121094,
      "logps/ref_rejected": -73.67523193359375,
      "logps/rejected": -82.68572235107422,
      "loss": 1.2112,
      "margin_dpo/margin_mean": 2.670844078063965,
      "margin_dpo/margin_std": 5.738737106323242,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": -6.028675079345703,
      "KL/mean": -7.320953845977783,
      "KL/rejected_KL_mean": -8.613231658935547,
      "KL/std": 4.104480266571045,
      "epoch": 0.18594104308390022,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.5845589637756348,
      "fcm_dpo/q_t": 0.4395887851715088,
      "grad_norm": 26.99787139892578,
      "learning_rate": 4.894973780788722e-07,
      "logits/chosen": 0.25603556632995605,
      "logits/rejected": 0.214134082198143,
      "logps/chosen": -62.92741775512695,
      "logps/ref_chosen": -56.89874267578125,
      "logps/ref_rejected": -78.97028350830078,
      "logps/rejected": -87.58351135253906,
      "loss": 1.1913,
      "margin_dpo/margin_mean": 2.5845589637756348,
      "margin_dpo/margin_std": 4.423001289367676,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": -6.372430801391602,
      "KL/mean": -8.195686340332031,
      "KL/rejected_KL_mean": -10.018943786621094,
      "KL/std": 4.532326698303223,
      "epoch": 0.1874527588813303,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6465113162994385,
      "fcm_dpo/q_t": 0.41543495655059814,
      "grad_norm": 27.253131866455078,
      "learning_rate": 4.89114813497619e-07,
      "logits/chosen": 0.2840479016304016,
      "logits/rejected": 0.22104603052139282,
      "logps/chosen": -63.48851776123047,
      "logps/ref_chosen": -57.116085052490234,
      "logps/ref_rejected": -87.93074035644531,
      "logps/rejected": -97.94969177246094,
      "loss": 1.1103,
      "margin_dpo/margin_mean": 3.6465108394622803,
      "margin_dpo/margin_std": 4.869706153869629,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": -6.931028366088867,
      "KL/mean": -8.51264762878418,
      "KL/rejected_KL_mean": -10.094261169433594,
      "KL/std": 4.857443332672119,
      "epoch": 0.1889644746787604,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.163234233856201,
      "fcm_dpo/q_t": 0.4276365041732788,
      "grad_norm": 29.05904769897461,
      "learning_rate": 4.887255603610184e-07,
      "logits/chosen": 0.30631697177886963,
      "logits/rejected": 0.24442438781261444,
      "logps/chosen": -72.63720703125,
      "logps/ref_chosen": -65.7061767578125,
      "logps/ref_rejected": -91.72711944580078,
      "logps/rejected": -101.82138061523438,
      "loss": 1.1561,
      "margin_dpo/margin_mean": 3.163234233856201,
      "margin_dpo/margin_std": 5.063149452209473,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": -6.585702896118164,
      "KL/mean": -7.441903114318848,
      "KL/rejected_KL_mean": -8.298105239868164,
      "KL/std": 4.394684314727783,
      "epoch": 0.19047619047619047,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7124006748199463,
      "fcm_dpo/q_t": 0.45973747968673706,
      "grad_norm": 28.884706497192383,
      "learning_rate": 4.883296295573176e-07,
      "logits/chosen": 0.1231585294008255,
      "logits/rejected": 0.11754154413938522,
      "logps/chosen": -74.76179504394531,
      "logps/ref_chosen": -68.17608642578125,
      "logps/ref_rejected": -65.1175537109375,
      "logps/rejected": -73.41565704345703,
      "loss": 1.2761,
      "margin_dpo/margin_mean": 1.7124009132385254,
      "margin_dpo/margin_std": 4.733203887939453,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": -6.761753082275391,
      "KL/mean": -8.410690307617188,
      "KL/rejected_KL_mean": -10.059627532958984,
      "KL/std": 3.970813512802124,
      "epoch": 0.19198790627362056,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.297870635986328,
      "fcm_dpo/q_t": 0.4221423864364624,
      "grad_norm": 26.59172821044922,
      "learning_rate": 4.87927032161552e-07,
      "logits/chosen": 0.23097413778305054,
      "logits/rejected": 0.2001955211162567,
      "logps/chosen": -68.64198303222656,
      "logps/ref_chosen": -61.88023376464844,
      "logps/ref_rejected": -68.46012878417969,
      "logps/rejected": -78.5197525024414,
      "loss": 1.1298,
      "margin_dpo/margin_mean": 3.29787015914917,
      "margin_dpo/margin_std": 4.276577949523926,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": -7.305440902709961,
      "KL/mean": -9.139411926269531,
      "KL/rejected_KL_mean": -10.973388671875,
      "KL/std": 5.392538070678711,
      "epoch": 0.19349962207105065,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.667938709259033,
      "fcm_dpo/q_t": 0.4183761477470398,
      "grad_norm": 27.74724006652832,
      "learning_rate": 4.875177794352363e-07,
      "logits/chosen": 0.25080034136772156,
      "logits/rejected": 0.19237586855888367,
      "logps/chosen": -74.01441955566406,
      "logps/ref_chosen": -66.708984375,
      "logps/ref_rejected": -94.97969055175781,
      "logps/rejected": -105.95307922363281,
      "loss": 1.1497,
      "margin_dpo/margin_mean": 3.667938232421875,
      "margin_dpo/margin_std": 6.488053321838379,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": -8.281728744506836,
      "KL/mean": -9.327621459960938,
      "KL/rejected_KL_mean": -10.373516082763672,
      "KL/std": 4.979771137237549,
      "epoch": 0.19501133786848074,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.0917892456054688,
      "fcm_dpo/q_t": 0.451565682888031,
      "grad_norm": 34.47330093383789,
      "learning_rate": 4.871018828260491e-07,
      "logits/chosen": 0.2508179247379303,
      "logits/rejected": 0.24487638473510742,
      "logps/chosen": -73.62055969238281,
      "logps/ref_chosen": -65.33882904052734,
      "logps/ref_rejected": -68.06109619140625,
      "logps/rejected": -78.43460845947266,
      "loss": 1.2727,
      "margin_dpo/margin_mean": 2.091789722442627,
      "margin_dpo/margin_std": 5.9948272705078125,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": -7.84025764465332,
      "KL/mean": -9.535599708557129,
      "KL/rejected_KL_mean": -11.230945587158203,
      "KL/std": 4.948360443115234,
      "epoch": 0.1965230536659108,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.390688419342041,
      "fcm_dpo/q_t": 0.4209347069263458,
      "grad_norm": 30.479267120361328,
      "learning_rate": 4.866793539675126e-07,
      "logits/chosen": 0.2105114459991455,
      "logits/rejected": 0.15794897079467773,
      "logps/chosen": -66.50100708007812,
      "logps/ref_chosen": -58.660743713378906,
      "logps/ref_rejected": -79.24510192871094,
      "logps/rejected": -90.47604370117188,
      "loss": 1.1345,
      "margin_dpo/margin_mean": 3.390688180923462,
      "margin_dpo/margin_std": 4.888091087341309,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": -7.463371276855469,
      "KL/mean": -9.659799575805664,
      "KL/rejected_KL_mean": -11.856231689453125,
      "KL/std": 5.699565410614014,
      "epoch": 0.1980347694633409,
      "fcm_dpo/beta": 0.09919185191392899,
      "fcm_dpo/delta": -0.04057179018855095,
      "fcm_dpo/margin": 4.392857551574707,
      "fcm_dpo/q_t": 0.4031718075275421,
      "grad_norm": 25.621009826660156,
      "learning_rate": 4.86250204678667e-07,
      "logits/chosen": 0.20928484201431274,
      "logits/rejected": 0.1438637375831604,
      "logps/chosen": -59.9779052734375,
      "logps/ref_chosen": -52.51453399658203,
      "logps/ref_rejected": -85.18299865722656,
      "logps/rejected": -97.03923034667969,
      "loss": 1.0916,
      "margin_dpo/margin_mean": 4.392857551574707,
      "margin_dpo/margin_std": 6.408463478088379,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": -8.304485321044922,
      "KL/mean": -10.10638427734375,
      "KL/rejected_KL_mean": -11.908285140991211,
      "KL/std": 5.412184715270996,
      "epoch": 0.19954648526077098,
      "fcm_dpo/beta": 0.09919185191392899,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.603799343109131,
      "fcm_dpo/q_t": 0.41539686918258667,
      "grad_norm": 27.230810165405273,
      "learning_rate": 4.858144469637408e-07,
      "logits/chosen": 0.31601205468177795,
      "logits/rejected": 0.2823875844478607,
      "logps/chosen": -73.98961639404297,
      "logps/ref_chosen": -65.68513488769531,
      "logps/ref_rejected": -69.54120635986328,
      "logps/rejected": -81.44949340820312,
      "loss": 1.125,
      "margin_dpo/margin_mean": 3.603799343109131,
      "margin_dpo/margin_std": 5.2057013511657715,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": -8.788467407226562,
      "KL/mean": -10.21713638305664,
      "KL/rejected_KL_mean": -11.645797729492188,
      "KL/std": 5.376225471496582,
      "epoch": 0.20105820105820105,
      "fcm_dpo/beta": 0.09919185191392899,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.857332944869995,
      "fcm_dpo/q_t": 0.4344499409198761,
      "grad_norm": 31.633960723876953,
      "learning_rate": 4.853720930118138e-07,
      "logits/chosen": 0.19638074934482574,
      "logits/rejected": 0.18663090467453003,
      "logps/chosen": -72.38658142089844,
      "logps/ref_chosen": -63.598114013671875,
      "logps/ref_rejected": -73.72798156738281,
      "logps/rejected": -85.373779296875,
      "loss": 1.2047,
      "margin_dpo/margin_mean": 2.8573334217071533,
      "margin_dpo/margin_std": 5.904394149780273,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": -8.176952362060547,
      "KL/mean": -10.648435592651367,
      "KL/rejected_KL_mean": -13.119918823242188,
      "KL/std": 6.183529376983643,
      "epoch": 0.20256991685563114,
      "fcm_dpo/beta": 0.09795012325048447,
      "fcm_dpo/delta": -0.0884043499827385,
      "fcm_dpo/margin": 4.94296407699585,
      "fcm_dpo/q_t": 0.3913155496120453,
      "grad_norm": 24.997631072998047,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": 0.3347264528274536,
      "logits/rejected": 0.27579599618911743,
      "logps/chosen": -61.97152328491211,
      "logps/ref_chosen": -53.79457092285156,
      "logps/ref_rejected": -74.16741943359375,
      "logps/rejected": -87.28733825683594,
      "loss": 1.0465,
      "margin_dpo/margin_mean": 4.942964553833008,
      "margin_dpo/margin_std": 6.3145341873168945,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": -8.215858459472656,
      "KL/mean": -9.784494400024414,
      "KL/rejected_KL_mean": -11.353134155273438,
      "KL/std": 5.727636814117432,
      "epoch": 0.20408163265306123,
      "fcm_dpo/beta": 0.09745346754789352,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.1372694969177246,
      "fcm_dpo/q_t": 0.42929476499557495,
      "grad_norm": 24.82374382019043,
      "learning_rate": 4.844676460754862e-07,
      "logits/chosen": 0.24679645895957947,
      "logits/rejected": 0.21434196829795837,
      "logps/chosen": -57.65693664550781,
      "logps/ref_chosen": -49.441078186035156,
      "logps/ref_rejected": -65.96878051757812,
      "logps/rejected": -77.32191467285156,
      "loss": 1.1811,
      "margin_dpo/margin_mean": 3.1372690200805664,
      "margin_dpo/margin_std": 5.890544891357422,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": -10.235748291015625,
      "KL/mean": -12.271726608276367,
      "KL/rejected_KL_mean": -14.30771255493164,
      "KL/std": 6.477873802185059,
      "epoch": 0.20559334845049132,
      "fcm_dpo/beta": 0.0964834988117218,
      "fcm_dpo/delta": -0.05001489818096161,
      "fcm_dpo/margin": 4.071964740753174,
      "fcm_dpo/q_t": 0.41472935676574707,
      "grad_norm": 30.083513259887695,
      "learning_rate": 4.840055783904106e-07,
      "logits/chosen": 0.23875057697296143,
      "logits/rejected": 0.16482864320278168,
      "logps/chosen": -76.99501037597656,
      "logps/ref_chosen": -66.75926208496094,
      "logps/ref_rejected": -94.61787414550781,
      "logps/rejected": -108.92558288574219,
      "loss": 1.1717,
      "margin_dpo/margin_mean": 4.071965217590332,
      "margin_dpo/margin_std": 8.082297325134277,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": -9.556255340576172,
      "KL/mean": -11.618932723999023,
      "KL/rejected_KL_mean": -13.68161392211914,
      "KL/std": 6.124992847442627,
      "epoch": 0.20710506424792138,
      "fcm_dpo/beta": 0.09615467488765717,
      "fcm_dpo/delta": -0.034197039902210236,
      "fcm_dpo/margin": 4.125356674194336,
      "fcm_dpo/q_t": 0.41115665435791016,
      "grad_norm": 25.18506622314453,
      "learning_rate": 4.835369650662767e-07,
      "logits/chosen": 0.2673400044441223,
      "logits/rejected": 0.23783737421035767,
      "logps/chosen": -66.34005737304688,
      "logps/ref_chosen": -56.78379821777344,
      "logps/ref_rejected": -69.89952087402344,
      "logps/rejected": -83.58113098144531,
      "loss": 1.1175,
      "margin_dpo/margin_mean": 4.125357151031494,
      "margin_dpo/margin_std": 6.468832015991211,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": -11.306020736694336,
      "KL/mean": -13.123923301696777,
      "KL/rejected_KL_mean": -14.941829681396484,
      "KL/std": 6.329860687255859,
      "epoch": 0.20861678004535147,
      "fcm_dpo/beta": 0.09582585841417313,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6358046531677246,
      "fcm_dpo/q_t": 0.42061278223991394,
      "grad_norm": 26.653167724609375,
      "learning_rate": 4.830618192112065e-07,
      "logits/chosen": 0.2642987370491028,
      "logits/rejected": 0.22775018215179443,
      "logps/chosen": -70.07203674316406,
      "logps/ref_chosen": -58.766014099121094,
      "logps/ref_rejected": -68.12371826171875,
      "logps/rejected": -83.0655517578125,
      "loss": 1.181,
      "margin_dpo/margin_mean": 3.6358046531677246,
      "margin_dpo/margin_std": 7.232297897338867,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": -12.002849578857422,
      "KL/mean": -14.108619689941406,
      "KL/rejected_KL_mean": -16.214393615722656,
      "KL/std": 6.1752753257751465,
      "epoch": 0.21012849584278157,
      "fcm_dpo/beta": 0.09582585841417313,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.211538791656494,
      "fcm_dpo/q_t": 0.40779027342796326,
      "grad_norm": 29.489917755126953,
      "learning_rate": 4.825801541160509e-07,
      "logits/chosen": 0.23293596506118774,
      "logits/rejected": 0.20248082280158997,
      "logps/chosen": -83.22843933105469,
      "logps/ref_chosen": -71.2255859375,
      "logps/ref_rejected": -82.1834716796875,
      "logps/rejected": -98.39786529541016,
      "loss": 1.1095,
      "margin_dpo/margin_mean": 4.211538314819336,
      "margin_dpo/margin_std": 6.356287956237793,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": -10.922605514526367,
      "KL/mean": -13.90744400024414,
      "KL/rejected_KL_mean": -16.892284393310547,
      "KL/std": 8.277734756469727,
      "epoch": 0.21164021164021163,
      "fcm_dpo/beta": 0.09324315190315247,
      "fcm_dpo/delta": -0.16643883287906647,
      "fcm_dpo/margin": 5.9696760177612305,
      "fcm_dpo/q_t": 0.37920305132865906,
      "grad_norm": 29.492624282836914,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 0.2372860610485077,
      "logits/rejected": 0.1888073980808258,
      "logps/chosen": -74.20027160644531,
      "logps/ref_chosen": -63.27766418457031,
      "logps/ref_rejected": -83.30647277832031,
      "logps/rejected": -100.19876098632812,
      "loss": 1.074,
      "margin_dpo/margin_mean": 5.969675540924072,
      "margin_dpo/margin_std": 9.342610359191895,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": -12.200645446777344,
      "KL/mean": -14.866783142089844,
      "KL/rejected_KL_mean": -17.53292465209961,
      "KL/std": 6.956808090209961,
      "epoch": 0.21315192743764172,
      "fcm_dpo/beta": 0.09056098014116287,
      "fcm_dpo/delta": -0.11610686033964157,
      "fcm_dpo/margin": 5.332277774810791,
      "fcm_dpo/q_t": 0.3958725929260254,
      "grad_norm": 27.12528419494629,
      "learning_rate": 4.815973202802966e-07,
      "logits/chosen": 0.2885650098323822,
      "logits/rejected": 0.2414112687110901,
      "logps/chosen": -73.96741485595703,
      "logps/ref_chosen": -61.76676940917969,
      "logps/ref_rejected": -88.60601806640625,
      "logps/rejected": -106.13894653320312,
      "loss": 1.1056,
      "margin_dpo/margin_mean": 5.332278251647949,
      "margin_dpo/margin_std": 8.744904518127441,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": -11.824771881103516,
      "KL/mean": -14.046581268310547,
      "KL/rejected_KL_mean": -16.268394470214844,
      "KL/std": 6.713468551635742,
      "epoch": 0.2146636432350718,
      "fcm_dpo/beta": 0.09056098014116287,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.443621635437012,
      "fcm_dpo/q_t": 0.40913692116737366,
      "grad_norm": 25.67524528503418,
      "learning_rate": 4.810961790316729e-07,
      "logits/chosen": 0.28370508551597595,
      "logits/rejected": 0.2581597864627838,
      "logps/chosen": -77.09954833984375,
      "logps/ref_chosen": -65.2747802734375,
      "logps/ref_rejected": -81.1378173828125,
      "logps/rejected": -97.40621185302734,
      "loss": 1.1139,
      "margin_dpo/margin_mean": 4.443620681762695,
      "margin_dpo/margin_std": 6.891501426696777,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": -12.741323471069336,
      "KL/mean": -14.630672454833984,
      "KL/rejected_KL_mean": -16.520015716552734,
      "KL/std": 8.059417724609375,
      "epoch": 0.2161753590325019,
      "fcm_dpo/beta": 0.09056098014116287,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.7786927223205566,
      "fcm_dpo/q_t": 0.4216327667236328,
      "grad_norm": 35.36532974243164,
      "learning_rate": 4.805885735261454e-07,
      "logits/chosen": 0.3095983862876892,
      "logits/rejected": 0.2914605736732483,
      "logps/chosen": -75.35914611816406,
      "logps/ref_chosen": -62.617828369140625,
      "logps/ref_rejected": -70.39239501953125,
      "logps/rejected": -86.91240692138672,
      "loss": 1.2409,
      "margin_dpo/margin_mean": 3.7786929607391357,
      "margin_dpo/margin_std": 9.52112102508545,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": -12.927085876464844,
      "KL/mean": -15.399724960327148,
      "KL/rejected_KL_mean": -17.872360229492188,
      "KL/std": 7.987780570983887,
      "epoch": 0.21768707482993196,
      "fcm_dpo/beta": 0.08943817764520645,
      "fcm_dpo/delta": -0.06237911060452461,
      "fcm_dpo/margin": 4.945269584655762,
      "fcm_dpo/q_t": 0.4067836105823517,
      "grad_norm": 27.427934646606445,
      "learning_rate": 4.800745179625307e-07,
      "logits/chosen": 0.283732146024704,
      "logits/rejected": 0.2507033348083496,
      "logps/chosen": -73.72977447509766,
      "logps/ref_chosen": -60.80268859863281,
      "logps/ref_rejected": -79.07284545898438,
      "logps/rejected": -96.94520568847656,
      "loss": 1.1433,
      "margin_dpo/margin_mean": 4.945269584655762,
      "margin_dpo/margin_std": 9.206413269042969,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": -12.03497314453125,
      "KL/mean": -14.69092845916748,
      "KL/rejected_KL_mean": -17.346881866455078,
      "KL/std": 8.228681564331055,
      "epoch": 0.21919879062736206,
      "fcm_dpo/beta": 0.08845508843660355,
      "fcm_dpo/delta": -0.0552634559571743,
      "fcm_dpo/margin": 5.31191349029541,
      "fcm_dpo/q_t": 0.3977242112159729,
      "grad_norm": 30.822948455810547,
      "learning_rate": 4.795540267200686e-07,
      "logits/chosen": 0.24015925824642181,
      "logits/rejected": 0.256152868270874,
      "logps/chosen": -86.64643859863281,
      "logps/ref_chosen": -74.61146545410156,
      "logps/ref_rejected": -83.24461364746094,
      "logps/rejected": -100.59149169921875,
      "loss": 1.1399,
      "margin_dpo/margin_mean": 5.31191349029541,
      "margin_dpo/margin_std": 9.92963981628418,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": -11.776222229003906,
      "KL/mean": -14.419098854064941,
      "KL/rejected_KL_mean": -17.06197738647461,
      "KL/std": 8.018354415893555,
      "epoch": 0.22071050642479215,
      "fcm_dpo/beta": 0.08789414912462234,
      "fcm_dpo/delta": -0.06382076442241669,
      "fcm_dpo/margin": 5.285751819610596,
      "fcm_dpo/q_t": 0.39964884519577026,
      "grad_norm": 26.835702896118164,
      "learning_rate": 4.790271143580173e-07,
      "logits/chosen": 0.23064787685871124,
      "logits/rejected": 0.21661049127578735,
      "logps/chosen": -69.61720275878906,
      "logps/ref_chosen": -57.84098434448242,
      "logps/ref_rejected": -67.47422790527344,
      "logps/rejected": -84.53620910644531,
      "loss": 1.113,
      "margin_dpo/margin_mean": 5.2857513427734375,
      "margin_dpo/margin_std": 8.87518310546875,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": -13.491147994995117,
      "KL/mean": -15.663849830627441,
      "KL/rejected_KL_mean": -17.836559295654297,
      "KL/std": 8.625712394714355,
      "epoch": 0.2222222222222222,
      "fcm_dpo/beta": 0.08733320981264114,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.3454060554504395,
      "fcm_dpo/q_t": 0.4166657328605652,
      "grad_norm": 32.175376892089844,
      "learning_rate": 4.784937956152489e-07,
      "logits/chosen": 0.278936505317688,
      "logits/rejected": 0.2334621697664261,
      "logps/chosen": -80.30461883544922,
      "logps/ref_chosen": -66.81346893310547,
      "logps/ref_rejected": -81.1796875,
      "logps/rejected": -99.01625061035156,
      "loss": 1.1904,
      "margin_dpo/margin_mean": 4.345406532287598,
      "margin_dpo/margin_std": 9.280193328857422,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": -12.208412170410156,
      "KL/mean": -15.023258209228516,
      "KL/rejected_KL_mean": -17.83810043334961,
      "KL/std": 8.074333190917969,
      "epoch": 0.2237339380196523,
      "fcm_dpo/beta": 0.08660124242305756,
      "fcm_dpo/delta": -0.08452388644218445,
      "fcm_dpo/margin": 5.629689693450928,
      "fcm_dpo/q_t": 0.3923068046569824,
      "grad_norm": 22.291276931762695,
      "learning_rate": 4.779540854098347e-07,
      "logits/chosen": 0.3799922466278076,
      "logits/rejected": 0.30557140707969666,
      "logps/chosen": -60.89616775512695,
      "logps/ref_chosen": -48.6877555847168,
      "logps/ref_rejected": -67.50503540039062,
      "logps/rejected": -85.3431396484375,
      "loss": 1.0956,
      "margin_dpo/margin_mean": 5.629690170288086,
      "margin_dpo/margin_std": 8.904212951660156,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": -14.778066635131836,
      "KL/mean": -17.901935577392578,
      "KL/rejected_KL_mean": -21.025802612304688,
      "KL/std": 8.484848022460938,
      "epoch": 0.2252456538170824,
      "fcm_dpo/beta": 0.0842820554971695,
      "fcm_dpo/delta": -0.13348132371902466,
      "fcm_dpo/margin": 6.247742176055908,
      "fcm_dpo/q_t": 0.3821312189102173,
      "grad_norm": 23.382434844970703,
      "learning_rate": 4.774079988386296e-07,
      "logits/chosen": 0.24903056025505066,
      "logits/rejected": 0.20170432329177856,
      "logps/chosen": -69.92184448242188,
      "logps/ref_chosen": -55.143775939941406,
      "logps/ref_rejected": -64.79888916015625,
      "logps/rejected": -85.82469177246094,
      "loss": 1.0324,
      "margin_dpo/margin_mean": 6.24774169921875,
      "margin_dpo/margin_std": 8.07992935180664,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": -12.613779067993164,
      "KL/mean": -16.744375228881836,
      "KL/rejected_KL_mean": -20.87497329711914,
      "KL/std": 9.419195175170898,
      "epoch": 0.22675736961451248,
      "fcm_dpo/beta": 0.08011375367641449,
      "fcm_dpo/delta": -0.28073906898498535,
      "fcm_dpo/margin": 8.261198997497559,
      "fcm_dpo/q_t": 0.35848724842071533,
      "grad_norm": 22.24016761779785,
      "learning_rate": 4.768555511768486e-07,
      "logits/chosen": 0.24488872289657593,
      "logits/rejected": 0.20085716247558594,
      "logps/chosen": -80.08452606201172,
      "logps/ref_chosen": -67.47074890136719,
      "logps/ref_rejected": -89.21170806884766,
      "logps/rejected": -110.08668518066406,
      "loss": 0.9712,
      "margin_dpo/margin_mean": 8.261198043823242,
      "margin_dpo/margin_std": 9.845287322998047,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": -10.584939956665039,
      "KL/mean": -15.298471450805664,
      "KL/rejected_KL_mean": -20.012008666992188,
      "KL/std": 10.123547554016113,
      "epoch": 0.22826908541194255,
      "fcm_dpo/beta": 0.07523253560066223,
      "fcm_dpo/delta": -0.33289921283721924,
      "fcm_dpo/margin": 9.427069664001465,
      "fcm_dpo/q_t": 0.3464317321777344,
      "grad_norm": 20.026641845703125,
      "learning_rate": 4.762967578776406e-07,
      "logits/chosen": 0.27385711669921875,
      "logits/rejected": 0.21827185153961182,
      "logps/chosen": -63.04448318481445,
      "logps/ref_chosen": -52.45954132080078,
      "logps/ref_rejected": -79.0630111694336,
      "logps/rejected": -99.07501983642578,
      "loss": 0.9213,
      "margin_dpo/margin_mean": 9.427068710327148,
      "margin_dpo/margin_std": 10.230758666992188,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": -13.667095184326172,
      "KL/mean": -16.71866226196289,
      "KL/rejected_KL_mean": -19.770233154296875,
      "KL/std": 9.496614456176758,
      "epoch": 0.22978080120937264,
      "fcm_dpo/beta": 0.0733184814453125,
      "fcm_dpo/delta": -0.08626732975244522,
      "fcm_dpo/margin": 6.103139877319336,
      "fcm_dpo/q_t": 0.40311574935913086,
      "grad_norm": 22.40052604675293,
      "learning_rate": 4.757316345716553e-07,
      "logits/chosen": 0.343588650226593,
      "logits/rejected": 0.288934588432312,
      "logps/chosen": -70.22093200683594,
      "logps/ref_chosen": -56.5538330078125,
      "logps/ref_rejected": -76.55074310302734,
      "logps/rejected": -96.32097625732422,
      "loss": 1.1362,
      "margin_dpo/margin_mean": 6.103139877319336,
      "margin_dpo/margin_std": 10.882282257080078,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": -12.759742736816406,
      "KL/mean": -16.359935760498047,
      "KL/rejected_KL_mean": -19.960128784179688,
      "KL/std": 9.789167404174805,
      "epoch": 0.23129251700680273,
      "fcm_dpo/beta": 0.07141783833503723,
      "fcm_dpo/delta": -0.12039655447006226,
      "fcm_dpo/margin": 7.200382232666016,
      "fcm_dpo/q_t": 0.3846072554588318,
      "grad_norm": 20.95531463623047,
      "learning_rate": 4.751601970666064e-07,
      "logits/chosen": 0.2524608373641968,
      "logits/rejected": 0.21613654494285583,
      "logps/chosen": -80.76663208007812,
      "logps/ref_chosen": -68.00689697265625,
      "logps/ref_rejected": -74.83482360839844,
      "logps/rejected": -94.79495239257812,
      "loss": 1.0378,
      "margin_dpo/margin_mean": 7.200382709503174,
      "margin_dpo/margin_std": 9.319150924682617,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -13.830446243286133,
      "KL/mean": -16.472694396972656,
      "KL/rejected_KL_mean": -19.114944458007812,
      "KL/std": 10.336735725402832,
      "epoch": 0.2328042328042328,
      "fcm_dpo/beta": 0.07095666974782944,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.2844977378845215,
      "fcm_dpo/q_t": 0.4158506393432617,
      "grad_norm": 22.322782516479492,
      "learning_rate": 4.745824613468292e-07,
      "logits/chosen": 0.3161347806453705,
      "logits/rejected": 0.31378883123397827,
      "logps/chosen": -73.05298614501953,
      "logps/ref_chosen": -59.222537994384766,
      "logps/ref_rejected": -64.19131469726562,
      "logps/rejected": -83.30625915527344,
      "loss": 1.1798,
      "margin_dpo/margin_mean": 5.28449821472168,
      "margin_dpo/margin_std": 10.676871299743652,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -14.819591522216797,
      "KL/mean": -18.539260864257812,
      "KL/rejected_KL_mean": -22.258926391601562,
      "KL/std": 10.206151962280273,
      "epoch": 0.23431594860166288,
      "fcm_dpo/beta": 0.06935182213783264,
      "fcm_dpo/delta": -0.11438489705324173,
      "fcm_dpo/margin": 7.439325332641602,
      "fcm_dpo/q_t": 0.3903166353702545,
      "grad_norm": 21.931781768798828,
      "learning_rate": 4.7399844357283393e-07,
      "logits/chosen": 0.32706189155578613,
      "logits/rejected": 0.30682075023651123,
      "logps/chosen": -83.2742919921875,
      "logps/ref_chosen": -68.45469665527344,
      "logps/ref_rejected": -77.91763305664062,
      "logps/rejected": -100.17655944824219,
      "loss": 1.1136,
      "margin_dpo/margin_mean": 7.439325332641602,
      "margin_dpo/margin_std": 12.90648365020752,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -14.782560348510742,
      "KL/mean": -19.166622161865234,
      "KL/rejected_KL_mean": -23.55068588256836,
      "KL/std": 11.518760681152344,
      "epoch": 0.23582766439909297,
      "fcm_dpo/beta": 0.06735508888959885,
      "fcm_dpo/delta": -0.20239472389221191,
      "fcm_dpo/margin": 8.76812744140625,
      "fcm_dpo/q_t": 0.37083950638771057,
      "grad_norm": 20.634889602661133,
      "learning_rate": 4.7340816008085305e-07,
      "logits/chosen": 0.2739195227622986,
      "logits/rejected": 0.22890859842300415,
      "logps/chosen": -82.05215454101562,
      "logps/ref_chosen": -67.26959991455078,
      "logps/ref_rejected": -86.95914459228516,
      "logps/rejected": -110.50982666015625,
      "loss": 0.9973,
      "margin_dpo/margin_mean": 8.76812744140625,
      "margin_dpo/margin_std": 10.767464637756348,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -12.96525764465332,
      "KL/mean": -16.47336196899414,
      "KL/rejected_KL_mean": -19.981468200683594,
      "KL/std": 10.980501174926758,
      "epoch": 0.23733938019652306,
      "fcm_dpo/beta": 0.06471969187259674,
      "fcm_dpo/delta": -0.14324051141738892,
      "fcm_dpo/margin": 7.016205310821533,
      "fcm_dpo/q_t": 0.3986649513244629,
      "grad_norm": 20.302101135253906,
      "learning_rate": 4.728116273823847e-07,
      "logits/chosen": 0.2908157408237457,
      "logits/rejected": 0.27248579263687134,
      "logps/chosen": -67.73812866210938,
      "logps/ref_chosen": -54.77287292480469,
      "logps/ref_rejected": -63.87866973876953,
      "logps/rejected": -83.86013793945312,
      "loss": 1.1058,
      "margin_dpo/margin_mean": 7.016204833984375,
      "margin_dpo/margin_std": 10.806330680847168,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -14.528236389160156,
      "KL/mean": -18.347126007080078,
      "KL/rejected_KL_mean": -22.166015625,
      "KL/std": 11.439427375793457,
      "epoch": 0.23885109599395313,
      "fcm_dpo/beta": 0.06432721018791199,
      "fcm_dpo/delta": -0.061014823615550995,
      "fcm_dpo/margin": 7.637775897979736,
      "fcm_dpo/q_t": 0.39471274614334106,
      "grad_norm": 20.84183692932129,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 0.32839739322662354,
      "logits/rejected": 0.28976303339004517,
      "logps/chosen": -79.45095825195312,
      "logps/ref_chosen": -64.92271423339844,
      "logps/ref_rejected": -82.23789978027344,
      "logps/rejected": -104.40391540527344,
      "loss": 1.0947,
      "margin_dpo/margin_mean": 7.637775897979736,
      "margin_dpo/margin_std": 12.304495811462402,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -14.803840637207031,
      "KL/mean": -19.1191463470459,
      "KL/rejected_KL_mean": -23.4344482421875,
      "KL/std": 12.163007736206055,
      "epoch": 0.24036281179138322,
      "fcm_dpo/beta": 0.06311751902103424,
      "fcm_dpo/delta": -0.1294807642698288,
      "fcm_dpo/margin": 8.630603790283203,
      "fcm_dpo/q_t": 0.3808811604976654,
      "grad_norm": 21.571117401123047,
      "learning_rate": 4.715998812855304e-07,
      "logits/chosen": 0.357462614774704,
      "logits/rejected": 0.316484272480011,
      "logps/chosen": -71.850830078125,
      "logps/ref_chosen": -57.046993255615234,
      "logps/ref_rejected": -73.32441711425781,
      "logps/rejected": -96.75886535644531,
      "loss": 1.0613,
      "margin_dpo/margin_mean": 8.630603790283203,
      "margin_dpo/margin_std": 12.803619384765625,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": -16.574190139770508,
      "KL/mean": -20.34947967529297,
      "KL/rejected_KL_mean": -24.124771118164062,
      "KL/std": 11.359709739685059,
      "epoch": 0.2418745275888133,
      "fcm_dpo/beta": 0.0620175302028656,
      "fcm_dpo/delta": -0.04559735208749771,
      "fcm_dpo/margin": 7.550580978393555,
      "fcm_dpo/q_t": 0.4014434516429901,
      "grad_norm": 20.139118194580078,
      "learning_rate": 4.7098470178228755e-07,
      "logits/chosen": 0.18542692065238953,
      "logits/rejected": 0.14045989513397217,
      "logps/chosen": -66.381103515625,
      "logps/ref_chosen": -49.806915283203125,
      "logps/ref_rejected": -68.3370132446289,
      "logps/rejected": -92.46178436279297,
      "loss": 1.1185,
      "margin_dpo/margin_mean": 7.550581932067871,
      "margin_dpo/margin_std": 13.154045104980469,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": -16.436386108398438,
      "KL/mean": -20.611623764038086,
      "KL/rejected_KL_mean": -24.786865234375,
      "KL/std": 11.288849830627441,
      "epoch": 0.24338624338624337,
      "fcm_dpo/beta": 0.060755591839551926,
      "fcm_dpo/delta": -0.11299815028905869,
      "fcm_dpo/margin": 8.35047721862793,
      "fcm_dpo/q_t": 0.3882335424423218,
      "grad_norm": 18.839397430419922,
      "learning_rate": 4.703633408618955e-07,
      "logits/chosen": 0.3484821915626526,
      "logits/rejected": 0.30886244773864746,
      "logps/chosen": -68.93687438964844,
      "logps/ref_chosen": -52.50048828125,
      "logps/ref_rejected": -66.04540252685547,
      "logps/rejected": -90.83226776123047,
      "loss": 1.0766,
      "margin_dpo/margin_mean": 8.35047721862793,
      "margin_dpo/margin_std": 12.587860107421875,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": -17.52142333984375,
      "KL/mean": -22.971660614013672,
      "KL/rejected_KL_mean": -28.421897888183594,
      "KL/std": 12.801250457763672,
      "epoch": 0.24489795918367346,
      "fcm_dpo/beta": 0.05810220539569855,
      "fcm_dpo/delta": -0.2494203895330429,
      "fcm_dpo/margin": 10.90047836303711,
      "fcm_dpo/q_t": 0.3596557080745697,
      "grad_norm": 20.55975341796875,
      "learning_rate": 4.697358159051549e-07,
      "logits/chosen": 0.37461674213409424,
      "logits/rejected": 0.32385122776031494,
      "logps/chosen": -86.99061584472656,
      "logps/ref_chosen": -69.46919250488281,
      "logps/ref_rejected": -92.00952911376953,
      "logps/rejected": -120.43142700195312,
      "loss": 0.983,
      "margin_dpo/margin_mean": 10.900479316711426,
      "margin_dpo/margin_std": 13.380390167236328,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": -14.830795288085938,
      "KL/mean": -20.422958374023438,
      "KL/rejected_KL_mean": -26.015121459960938,
      "KL/std": 12.709386825561523,
      "epoch": 0.24640967498110355,
      "fcm_dpo/beta": 0.05606578290462494,
      "fcm_dpo/delta": -0.24128547310829163,
      "fcm_dpo/margin": 11.184324264526367,
      "fcm_dpo/q_t": 0.36260414123535156,
      "grad_norm": 18.12739372253418,
      "learning_rate": 4.691021444652876e-07,
      "logits/chosen": 0.31139200925827026,
      "logits/rejected": 0.2624328136444092,
      "logps/chosen": -65.44462585449219,
      "logps/ref_chosen": -50.613834381103516,
      "logps/ref_rejected": -74.62033081054688,
      "logps/rejected": -100.63545227050781,
      "loss": 0.9951,
      "margin_dpo/margin_mean": 11.184325218200684,
      "margin_dpo/margin_std": 13.240589141845703,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -16.169986724853516,
      "KL/mean": -21.591407775878906,
      "KL/rejected_KL_mean": -27.012828826904297,
      "KL/std": 13.454334259033203,
      "epoch": 0.24792139077853365,
      "fcm_dpo/beta": 0.05384572595357895,
      "fcm_dpo/delta": -0.16112825274467468,
      "fcm_dpo/margin": 10.842841148376465,
      "fcm_dpo/q_t": 0.37319236993789673,
      "grad_norm": 16.985376358032227,
      "learning_rate": 4.6846234426744624e-07,
      "logits/chosen": 0.32287657260894775,
      "logits/rejected": 0.256435751914978,
      "logps/chosen": -71.01809692382812,
      "logps/ref_chosen": -54.848114013671875,
      "logps/ref_rejected": -79.0630111694336,
      "logps/rejected": -106.07583618164062,
      "loss": 1.0313,
      "margin_dpo/margin_mean": 10.842840194702148,
      "margin_dpo/margin_std": 14.509929656982422,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -18.375152587890625,
      "KL/mean": -23.237693786621094,
      "KL/rejected_KL_mean": -28.100234985351562,
      "KL/std": 13.708030700683594,
      "epoch": 0.2494331065759637,
      "fcm_dpo/beta": 0.05240638554096222,
      "fcm_dpo/delta": -0.10911506414413452,
      "fcm_dpo/margin": 9.725078582763672,
      "fcm_dpo/q_t": 0.3871835172176361,
      "grad_norm": 17.69078254699707,
      "learning_rate": 4.678164332082175e-07,
      "logits/chosen": 0.3671991229057312,
      "logits/rejected": 0.31109094619750977,
      "logps/chosen": -69.46436309814453,
      "logps/ref_chosen": -51.089210510253906,
      "logps/ref_rejected": -71.23370361328125,
      "logps/rejected": -99.33393859863281,
      "loss": 1.0871,
      "margin_dpo/margin_mean": 9.725078582763672,
      "margin_dpo/margin_std": 14.881806373596191,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -17.435197830200195,
      "KL/mean": -21.304927825927734,
      "KL/rejected_KL_mean": -25.174659729003906,
      "KL/std": 13.740734100341797,
      "epoch": 0.2509448223733938,
      "fcm_dpo/beta": 0.051834575831890106,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.7394609451293945,
      "fcm_dpo/q_t": 0.41240036487579346,
      "grad_norm": 20.236228942871094,
      "learning_rate": 4.6716442935512214e-07,
      "logits/chosen": 0.3307892084121704,
      "logits/rejected": 0.2354990839958191,
      "logps/chosen": -80.62600708007812,
      "logps/ref_chosen": -63.19081115722656,
      "logps/ref_rejected": -93.8402099609375,
      "logps/rejected": -119.0148696899414,
      "loss": 1.1315,
      "margin_dpo/margin_mean": 7.739460468292236,
      "margin_dpo/margin_std": 13.147438049316406,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -14.410505294799805,
      "KL/mean": -20.310304641723633,
      "KL/rejected_KL_mean": -26.210105895996094,
      "KL/std": 13.815751075744629,
      "epoch": 0.25245653817082386,
      "fcm_dpo/beta": 0.050010696053504944,
      "fcm_dpo/delta": -0.20380280911922455,
      "fcm_dpo/margin": 11.79959487915039,
      "fcm_dpo/q_t": 0.3688342571258545,
      "grad_norm": 18.09819793701172,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": 0.2875286936759949,
      "logits/rejected": 0.2525779902935028,
      "logps/chosen": -73.33477783203125,
      "logps/ref_chosen": -58.92427062988281,
      "logps/ref_rejected": -72.97377014160156,
      "logps/rejected": -99.18387603759766,
      "loss": 0.985,
      "margin_dpo/margin_mean": 11.79959487915039,
      "margin_dpo/margin_std": 13.582077026367188,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": -17.27781105041504,
      "KL/mean": -22.058712005615234,
      "KL/rejected_KL_mean": -26.839614868164062,
      "KL/std": 15.021439552307129,
      "epoch": 0.25396825396825395,
      "fcm_dpo/beta": 0.049220357090234756,
      "fcm_dpo/delta": -0.05494771525263786,
      "fcm_dpo/margin": 9.561796188354492,
      "fcm_dpo/q_t": 0.39759543538093567,
      "grad_norm": 18.367740631103516,
      "learning_rate": 4.6584221638904767e-07,
      "logits/chosen": 0.32051700353622437,
      "logits/rejected": 0.282333642244339,
      "logps/chosen": -82.92919158935547,
      "logps/ref_chosen": -65.65138244628906,
      "logps/ref_rejected": -79.71418762207031,
      "logps/rejected": -106.55380249023438,
      "loss": 1.0793,
      "margin_dpo/margin_mean": 9.561796188354492,
      "margin_dpo/margin_std": 14.055609703063965,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -13.753213882446289,
      "KL/mean": -19.046161651611328,
      "KL/rejected_KL_mean": -24.339111328125,
      "KL/std": 14.64225959777832,
      "epoch": 0.25547996976568405,
      "fcm_dpo/beta": 0.0483536496758461,
      "fcm_dpo/delta": -0.11793233454227448,
      "fcm_dpo/margin": 10.585901260375977,
      "fcm_dpo/q_t": 0.39021724462509155,
      "grad_norm": 20.023157119750977,
      "learning_rate": 4.651720442612075e-07,
      "logits/chosen": 0.39085686206817627,
      "logits/rejected": 0.35531675815582275,
      "logps/chosen": -75.1790771484375,
      "logps/ref_chosen": -61.425865173339844,
      "logps/ref_rejected": -76.09590148925781,
      "logps/rejected": -100.43501281738281,
      "loss": 1.0937,
      "margin_dpo/margin_mean": 10.585901260375977,
      "margin_dpo/margin_std": 17.442779541015625,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": -12.770061492919922,
      "KL/mean": -18.207551956176758,
      "KL/rejected_KL_mean": -23.645044326782227,
      "KL/std": 14.344905853271484,
      "epoch": 0.25699168556311414,
      "fcm_dpo/beta": 0.04731673002243042,
      "fcm_dpo/delta": -0.12062279880046844,
      "fcm_dpo/margin": 10.874979972839355,
      "fcm_dpo/q_t": 0.38753047585487366,
      "grad_norm": 16.221012115478516,
      "learning_rate": 4.6449585330874425e-07,
      "logits/chosen": 0.33115026354789734,
      "logits/rejected": 0.3292052447795868,
      "logps/chosen": -69.42324829101562,
      "logps/ref_chosen": -56.65319061279297,
      "logps/ref_rejected": -63.45965576171875,
      "logps/rejected": -87.10469818115234,
      "loss": 1.1045,
      "margin_dpo/margin_mean": 10.874979019165039,
      "margin_dpo/margin_std": 17.857650756835938,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -15.31631088256836,
      "KL/mean": -21.237224578857422,
      "KL/rejected_KL_mean": -27.15814208984375,
      "KL/std": 16.512523651123047,
      "epoch": 0.2585034013605442,
      "fcm_dpo/beta": 0.045385200530290604,
      "fcm_dpo/delta": -0.1670508235692978,
      "fcm_dpo/margin": 11.841828346252441,
      "fcm_dpo/q_t": 0.38251635432243347,
      "grad_norm": 16.580425262451172,
      "learning_rate": 4.6381366244617224e-07,
      "logits/chosen": 0.401705265045166,
      "logits/rejected": 0.34923315048217773,
      "logps/chosen": -79.05107116699219,
      "logps/ref_chosen": -63.73476028442383,
      "logps/ref_rejected": -78.50328063964844,
      "logps/rejected": -105.66142272949219,
      "loss": 1.0877,
      "margin_dpo/margin_mean": 11.841829299926758,
      "margin_dpo/margin_std": 18.744915008544922,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": -17.131195068359375,
      "KL/mean": -22.653350830078125,
      "KL/rejected_KL_mean": -28.175498962402344,
      "KL/std": 16.811195373535156,
      "epoch": 0.2600151171579743,
      "fcm_dpo/beta": 0.044791221618652344,
      "fcm_dpo/delta": -0.13261796534061432,
      "fcm_dpo/margin": 11.044305801391602,
      "fcm_dpo/q_t": 0.3933247923851013,
      "grad_norm": 17.82714080810547,
      "learning_rate": 4.631254907558365e-07,
      "logits/chosen": 0.4093649089336395,
      "logits/rejected": 0.34693312644958496,
      "logps/chosen": -69.33295440673828,
      "logps/ref_chosen": -52.201759338378906,
      "logps/ref_rejected": -82.85285949707031,
      "logps/rejected": -111.02836608886719,
      "loss": 1.0927,
      "margin_dpo/margin_mean": 11.044305801391602,
      "margin_dpo/margin_std": 17.3895263671875,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -15.58431625366211,
      "KL/mean": -21.632266998291016,
      "KL/rejected_KL_mean": -27.680221557617188,
      "KL/std": 17.105926513671875,
      "epoch": 0.2615268329554044,
      "fcm_dpo/beta": 0.04276077076792717,
      "fcm_dpo/delta": -0.16520676016807556,
      "fcm_dpo/margin": 12.095901489257812,
      "fcm_dpo/q_t": 0.3909044563770294,
      "grad_norm": 17.054214477539062,
      "learning_rate": 4.624313574873786e-07,
      "logits/chosen": 0.3971649408340454,
      "logits/rejected": 0.30516812205314636,
      "logps/chosen": -71.01904296875,
      "logps/ref_chosen": -55.434722900390625,
      "logps/ref_rejected": -77.81967163085938,
      "logps/rejected": -105.49989318847656,
      "loss": 1.1222,
      "margin_dpo/margin_mean": 12.095902442932129,
      "margin_dpo/margin_std": 21.07677459716797,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -17.717504501342773,
      "KL/mean": -24.340293884277344,
      "KL/rejected_KL_mean": -30.963085174560547,
      "KL/std": 17.349761962890625,
      "epoch": 0.26303854875283444,
      "fcm_dpo/beta": 0.04165055602788925,
      "fcm_dpo/delta": -0.16104662418365479,
      "fcm_dpo/margin": 13.245588302612305,
      "fcm_dpo/q_t": 0.3786090612411499,
      "grad_norm": 17.300262451171875,
      "learning_rate": 4.61731282057198e-07,
      "logits/chosen": 0.374323308467865,
      "logits/rejected": 0.3048890233039856,
      "logps/chosen": -74.88945770263672,
      "logps/ref_chosen": -57.17195129394531,
      "logps/ref_rejected": -85.47578430175781,
      "logps/rejected": -116.4388656616211,
      "loss": 1.0454,
      "margin_dpo/margin_mean": 13.245588302612305,
      "margin_dpo/margin_std": 18.74484634399414,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -15.173547744750977,
      "KL/mean": -22.60834503173828,
      "KL/rejected_KL_mean": -30.043148040771484,
      "KL/std": 17.65919303894043,
      "epoch": 0.26455026455026454,
      "fcm_dpo/beta": 0.04021874815225601,
      "fcm_dpo/delta": -0.21028748154640198,
      "fcm_dpo/margin": 14.869599342346191,
      "fcm_dpo/q_t": 0.37366974353790283,
      "grad_norm": 16.226505279541016,
      "learning_rate": 4.6102528404790965e-07,
      "logits/chosen": 0.3995314836502075,
      "logits/rejected": 0.3691650629043579,
      "logps/chosen": -82.83917236328125,
      "logps/ref_chosen": -67.6656265258789,
      "logps/ref_rejected": -84.36766815185547,
      "logps/rejected": -114.41081237792969,
      "loss": 1.0296,
      "margin_dpo/margin_mean": 14.869599342346191,
      "margin_dpo/margin_std": 20.803794860839844,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -19.654399871826172,
      "KL/mean": -24.40457534790039,
      "KL/rejected_KL_mean": -29.154754638671875,
      "KL/std": 18.916370391845703,
      "epoch": 0.2660619803476946,
      "fcm_dpo/beta": 0.039426859468221664,
      "fcm_dpo/delta": -0.03453513979911804,
      "fcm_dpo/margin": 9.500359535217285,
      "fcm_dpo/q_t": 0.41981422901153564,
      "grad_norm": 20.208744049072266,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 0.3505083918571472,
      "logits/rejected": 0.3238475024700165,
      "logps/chosen": -97.5131607055664,
      "logps/ref_chosen": -77.8587646484375,
      "logps/ref_rejected": -81.08732604980469,
      "logps/rejected": -110.24208068847656,
      "loss": 1.2111,
      "margin_dpo/margin_mean": 9.500359535217285,
      "margin_dpo/margin_std": 21.4652099609375,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -14.536983489990234,
      "KL/mean": -23.876373291015625,
      "KL/rejected_KL_mean": -33.21575927734375,
      "KL/std": 19.081396102905273,
      "epoch": 0.2675736961451247,
      "fcm_dpo/beta": 0.03744693845510483,
      "fcm_dpo/delta": -0.3234514594078064,
      "fcm_dpo/margin": 18.67877197265625,
      "fcm_dpo/q_t": 0.34551703929901123,
      "grad_norm": 19.835054397583008,
      "learning_rate": 4.5959559945025183e-07,
      "logits/chosen": 0.4804215729236603,
      "logits/rejected": 0.38235175609588623,
      "logps/chosen": -69.75738525390625,
      "logps/ref_chosen": -55.22039794921875,
      "logps/ref_rejected": -92.54973602294922,
      "logps/rejected": -125.76549530029297,
      "loss": 0.9197,
      "margin_dpo/margin_mean": 18.67877197265625,
      "margin_dpo/margin_std": 19.456634521484375,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -17.895334243774414,
      "KL/mean": -23.562305450439453,
      "KL/rejected_KL_mean": -29.22927474975586,
      "KL/std": 18.05139923095703,
      "epoch": 0.2690854119425548,
      "fcm_dpo/beta": 0.036295369267463684,
      "fcm_dpo/delta": -0.09033413976430893,
      "fcm_dpo/margin": 11.333934783935547,
      "fcm_dpo/q_t": 0.40735214948654175,
      "grad_norm": 19.553966522216797,
      "learning_rate": 4.588719528532341e-07,
      "logits/chosen": 0.3673199415206909,
      "logits/rejected": 0.3156449496746063,
      "logps/chosen": -78.70582580566406,
      "logps/ref_chosen": -60.81049346923828,
      "logps/ref_rejected": -81.12973022460938,
      "logps/rejected": -110.35900115966797,
      "loss": 1.1214,
      "margin_dpo/margin_mean": 11.333934783935547,
      "margin_dpo/margin_std": 18.04642105102539,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -18.201736450195312,
      "KL/mean": -25.386974334716797,
      "KL/rejected_KL_mean": -32.57221221923828,
      "KL/std": 19.182567596435547,
      "epoch": 0.2705971277399849,
      "fcm_dpo/beta": 0.0356178879737854,
      "fcm_dpo/delta": -0.11806750297546387,
      "fcm_dpo/margin": 14.370477676391602,
      "fcm_dpo/q_t": 0.3880265951156616,
      "grad_norm": 17.179960250854492,
      "learning_rate": 4.581424636586928e-07,
      "logits/chosen": 0.4421459436416626,
      "logits/rejected": 0.4255564212799072,
      "logps/chosen": -83.87345123291016,
      "logps/ref_chosen": -65.67171478271484,
      "logps/ref_rejected": -75.32586669921875,
      "logps/rejected": -107.89807891845703,
      "loss": 1.0816,
      "margin_dpo/margin_mean": 14.370477676391602,
      "margin_dpo/margin_std": 22.50054931640625,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -13.901803970336914,
      "KL/mean": -19.616230010986328,
      "KL/rejected_KL_mean": -25.330659866333008,
      "KL/std": 18.365304946899414,
      "epoch": 0.272108843537415,
      "fcm_dpo/beta": 0.0354483462870121,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.428857803344727,
      "fcm_dpo/q_t": 0.41211187839508057,
      "grad_norm": 14.897982597351074,
      "learning_rate": 4.5740715227200897e-07,
      "logits/chosen": 0.2380199283361435,
      "logits/rejected": 0.21933087706565857,
      "logps/chosen": -70.58460998535156,
      "logps/ref_chosen": -56.68280792236328,
      "logps/ref_rejected": -64.94414520263672,
      "logps/rejected": -90.27481079101562,
      "loss": 1.1499,
      "margin_dpo/margin_mean": 11.428857803344727,
      "margin_dpo/margin_std": 21.246871948242188,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -12.012491226196289,
      "KL/mean": -20.349918365478516,
      "KL/rejected_KL_mean": -28.68734359741211,
      "KL/std": 19.755210876464844,
      "epoch": 0.273620559334845,
      "fcm_dpo/beta": 0.03441469371318817,
      "fcm_dpo/delta": -0.1849377453327179,
      "fcm_dpo/margin": 16.674854278564453,
      "fcm_dpo/q_t": 0.37329137325286865,
      "grad_norm": 17.271093368530273,
      "learning_rate": 4.566660392614228e-07,
      "logits/chosen": 0.4154462218284607,
      "logits/rejected": 0.37344780564308167,
      "logps/chosen": -72.78854370117188,
      "logps/ref_chosen": -60.77604675292969,
      "logps/ref_rejected": -83.98361206054688,
      "logps/rejected": -112.67095947265625,
      "loss": 0.9922,
      "margin_dpo/margin_mean": 16.674854278564453,
      "margin_dpo/margin_std": 19.586292266845703,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -18.33358383178711,
      "KL/mean": -26.865509033203125,
      "KL/rejected_KL_mean": -35.397430419921875,
      "KL/std": 21.36233139038086,
      "epoch": 0.2751322751322751,
      "fcm_dpo/beta": 0.03329051285982132,
      "fcm_dpo/delta": -0.17808839678764343,
      "fcm_dpo/margin": 17.06385040283203,
      "fcm_dpo/q_t": 0.37668097019195557,
      "grad_norm": 18.594280242919922,
      "learning_rate": 4.5591914535745817e-07,
      "logits/chosen": 0.3790324926376343,
      "logits/rejected": 0.29383039474487305,
      "logps/chosen": -78.58737182617188,
      "logps/ref_chosen": -60.2537841796875,
      "logps/ref_rejected": -89.7706298828125,
      "logps/rejected": -125.16806030273438,
      "loss": 1.0409,
      "margin_dpo/margin_mean": 17.06385040283203,
      "margin_dpo/margin_std": 24.047889709472656,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -22.590940475463867,
      "KL/mean": -26.173816680908203,
      "KL/rejected_KL_mean": -29.75668716430664,
      "KL/std": 20.493114471435547,
      "epoch": 0.2766439909297052,
      "fcm_dpo/beta": 0.032965824007987976,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.1657514572143555,
      "fcm_dpo/q_t": 0.44580399990081787,
      "grad_norm": 16.00063705444336,
      "learning_rate": 4.551664914523433e-07,
      "logits/chosen": 0.3670530319213867,
      "logits/rejected": 0.3447875380516052,
      "logps/chosen": -84.35235595703125,
      "logps/ref_chosen": -61.76142120361328,
      "logps/ref_rejected": -72.54627990722656,
      "logps/rejected": -102.30297088623047,
      "loss": 1.2746,
      "margin_dpo/margin_mean": 7.1657514572143555,
      "margin_dpo/margin_std": 20.836429595947266,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -13.242368698120117,
      "KL/mean": -19.70210838317871,
      "KL/rejected_KL_mean": -26.161846160888672,
      "KL/std": 17.709320068359375,
      "epoch": 0.2781557067271353,
      "fcm_dpo/beta": 0.03258639574050903,
      "fcm_dpo/delta": -0.0578826479613781,
      "fcm_dpo/margin": 12.919473648071289,
      "fcm_dpo/q_t": 0.40624159574508667,
      "grad_norm": 16.270227432250977,
      "learning_rate": 4.544080985994258e-07,
      "logits/chosen": 0.4893711805343628,
      "logits/rejected": 0.42196178436279297,
      "logps/chosen": -60.083091735839844,
      "logps/ref_chosen": -46.840721130371094,
      "logps/ref_rejected": -69.3609390258789,
      "logps/rejected": -95.52278137207031,
      "loss": 1.097,
      "margin_dpo/margin_mean": 12.919473648071289,
      "margin_dpo/margin_std": 18.708168029785156,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -17.824491500854492,
      "KL/mean": -25.49761199951172,
      "KL/rejected_KL_mean": -33.17073440551758,
      "KL/std": 21.816608428955078,
      "epoch": 0.2796674225245654,
      "fcm_dpo/beta": 0.03175631910562515,
      "fcm_dpo/delta": -0.12901580333709717,
      "fcm_dpo/margin": 15.346242904663086,
      "fcm_dpo/q_t": 0.3941470980644226,
      "grad_norm": 14.84249496459961,
      "learning_rate": 4.5364398801258394e-07,
      "logits/chosen": 0.4229778051376343,
      "logits/rejected": 0.37561944127082825,
      "logps/chosen": -70.1456298828125,
      "logps/ref_chosen": -52.32114028930664,
      "logps/ref_rejected": -68.3885726928711,
      "logps/rejected": -101.55931091308594,
      "loss": 1.1206,
      "margin_dpo/margin_mean": 15.346242904663086,
      "margin_dpo/margin_std": 26.30462646484375,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -15.705266952514648,
      "KL/mean": -24.041990280151367,
      "KL/rejected_KL_mean": -32.37870788574219,
      "KL/std": 21.698299407958984,
      "epoch": 0.2811791383219955,
      "fcm_dpo/beta": 0.0311187282204628,
      "fcm_dpo/delta": -0.1256175935268402,
      "fcm_dpo/margin": 16.673439025878906,
      "fcm_dpo/q_t": 0.3876885771751404,
      "grad_norm": 18.64262580871582,
      "learning_rate": 4.5287418106563354e-07,
      "logits/chosen": 0.3782210350036621,
      "logits/rejected": 0.33319205045700073,
      "logps/chosen": -83.12539672851562,
      "logps/ref_chosen": -67.42012786865234,
      "logps/ref_rejected": -82.50968933105469,
      "logps/rejected": -114.88839721679688,
      "loss": 1.0813,
      "margin_dpo/margin_mean": 16.673439025878906,
      "margin_dpo/margin_std": 26.293109893798828,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -20.900047302246094,
      "KL/mean": -28.237571716308594,
      "KL/rejected_KL_mean": -35.575096130371094,
      "KL/std": 23.087440490722656,
      "epoch": 0.28269085411942557,
      "fcm_dpo/beta": 0.030645418912172318,
      "fcm_dpo/delta": -0.05242550000548363,
      "fcm_dpo/margin": 14.67505168914795,
      "fcm_dpo/q_t": 0.4052157998085022,
      "grad_norm": 17.817066192626953,
      "learning_rate": 4.520986992917297e-07,
      "logits/chosen": 0.40862882137298584,
      "logits/rejected": 0.35285601019859314,
      "logps/chosen": -96.425537109375,
      "logps/ref_chosen": -75.52549743652344,
      "logps/ref_rejected": -94.76289367675781,
      "logps/rejected": -130.33798217773438,
      "loss": 1.141,
      "margin_dpo/margin_mean": 14.675050735473633,
      "margin_dpo/margin_std": 27.1290283203125,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -18.666240692138672,
      "KL/mean": -27.357372283935547,
      "KL/rejected_KL_mean": -36.048500061035156,
      "KL/std": 23.889400482177734,
      "epoch": 0.2842025699168556,
      "fcm_dpo/beta": 0.030069969594478607,
      "fcm_dpo/delta": -0.1294178068637848,
      "fcm_dpo/margin": 17.38225555419922,
      "fcm_dpo/q_t": 0.38837963342666626,
      "grad_norm": 14.66856575012207,
      "learning_rate": 4.5131756438276466e-07,
      "logits/chosen": 0.4121420383453369,
      "logits/rejected": 0.36901307106018066,
      "logps/chosen": -90.1895751953125,
      "logps/ref_chosen": -71.52333068847656,
      "logps/ref_rejected": -78.29949951171875,
      "logps/rejected": -114.3479995727539,
      "loss": 1.0634,
      "margin_dpo/margin_mean": 17.38225555419922,
      "margin_dpo/margin_std": 25.864315032958984,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -18.839712142944336,
      "KL/mean": -26.519317626953125,
      "KL/rejected_KL_mean": -34.19892501831055,
      "KL/std": 23.881752014160156,
      "epoch": 0.2857142857142857,
      "fcm_dpo/beta": 0.029213856905698776,
      "fcm_dpo/delta": -0.10978260636329651,
      "fcm_dpo/margin": 15.359216690063477,
      "fcm_dpo/q_t": 0.40099185705184937,
      "grad_norm": 16.19655418395996,
      "learning_rate": 4.5053079818876096e-07,
      "logits/chosen": 0.4219725728034973,
      "logits/rejected": 0.43759363889694214,
      "logps/chosen": -91.01597595214844,
      "logps/ref_chosen": -72.17626953125,
      "logps/ref_rejected": -75.26313781738281,
      "logps/rejected": -109.4620590209961,
      "loss": 1.11,
      "margin_dpo/margin_mean": 15.35921859741211,
      "margin_dpo/margin_std": 24.676734924316406,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -18.256019592285156,
      "KL/mean": -29.198854446411133,
      "KL/rejected_KL_mean": -40.141685485839844,
      "KL/std": 24.935558319091797,
      "epoch": 0.2872260015117158,
      "fcm_dpo/beta": 0.0281969103962183,
      "fcm_dpo/delta": -0.231668621301651,
      "fcm_dpo/margin": 21.885665893554688,
      "fcm_dpo/q_t": 0.36639824509620667,
      "grad_norm": 20.939247131347656,
      "learning_rate": 4.4973842271726024e-07,
      "logits/chosen": 0.5043848752975464,
      "logits/rejected": 0.3508925437927246,
      "logps/chosen": -72.88029479980469,
      "logps/ref_chosen": -54.624271392822266,
      "logps/ref_rejected": -101.47068786621094,
      "logps/rejected": -141.61236572265625,
      "loss": 0.9898,
      "margin_dpo/margin_mean": 21.885665893554688,
      "margin_dpo/margin_std": 27.162105560302734,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -26.937339782714844,
      "KL/mean": -35.45838165283203,
      "KL/rejected_KL_mean": -43.97942352294922,
      "KL/std": 26.476070404052734,
      "epoch": 0.2887377173091459,
      "fcm_dpo/beta": 0.027564112097024918,
      "fcm_dpo/delta": -0.07326999306678772,
      "fcm_dpo/margin": 17.042081832885742,
      "fcm_dpo/q_t": 0.3958955407142639,
      "grad_norm": 17.457185745239258,
      "learning_rate": 4.48940460132708e-07,
      "logits/chosen": 0.46865350008010864,
      "logits/rejected": 0.43871521949768066,
      "logps/chosen": -99.86985778808594,
      "logps/ref_chosen": -72.93251037597656,
      "logps/ref_rejected": -89.95103454589844,
      "logps/rejected": -133.93045043945312,
      "loss": 1.1102,
      "margin_dpo/margin_mean": 17.042081832885742,
      "margin_dpo/margin_std": 28.385475158691406,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -30.011014938354492,
      "KL/mean": -35.43492889404297,
      "KL/rejected_KL_mean": -40.858848571777344,
      "KL/std": 27.704442977905273,
      "epoch": 0.29024943310657597,
      "fcm_dpo/beta": 0.02748541347682476,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.847834587097168,
      "fcm_dpo/q_t": 0.4331100285053253,
      "grad_norm": 16.07075309753418,
      "learning_rate": 4.481369327558329e-07,
      "logits/chosen": 0.5033490657806396,
      "logits/rejected": 0.47497987747192383,
      "logps/chosen": -84.01213073730469,
      "logps/ref_chosen": -54.001121520996094,
      "logps/ref_rejected": -63.531551361083984,
      "logps/rejected": -104.39039611816406,
      "loss": 1.2127,
      "margin_dpo/margin_mean": 10.847833633422852,
      "margin_dpo/margin_std": 24.105567932128906,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -24.1334285736084,
      "KL/mean": -34.30812454223633,
      "KL/rejected_KL_mean": -44.48281478881836,
      "KL/std": 25.59041404724121,
      "epoch": 0.29176114890400606,
      "fcm_dpo/beta": 0.026737425476312637,
      "fcm_dpo/delta": -0.15358255803585052,
      "fcm_dpo/margin": 20.349384307861328,
      "fcm_dpo/q_t": 0.37987715005874634,
      "grad_norm": 13.550483703613281,
      "learning_rate": 4.47327863063023e-07,
      "logits/chosen": 0.44205987453460693,
      "logits/rejected": 0.4210042357444763,
      "logps/chosen": -80.88270568847656,
      "logps/ref_chosen": -56.74927520751953,
      "logps/ref_rejected": -58.80629348754883,
      "logps/rejected": -103.28910827636719,
      "loss": 1.0276,
      "margin_dpo/margin_mean": 20.349384307861328,
      "margin_dpo/margin_std": 26.462234497070312,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -26.698780059814453,
      "KL/mean": -35.048187255859375,
      "KL/rejected_KL_mean": -43.39760208129883,
      "KL/std": 27.187034606933594,
      "epoch": 0.29327286470143615,
      "fcm_dpo/beta": 0.026069460436701775,
      "fcm_dpo/delta": -0.11087217181921005,
      "fcm_dpo/margin": 16.698816299438477,
      "fcm_dpo/q_t": 0.4047401547431946,
      "grad_norm": 18.267810821533203,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 0.5409113168716431,
      "logits/rejected": 0.5088620185852051,
      "logps/chosen": -83.34822082519531,
      "logps/ref_chosen": -56.64944076538086,
      "logps/ref_rejected": -69.98954772949219,
      "logps/rejected": -113.38714599609375,
      "loss": 1.1575,
      "margin_dpo/margin_mean": 16.69881820678711,
      "margin_dpo/margin_std": 31.43659019470215,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -31.998184204101562,
      "KL/mean": -42.90058898925781,
      "KL/rejected_KL_mean": -53.80299758911133,
      "KL/std": 29.968097686767578,
      "epoch": 0.2947845804988662,
      "fcm_dpo/beta": 0.025347769260406494,
      "fcm_dpo/delta": -0.16259239614009857,
      "fcm_dpo/margin": 21.80481719970703,
      "fcm_dpo/q_t": 0.3793768882751465,
      "grad_norm": 17.709516525268555,
      "learning_rate": 4.4569318740967043e-07,
      "logits/chosen": 0.3753807246685028,
      "logits/rejected": 0.3845909535884857,
      "logps/chosen": -102.407958984375,
      "logps/ref_chosen": -70.40977478027344,
      "logps/ref_rejected": -74.39448547363281,
      "logps/rejected": -128.19747924804688,
      "loss": 1.0462,
      "margin_dpo/margin_mean": 21.80481719970703,
      "margin_dpo/margin_std": 30.805763244628906,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -31.96177101135254,
      "KL/mean": -40.567466735839844,
      "KL/rejected_KL_mean": -49.173160552978516,
      "KL/std": 30.731754302978516,
      "epoch": 0.2962962962962963,
      "fcm_dpo/beta": 0.02507277950644493,
      "fcm_dpo/delta": -0.06484292447566986,
      "fcm_dpo/margin": 17.211387634277344,
      "fcm_dpo/q_t": 0.40247493982315063,
      "grad_norm": 18.032867431640625,
      "learning_rate": 4.448676271745197e-07,
      "logits/chosen": 0.4948139786720276,
      "logits/rejected": 0.4499012231826782,
      "logps/chosen": -91.18934631347656,
      "logps/ref_chosen": -59.227577209472656,
      "logps/ref_rejected": -83.54757690429688,
      "logps/rejected": -132.72073364257812,
      "loss": 1.1033,
      "margin_dpo/margin_mean": 17.211387634277344,
      "margin_dpo/margin_std": 26.30760955810547,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -28.26848793029785,
      "KL/mean": -38.98504638671875,
      "KL/rejected_KL_mean": -49.701595306396484,
      "KL/std": 29.063312530517578,
      "epoch": 0.29780801209372637,
      "fcm_dpo/beta": 0.024582477286458015,
      "fcm_dpo/delta": -0.13332447409629822,
      "fcm_dpo/margin": 21.433109283447266,
      "fcm_dpo/q_t": 0.38597390055656433,
      "grad_norm": 17.628982543945312,
      "learning_rate": 4.440366160729392e-07,
      "logits/chosen": 0.6032913327217102,
      "logits/rejected": 0.5473294258117676,
      "logps/chosen": -79.79762268066406,
      "logps/ref_chosen": -51.52912902832031,
      "logps/ref_rejected": -73.70631408691406,
      "logps/rejected": -123.40790557861328,
      "loss": 1.1096,
      "margin_dpo/margin_mean": 21.433109283447266,
      "margin_dpo/margin_std": 36.053382873535156,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -31.24224090576172,
      "KL/mean": -42.64985656738281,
      "KL/rejected_KL_mean": -54.0574836730957,
      "KL/std": 29.603408813476562,
      "epoch": 0.29931972789115646,
      "fcm_dpo/beta": 0.023822344839572906,
      "fcm_dpo/delta": -0.15149368345737457,
      "fcm_dpo/margin": 22.815235137939453,
      "fcm_dpo/q_t": 0.3782616853713989,
      "grad_norm": 16.272396087646484,
      "learning_rate": 4.432001773500957e-07,
      "logits/chosen": 0.572478711605072,
      "logits/rejected": 0.5288437604904175,
      "logps/chosen": -91.02491760253906,
      "logps/ref_chosen": -59.78268051147461,
      "logps/ref_rejected": -72.24533081054688,
      "logps/rejected": -126.30281066894531,
      "loss": 1.0302,
      "margin_dpo/margin_mean": 22.815235137939453,
      "margin_dpo/margin_std": 29.531604766845703,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -37.891090393066406,
      "KL/mean": -46.90549087524414,
      "KL/rejected_KL_mean": -55.91989517211914,
      "KL/std": 30.329345703125,
      "epoch": 0.30083144368858655,
      "fcm_dpo/beta": 0.023282885551452637,
      "fcm_dpo/delta": -0.10651887953281403,
      "fcm_dpo/margin": 18.028810501098633,
      "fcm_dpo/q_t": 0.40665411949157715,
      "grad_norm": 15.3820219039917,
      "learning_rate": 4.4235833440297856e-07,
      "logits/chosen": 0.49368178844451904,
      "logits/rejected": 0.40359988808631897,
      "logps/chosen": -94.27786254882812,
      "logps/ref_chosen": -56.38677215576172,
      "logps/ref_rejected": -74.56779479980469,
      "logps/rejected": -130.48770141601562,
      "loss": 1.1625,
      "margin_dpo/margin_mean": 18.028810501098633,
      "margin_dpo/margin_std": 33.92662048339844,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -30.862167358398438,
      "KL/mean": -43.695106506347656,
      "KL/rejected_KL_mean": -56.528053283691406,
      "KL/std": 31.77981185913086,
      "epoch": 0.30234315948601664,
      "fcm_dpo/beta": 0.0223165862262249,
      "fcm_dpo/delta": -0.184445321559906,
      "fcm_dpo/margin": 25.66588592529297,
      "fcm_dpo/q_t": 0.37537479400634766,
      "grad_norm": 14.177912712097168,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": 0.5794877409934998,
      "logits/rejected": 0.49834418296813965,
      "logps/chosen": -88.68649291992188,
      "logps/ref_chosen": -57.82432556152344,
      "logps/ref_rejected": -89.28246307373047,
      "logps/rejected": -145.81051635742188,
      "loss": 1.0502,
      "margin_dpo/margin_mean": 25.66588592529297,
      "margin_dpo/margin_std": 37.56428527832031,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -42.7253303527832,
      "KL/mean": -52.768768310546875,
      "KL/rejected_KL_mean": -62.81220626831055,
      "KL/std": 33.54126739501953,
      "epoch": 0.30385487528344673,
      "fcm_dpo/beta": 0.02184602990746498,
      "fcm_dpo/delta": -0.0805087760090828,
      "fcm_dpo/margin": 20.086875915527344,
      "fcm_dpo/q_t": 0.403793066740036,
      "grad_norm": 16.322202682495117,
      "learning_rate": 4.4065853017905953e-07,
      "logits/chosen": 0.5901821851730347,
      "logits/rejected": 0.5399671196937561,
      "logps/chosen": -101.72509002685547,
      "logps/ref_chosen": -58.999759674072266,
      "logps/ref_rejected": -84.67575073242188,
      "logps/rejected": -147.48794555664062,
      "loss": 1.1266,
      "margin_dpo/margin_mean": 20.086875915527344,
      "margin_dpo/margin_std": 34.55057907104492,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -36.28567886352539,
      "KL/mean": -49.248268127441406,
      "KL/rejected_KL_mean": -62.21084976196289,
      "KL/std": 31.714256286621094,
      "epoch": 0.30536659108087677,
      "fcm_dpo/beta": 0.021372711285948753,
      "fcm_dpo/delta": -0.16286848485469818,
      "fcm_dpo/margin": 25.925172805786133,
      "fcm_dpo/q_t": 0.3748985528945923,
      "grad_norm": 14.946732521057129,
      "learning_rate": 4.3980061644943575e-07,
      "logits/chosen": 0.4884384274482727,
      "logits/rejected": 0.41029107570648193,
      "logps/chosen": -83.94632720947266,
      "logps/ref_chosen": -47.660648345947266,
      "logps/ref_rejected": -73.63249969482422,
      "logps/rejected": -135.84335327148438,
      "loss": 1.0348,
      "margin_dpo/margin_mean": 25.925174713134766,
      "margin_dpo/margin_std": 34.66651916503906,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -44.81160354614258,
      "KL/mean": -56.35142517089844,
      "KL/rejected_KL_mean": -67.89125061035156,
      "KL/std": 32.933311462402344,
      "epoch": 0.30687830687830686,
      "fcm_dpo/beta": 0.02090853452682495,
      "fcm_dpo/delta": -0.08666198700666428,
      "fcm_dpo/margin": 23.07965660095215,
      "fcm_dpo/q_t": 0.3943900465965271,
      "grad_norm": 17.459609985351562,
      "learning_rate": 4.3893739358856455e-07,
      "logits/chosen": 0.5723869800567627,
      "logits/rejected": 0.48567742109298706,
      "logps/chosen": -107.13713836669922,
      "logps/ref_chosen": -62.32553482055664,
      "logps/ref_rejected": -99.37226104736328,
      "logps/rejected": -167.26351928710938,
      "loss": 1.0892,
      "margin_dpo/margin_mean": 23.07965660095215,
      "margin_dpo/margin_std": 36.10200881958008,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -40.54402160644531,
      "KL/mean": -52.845306396484375,
      "KL/rejected_KL_mean": -65.14659118652344,
      "KL/std": 33.49970626831055,
      "epoch": 0.30839002267573695,
      "fcm_dpo/beta": 0.02015521563589573,
      "fcm_dpo/delta": -0.15325036644935608,
      "fcm_dpo/margin": 24.60256576538086,
      "fcm_dpo/q_t": 0.3900430500507355,
      "grad_norm": 16.921817779541016,
      "learning_rate": 4.380688857426449e-07,
      "logits/chosen": 0.5573503971099854,
      "logits/rejected": 0.48204463720321655,
      "logps/chosen": -91.17333984375,
      "logps/ref_chosen": -50.62931823730469,
      "logps/ref_rejected": -66.60475158691406,
      "logps/rejected": -131.7513427734375,
      "loss": 1.0682,
      "margin_dpo/margin_mean": 24.60256576538086,
      "margin_dpo/margin_std": 34.54716491699219,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -47.66603088378906,
      "KL/mean": -59.343788146972656,
      "KL/rejected_KL_mean": -71.02154541015625,
      "KL/std": 35.73222732543945,
      "epoch": 0.30990173847316704,
      "fcm_dpo/beta": 0.0199609212577343,
      "fcm_dpo/delta": -0.048433154821395874,
      "fcm_dpo/margin": 23.355518341064453,
      "fcm_dpo/q_t": 0.39880335330963135,
      "grad_norm": 18.173583984375,
      "learning_rate": 4.3719511720570814e-07,
      "logits/chosen": 0.5487678050994873,
      "logits/rejected": 0.4762420058250427,
      "logps/chosen": -118.02220916748047,
      "logps/ref_chosen": -70.3561782836914,
      "logps/ref_rejected": -93.39848327636719,
      "logps/rejected": -164.42002868652344,
      "loss": 1.1161,
      "margin_dpo/margin_mean": 23.355518341064453,
      "margin_dpo/margin_std": 39.755157470703125,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -49.99560546875,
      "KL/mean": -58.32989501953125,
      "KL/rejected_KL_mean": -66.6641845703125,
      "KL/std": 34.640377044677734,
      "epoch": 0.31141345427059713,
      "fcm_dpo/beta": 0.0199609212577343,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.668582916259766,
      "fcm_dpo/q_t": 0.42973265051841736,
      "grad_norm": 16.76526641845703,
      "learning_rate": 4.363161124189387e-07,
      "logits/chosen": 0.6296049356460571,
      "logits/rejected": 0.6118504405021667,
      "logps/chosen": -117.64108276367188,
      "logps/ref_chosen": -67.64547729492188,
      "logps/ref_rejected": -79.89584350585938,
      "logps/rejected": -146.56002807617188,
      "loss": 1.2418,
      "margin_dpo/margin_mean": 16.6685848236084,
      "margin_dpo/margin_std": 41.699256896972656,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -58.37126922607422,
      "KL/mean": -69.7347412109375,
      "KL/rejected_KL_mean": -81.09820556640625,
      "KL/std": 35.99120330810547,
      "epoch": 0.3129251700680272,
      "fcm_dpo/beta": 0.0199609212577343,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.726932525634766,
      "fcm_dpo/q_t": 0.4033956825733185,
      "grad_norm": 14.681884765625,
      "learning_rate": 4.3543189596998986e-07,
      "logits/chosen": 0.5348644852638245,
      "logits/rejected": 0.46388787031173706,
      "logps/chosen": -126.03546142578125,
      "logps/ref_chosen": -67.66419219970703,
      "logps/ref_rejected": -85.10249328613281,
      "logps/rejected": -166.20068359375,
      "loss": 1.1111,
      "margin_dpo/margin_mean": 22.726932525634766,
      "margin_dpo/margin_std": 37.863197326660156,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -49.66230773925781,
      "KL/mean": -55.65148162841797,
      "KL/rejected_KL_mean": -61.64065170288086,
      "KL/std": 34.22406005859375,
      "epoch": 0.3144368858654573,
      "fcm_dpo/beta": 0.0199609212577343,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.978349685668945,
      "fcm_dpo/q_t": 0.4482603669166565,
      "grad_norm": 21.4254207611084,
      "learning_rate": 4.3454249259229664e-07,
      "logits/chosen": 0.576629638671875,
      "logits/rejected": 0.5478168725967407,
      "logps/chosen": -107.3940200805664,
      "logps/ref_chosen": -57.731712341308594,
      "logps/ref_rejected": -74.19276428222656,
      "logps/rejected": -135.8334197998047,
      "loss": 1.3075,
      "margin_dpo/margin_mean": 11.978348731994629,
      "margin_dpo/margin_std": 39.34952926635742,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -49.522850036621094,
      "KL/mean": -64.13494873046875,
      "KL/rejected_KL_mean": -78.7470703125,
      "KL/std": 38.795494079589844,
      "epoch": 0.31594860166288735,
      "fcm_dpo/beta": 0.019417954608798027,
      "fcm_dpo/delta": -0.1777927577495575,
      "fcm_dpo/margin": 29.224210739135742,
      "fcm_dpo/q_t": 0.37642014026641846,
      "grad_norm": 16.401376724243164,
      "learning_rate": 4.336479271643833e-07,
      "logits/chosen": 0.4981778860092163,
      "logits/rejected": 0.44314074516296387,
      "logps/chosen": -118.07292938232422,
      "logps/ref_chosen": -68.55007934570312,
      "logps/ref_rejected": -87.90541076660156,
      "logps/rejected": -166.65248107910156,
      "loss": 1.0706,
      "margin_dpo/margin_mean": 29.224210739135742,
      "margin_dpo/margin_std": 45.77735137939453,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -49.30180358886719,
      "KL/mean": -63.557167053222656,
      "KL/rejected_KL_mean": -77.81253051757812,
      "KL/std": 40.000423431396484,
      "epoch": 0.31746031746031744,
      "fcm_dpo/beta": 0.01899714395403862,
      "fcm_dpo/delta": -0.14027619361877441,
      "fcm_dpo/margin": 28.510725021362305,
      "fcm_dpo/q_t": 0.38312628865242004,
      "grad_norm": 16.62250518798828,
      "learning_rate": 4.327482247091679e-07,
      "logits/chosen": 0.5970274209976196,
      "logits/rejected": 0.4921113848686218,
      "logps/chosen": -106.57007598876953,
      "logps/ref_chosen": -57.268272399902344,
      "logps/ref_rejected": -85.72807312011719,
      "logps/rejected": -163.54061889648438,
      "loss": 1.0505,
      "margin_dpo/margin_mean": 28.510725021362305,
      "margin_dpo/margin_std": 40.20121765136719,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -45.32893371582031,
      "KL/mean": -59.812034606933594,
      "KL/rejected_KL_mean": -74.2951431274414,
      "KL/std": 36.853477478027344,
      "epoch": 0.31897203325774753,
      "fcm_dpo/beta": 0.018486086279153824,
      "fcm_dpo/delta": -0.13231848180294037,
      "fcm_dpo/margin": 28.96620750427246,
      "fcm_dpo/q_t": 0.38412922620773315,
      "grad_norm": 17.82309341430664,
      "learning_rate": 4.3184341039326217e-07,
      "logits/chosen": 0.6039122343063354,
      "logits/rejected": 0.4997963309288025,
      "logps/chosen": -98.96964263916016,
      "logps/ref_chosen": -53.640708923339844,
      "logps/ref_rejected": -93.0387954711914,
      "logps/rejected": -167.3339385986328,
      "loss": 1.0304,
      "margin_dpo/margin_mean": 28.966205596923828,
      "margin_dpo/margin_std": 37.75336456298828,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -50.42332458496094,
      "KL/mean": -64.77632141113281,
      "KL/rejected_KL_mean": -79.12932586669922,
      "KL/std": 39.548606872558594,
      "epoch": 0.3204837490551776,
      "fcm_dpo/beta": 0.01791585050523281,
      "fcm_dpo/delta": -0.12050309777259827,
      "fcm_dpo/margin": 28.705997467041016,
      "fcm_dpo/q_t": 0.38503497838974,
      "grad_norm": 12.132513999938965,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 0.6209944486618042,
      "logits/rejected": 0.5416966676712036,
      "logps/chosen": -107.79006958007812,
      "logps/ref_chosen": -57.36674499511719,
      "logps/ref_rejected": -79.89643096923828,
      "logps/rejected": -159.0257568359375,
      "loss": 1.0681,
      "margin_dpo/margin_mean": 28.705997467041016,
      "margin_dpo/margin_std": 42.09522247314453,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -40.895442962646484,
      "KL/mean": -57.22308349609375,
      "KL/rejected_KL_mean": -73.55073547363281,
      "KL/std": 40.150550842285156,
      "epoch": 0.3219954648526077,
      "fcm_dpo/beta": 0.017257830128073692,
      "fcm_dpo/delta": -0.17485719919204712,
      "fcm_dpo/margin": 32.65528106689453,
      "fcm_dpo/q_t": 0.3760501742362976,
      "grad_norm": 13.000662803649902,
      "learning_rate": 4.3001854756006724e-07,
      "logits/chosen": 0.5805448293685913,
      "logits/rejected": 0.5558933019638062,
      "logps/chosen": -106.11656188964844,
      "logps/ref_chosen": -65.22111511230469,
      "logps/ref_rejected": -80.1810302734375,
      "logps/rejected": -153.73175048828125,
      "loss": 1.0314,
      "margin_dpo/margin_mean": 32.65528106689453,
      "margin_dpo/margin_std": 44.30155944824219,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -47.93132781982422,
      "KL/mean": -63.84461212158203,
      "KL/rejected_KL_mean": -79.75790405273438,
      "KL/std": 40.14862823486328,
      "epoch": 0.3235071806500378,
      "fcm_dpo/beta": 0.016959507018327713,
      "fcm_dpo/delta": -0.13894835114479065,
      "fcm_dpo/margin": 31.826576232910156,
      "fcm_dpo/q_t": 0.3835788369178772,
      "grad_norm": 16.028963088989258,
      "learning_rate": 4.290985500881143e-07,
      "logits/chosen": 0.4599738121032715,
      "logits/rejected": 0.44451236724853516,
      "logps/chosen": -109.2236557006836,
      "logps/ref_chosen": -61.292327880859375,
      "logps/ref_rejected": -67.69841003417969,
      "logps/rejected": -147.45631408691406,
      "loss": 1.0611,
      "margin_dpo/margin_mean": 31.82657814025879,
      "margin_dpo/margin_std": 46.38356018066406,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -53.18816375732422,
      "KL/mean": -69.61984252929688,
      "KL/rejected_KL_mean": -86.05149841308594,
      "KL/std": 41.73292541503906,
      "epoch": 0.3250188964474679,
      "fcm_dpo/beta": 0.016454260796308517,
      "fcm_dpo/delta": -0.16386979818344116,
      "fcm_dpo/margin": 32.86333465576172,
      "fcm_dpo/q_t": 0.38295409083366394,
      "grad_norm": 14.382888793945312,
      "learning_rate": 4.281735428447157e-07,
      "logits/chosen": 0.4620933532714844,
      "logits/rejected": 0.34720370173454285,
      "logps/chosen": -117.05729675292969,
      "logps/ref_chosen": -63.869136810302734,
      "logps/ref_rejected": -98.7657241821289,
      "logps/rejected": -184.81723022460938,
      "loss": 1.0662,
      "margin_dpo/margin_mean": 32.863338470458984,
      "margin_dpo/margin_std": 48.01820373535156,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -44.4687385559082,
      "KL/mean": -62.103668212890625,
      "KL/rejected_KL_mean": -79.73858642578125,
      "KL/std": 40.250755310058594,
      "epoch": 0.32653061224489793,
      "fcm_dpo/beta": 0.015767354518175125,
      "fcm_dpo/delta": -0.16560883820056915,
      "fcm_dpo/margin": 35.269840240478516,
      "fcm_dpo/q_t": 0.3732219636440277,
      "grad_norm": 25.336891174316406,
      "learning_rate": 4.2724355170431247e-07,
      "logits/chosen": 0.6101344227790833,
      "logits/rejected": 0.5182064771652222,
      "logps/chosen": -112.29369354248047,
      "logps/ref_chosen": -67.824951171875,
      "logps/ref_rejected": -96.40231323242188,
      "logps/rejected": -176.14089965820312,
      "loss": 0.9984,
      "margin_dpo/margin_mean": 35.26984405517578,
      "margin_dpo/margin_std": 40.447853088378906,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -48.50625228881836,
      "KL/mean": -66.75289916992188,
      "KL/rejected_KL_mean": -84.99954986572266,
      "KL/std": 42.09430694580078,
      "epoch": 0.328042328042328,
      "fcm_dpo/beta": 0.015221597626805305,
      "fcm_dpo/delta": -0.16550882160663605,
      "fcm_dpo/margin": 36.49330139160156,
      "fcm_dpo/q_t": 0.37577375769615173,
      "grad_norm": 12.231183052062988,
      "learning_rate": 4.26308602680756e-07,
      "logits/chosen": 0.5095263719558716,
      "logits/rejected": 0.40860888361930847,
      "logps/chosen": -109.01124572753906,
      "logps/ref_chosen": -60.5049934387207,
      "logps/ref_rejected": -84.26618194580078,
      "logps/rejected": -169.26573181152344,
      "loss": 1.0063,
      "margin_dpo/margin_mean": 36.49330139160156,
      "margin_dpo/margin_std": 43.78234100341797,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -53.025917053222656,
      "KL/mean": -63.77705001831055,
      "KL/rejected_KL_mean": -74.52819061279297,
      "KL/std": 41.17570495605469,
      "epoch": 0.3295540438397581,
      "fcm_dpo/beta": 0.015110185369849205,
      "fcm_dpo/delta": -0.024738004431128502,
      "fcm_dpo/margin": 21.50227165222168,
      "fcm_dpo/q_t": 0.42662978172302246,
      "grad_norm": 13.907445907592773,
      "learning_rate": 4.253687219265803e-07,
      "logits/chosen": 0.4400632381439209,
      "logits/rejected": 0.4424203336238861,
      "logps/chosen": -123.62023162841797,
      "logps/ref_chosen": -70.59431457519531,
      "logps/ref_rejected": -73.89038848876953,
      "logps/rejected": -148.4185791015625,
      "loss": 1.2201,
      "margin_dpo/margin_mean": 21.502273559570312,
      "margin_dpo/margin_std": 48.850921630859375,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -49.29433822631836,
      "KL/mean": -61.04737854003906,
      "KL/rejected_KL_mean": -72.80043029785156,
      "KL/std": 39.78154754638672,
      "epoch": 0.3310657596371882,
      "fcm_dpo/beta": 0.015072805806994438,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.506093978881836,
      "fcm_dpo/q_t": 0.4195045232772827,
      "grad_norm": 12.386738777160645,
      "learning_rate": 4.2442393573227043e-07,
      "logits/chosen": 0.509136974811554,
      "logits/rejected": 0.46663516759872437,
      "logps/chosen": -109.7852783203125,
      "logps/ref_chosen": -60.490943908691406,
      "logps/ref_rejected": -75.85001373291016,
      "logps/rejected": -148.6504364013672,
      "loss": 1.1503,
      "margin_dpo/margin_mean": 23.506093978881836,
      "margin_dpo/margin_std": 40.30815887451172,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -41.13189697265625,
      "KL/mean": -55.28445053100586,
      "KL/rejected_KL_mean": -69.43699645996094,
      "KL/std": 41.99296951293945,
      "epoch": 0.3325774754346183,
      "fcm_dpo/beta": 0.014830692671239376,
      "fcm_dpo/delta": -0.08096643537282944,
      "fcm_dpo/margin": 28.305110931396484,
      "fcm_dpo/q_t": 0.4073430895805359,
      "grad_norm": 13.636482238769531,
      "learning_rate": 4.234742705255272e-07,
      "logits/chosen": 0.6164438128471375,
      "logits/rejected": 0.5471941232681274,
      "logps/chosen": -86.14529418945312,
      "logps/ref_chosen": -45.013397216796875,
      "logps/ref_rejected": -70.49369812011719,
      "logps/rejected": -139.93069458007812,
      "loss": 1.134,
      "margin_dpo/margin_mean": 28.305110931396484,
      "margin_dpo/margin_std": 48.46826934814453,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -40.829010009765625,
      "KL/mean": -56.581878662109375,
      "KL/rejected_KL_mean": -72.33475494384766,
      "KL/std": 41.920936584472656,
      "epoch": 0.3340891912320484,
      "fcm_dpo/beta": 0.014756301417946815,
      "fcm_dpo/delta": -0.050413843244314194,
      "fcm_dpo/margin": 31.505746841430664,
      "fcm_dpo/q_t": 0.39809131622314453,
      "grad_norm": 15.061563491821289,
      "learning_rate": 4.22519752870528e-07,
      "logits/chosen": 0.5553480386734009,
      "logits/rejected": 0.47446322441101074,
      "logps/chosen": -99.92485046386719,
      "logps/ref_chosen": -59.09584045410156,
      "logps/ref_rejected": -88.64388275146484,
      "logps/rejected": -160.9786376953125,
      "loss": 1.0837,
      "margin_dpo/margin_mean": 31.50574493408203,
      "margin_dpo/margin_std": 47.23353576660156,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -40.73999786376953,
      "KL/mean": -62.21807098388672,
      "KL/rejected_KL_mean": -83.6961441040039,
      "KL/std": 47.69040298461914,
      "epoch": 0.3356009070294785,
      "fcm_dpo/beta": 0.014200271107256413,
      "fcm_dpo/delta": -0.22367945313453674,
      "fcm_dpo/margin": 42.956146240234375,
      "fcm_dpo/q_t": 0.3654256761074066,
      "grad_norm": 12.030962944030762,
      "learning_rate": 4.2156040946718343e-07,
      "logits/chosen": 0.6517788171768188,
      "logits/rejected": 0.5524156093597412,
      "logps/chosen": -96.73768615722656,
      "logps/ref_chosen": -55.9976921081543,
      "logps/ref_rejected": -111.94727325439453,
      "logps/rejected": -195.64341735839844,
      "loss": 0.9826,
      "margin_dpo/margin_mean": 42.956146240234375,
      "margin_dpo/margin_std": 50.614097595214844,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -48.65684509277344,
      "KL/mean": -64.96463012695312,
      "KL/rejected_KL_mean": -81.27240753173828,
      "KL/std": 45.18286895751953,
      "epoch": 0.3371126228269085,
      "fcm_dpo/beta": 0.013911095447838306,
      "fcm_dpo/delta": -0.04596729949116707,
      "fcm_dpo/margin": 32.615562438964844,
      "fcm_dpo/q_t": 0.39882034063339233,
      "grad_norm": 15.105962753295898,
      "learning_rate": 4.2059626715039065e-07,
      "logits/chosen": 0.6108689308166504,
      "logits/rejected": 0.5520835518836975,
      "logps/chosen": -108.54826354980469,
      "logps/ref_chosen": -59.891422271728516,
      "logps/ref_rejected": -86.28954315185547,
      "logps/rejected": -167.56195068359375,
      "loss": 1.0778,
      "margin_dpo/margin_mean": 32.615562438964844,
      "margin_dpo/margin_std": 46.08930206298828,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -56.427490234375,
      "KL/mean": -65.35730743408203,
      "KL/rejected_KL_mean": -74.287109375,
      "KL/std": 45.022212982177734,
      "epoch": 0.3386243386243386,
      "fcm_dpo/beta": 0.013911095447838306,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 17.859630584716797,
      "fcm_dpo/q_t": 0.44438493251800537,
      "grad_norm": 16.007753372192383,
      "learning_rate": 4.1962735288928304e-07,
      "logits/chosen": 0.6295701265335083,
      "logits/rejected": 0.6061124205589294,
      "logps/chosen": -120.47212219238281,
      "logps/ref_chosen": -64.04463195800781,
      "logps/ref_rejected": -75.05450439453125,
      "logps/rejected": -149.34161376953125,
      "loss": 1.2579,
      "margin_dpo/margin_mean": 17.859630584716797,
      "margin_dpo/margin_std": 48.06422424316406,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -52.12030029296875,
      "KL/mean": -70.82891845703125,
      "KL/rejected_KL_mean": -89.53753662109375,
      "KL/std": 50.072288513183594,
      "epoch": 0.3401360544217687,
      "fcm_dpo/beta": 0.013590985909104347,
      "fcm_dpo/delta": -0.11639999598264694,
      "fcm_dpo/margin": 37.4172477722168,
      "fcm_dpo/q_t": 0.38528722524642944,
      "grad_norm": 12.09056282043457,
      "learning_rate": 4.186536937864752e-07,
      "logits/chosen": 0.626471996307373,
      "logits/rejected": 0.5087643265724182,
      "logps/chosen": -118.21612548828125,
      "logps/ref_chosen": -66.0958251953125,
      "logps/ref_rejected": -97.68675231933594,
      "logps/rejected": -187.2242889404297,
      "loss": 1.0305,
      "margin_dpo/margin_mean": 37.4172477722168,
      "margin_dpo/margin_std": 45.303226470947266,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -49.675743103027344,
      "KL/mean": -64.10902404785156,
      "KL/rejected_KL_mean": -78.54232025146484,
      "KL/std": 44.898719787597656,
      "epoch": 0.3416477702191988,
      "fcm_dpo/beta": 0.013537578284740448,
      "fcm_dpo/delta": -0.039451513439416885,
      "fcm_dpo/margin": 28.8665771484375,
      "fcm_dpo/q_t": 0.4117968678474426,
      "grad_norm": 13.084617614746094,
      "learning_rate": 4.176753170773052e-07,
      "logits/chosen": 0.6425325274467468,
      "logits/rejected": 0.5956849455833435,
      "logps/chosen": -101.09261322021484,
      "logps/ref_chosen": -51.4168701171875,
      "logps/ref_rejected": -66.30068969726562,
      "logps/rejected": -144.84300231933594,
      "loss": 1.1617,
      "margin_dpo/margin_mean": 28.8665771484375,
      "margin_dpo/margin_std": 54.40742874145508,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -53.59423828125,
      "KL/mean": -68.63531494140625,
      "KL/rejected_KL_mean": -83.6763916015625,
      "KL/std": 49.33631896972656,
      "epoch": 0.3431594860166289,
      "fcm_dpo/beta": 0.013484170660376549,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 30.082157135009766,
      "fcm_dpo/q_t": 0.41013020277023315,
      "grad_norm": 14.161370277404785,
      "learning_rate": 4.166922501290729e-07,
      "logits/chosen": 0.6694810390472412,
      "logits/rejected": 0.631727933883667,
      "logps/chosen": -111.58401489257812,
      "logps/ref_chosen": -57.989776611328125,
      "logps/ref_rejected": -75.05464172363281,
      "logps/rejected": -158.73104858398438,
      "loss": 1.1473,
      "margin_dpo/margin_mean": 30.082157135009766,
      "margin_dpo/margin_std": 54.82373809814453,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -56.00676727294922,
      "KL/mean": -73.64747619628906,
      "KL/rejected_KL_mean": -91.2881851196289,
      "KL/std": 46.921348571777344,
      "epoch": 0.34467120181405897,
      "fcm_dpo/beta": 0.013376947492361069,
      "fcm_dpo/delta": -0.08015699684619904,
      "fcm_dpo/margin": 35.28141403198242,
      "fcm_dpo/q_t": 0.39347511529922485,
      "grad_norm": 14.644017219543457,
      "learning_rate": 4.1570452044027405e-07,
      "logits/chosen": 0.6866810321807861,
      "logits/rejected": 0.6021173000335693,
      "logps/chosen": -111.56613159179688,
      "logps/ref_chosen": -55.55936813354492,
      "logps/ref_rejected": -77.02364349365234,
      "logps/rejected": -168.31182861328125,
      "loss": 1.0706,
      "margin_dpo/margin_mean": 35.28141784667969,
      "margin_dpo/margin_std": 49.03618621826172,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -48.973968505859375,
      "KL/mean": -63.88433074951172,
      "KL/rejected_KL_mean": -78.79469299316406,
      "KL/std": 46.169944763183594,
      "epoch": 0.34618291761148906,
      "fcm_dpo/beta": 0.013269724324345589,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 29.820728302001953,
      "fcm_dpo/q_t": 0.4127005338668823,
      "grad_norm": 18.285600662231445,
      "learning_rate": 4.147121556398312e-07,
      "logits/chosen": 0.7295674085617065,
      "logits/rejected": 0.6509650945663452,
      "logps/chosen": -99.76863098144531,
      "logps/ref_chosen": -50.79466247558594,
      "logps/ref_rejected": -78.4474105834961,
      "logps/rejected": -157.24209594726562,
      "loss": 1.1571,
      "margin_dpo/margin_mean": 29.820728302001953,
      "margin_dpo/margin_std": 56.27452087402344,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -52.722755432128906,
      "KL/mean": -69.46250915527344,
      "KL/rejected_KL_mean": -86.2022705078125,
      "KL/std": 50.38365173339844,
      "epoch": 0.3476946334089191,
      "fcm_dpo/beta": 0.013133594766259193,
      "fcm_dpo/delta": -0.10365402698516846,
      "fcm_dpo/margin": 33.47950744628906,
      "fcm_dpo/q_t": 0.4013306498527527,
      "grad_norm": 13.488200187683105,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 0.6259239912033081,
      "logits/rejected": 0.6276857852935791,
      "logps/chosen": -109.45198059082031,
      "logps/ref_chosen": -56.729225158691406,
      "logps/ref_rejected": -62.99180603027344,
      "logps/rejected": -149.19407653808594,
      "loss": 1.1021,
      "margin_dpo/margin_mean": 33.47951126098633,
      "margin_dpo/margin_std": 50.97336959838867,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -58.87866973876953,
      "KL/mean": -84.7044677734375,
      "KL/rejected_KL_mean": -110.5302734375,
      "KL/std": 49.37828063964844,
      "epoch": 0.3492063492063492,
      "fcm_dpo/beta": 0.012457750737667084,
      "fcm_dpo/delta": -0.2614459991455078,
      "fcm_dpo/margin": 51.651607513427734,
      "fcm_dpo/q_t": 0.35266393423080444,
      "grad_norm": 13.210762977600098,
      "learning_rate": 4.1271363186719835e-07,
      "logits/chosen": 0.5706331729888916,
      "logits/rejected": 0.5676780939102173,
      "logps/chosen": -131.47576904296875,
      "logps/ref_chosen": -72.59709930419922,
      "logps/ref_rejected": -86.2322998046875,
      "logps/rejected": -196.7625732421875,
      "loss": 0.9433,
      "margin_dpo/margin_mean": 51.651607513427734,
      "margin_dpo/margin_std": 52.80973815917969,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -60.621498107910156,
      "KL/mean": -77.64936828613281,
      "KL/rejected_KL_mean": -94.67724609375,
      "KL/std": 54.37424850463867,
      "epoch": 0.3507180650037793,
      "fcm_dpo/beta": 0.012335299514234066,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.05574035644531,
      "fcm_dpo/q_t": 0.40971839427948,
      "grad_norm": 13.251051902770996,
      "learning_rate": 4.1170752879801436e-07,
      "logits/chosen": 0.586661696434021,
      "logits/rejected": 0.5547990202903748,
      "logps/chosen": -128.74002075195312,
      "logps/ref_chosen": -68.1185302734375,
      "logps/ref_rejected": -83.79415893554688,
      "logps/rejected": -178.47140502929688,
      "loss": 1.1428,
      "margin_dpo/margin_mean": 34.05574417114258,
      "margin_dpo/margin_std": 61.997886657714844,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -69.0023193359375,
      "KL/mean": -81.63502502441406,
      "KL/rejected_KL_mean": -94.26773071289062,
      "KL/std": 49.81992721557617,
      "epoch": 0.35222978080120937,
      "fcm_dpo/beta": 0.012211372144520283,
      "fcm_dpo/delta": -0.05048698931932449,
      "fcm_dpo/margin": 25.26541519165039,
      "fcm_dpo/q_t": 0.43137213587760925,
      "grad_norm": 14.052996635437012,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": 0.6705083847045898,
      "logits/rejected": 0.6091481447219849,
      "logps/chosen": -124.07246398925781,
      "logps/ref_chosen": -55.070152282714844,
      "logps/ref_rejected": -66.61845397949219,
      "logps/rejected": -160.8861846923828,
      "loss": 1.2025,
      "margin_dpo/margin_mean": 25.26541519165039,
      "margin_dpo/margin_std": 51.56385803222656,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -61.680076599121094,
      "KL/mean": -74.02011108398438,
      "KL/rejected_KL_mean": -86.36014556884766,
      "KL/std": 48.443763732910156,
      "epoch": 0.35374149659863946,
      "fcm_dpo/beta": 0.012122506275773048,
      "fcm_dpo/delta": -0.07330780476331711,
      "fcm_dpo/margin": 24.68006134033203,
      "fcm_dpo/q_t": 0.4330037236213684,
      "grad_norm": 19.079448699951172,
      "learning_rate": 4.09681781007452e-07,
      "logits/chosen": 0.581956148147583,
      "logits/rejected": 0.5792969465255737,
      "logps/chosen": -117.60597229003906,
      "logps/ref_chosen": -55.92589569091797,
      "logps/ref_rejected": -51.11608123779297,
      "logps/rejected": -137.47622680664062,
      "loss": 1.2224,
      "margin_dpo/margin_mean": 24.680063247680664,
      "margin_dpo/margin_std": 52.967437744140625,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -56.598243713378906,
      "KL/mean": -78.36715698242188,
      "KL/rejected_KL_mean": -100.13606262207031,
      "KL/std": 53.2874641418457,
      "epoch": 0.35525321239606955,
      "fcm_dpo/beta": 0.011834532022476196,
      "fcm_dpo/delta": -0.12138506025075912,
      "fcm_dpo/margin": 43.537818908691406,
      "fcm_dpo/q_t": 0.38227230310440063,
      "grad_norm": 15.617653846740723,
      "learning_rate": 4.08662192950594e-07,
      "logits/chosen": 0.6357525587081909,
      "logits/rejected": 0.6238740682601929,
      "logps/chosen": -121.13796997070312,
      "logps/ref_chosen": -64.53972625732422,
      "logps/ref_rejected": -77.69151306152344,
      "logps/rejected": -177.82757568359375,
      "loss": 1.0189,
      "margin_dpo/margin_mean": 43.537818908691406,
      "margin_dpo/margin_std": 50.59941864013672,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -81.9928970336914,
      "KL/mean": -98.34184265136719,
      "KL/rejected_KL_mean": -114.69078063964844,
      "KL/std": 51.35175323486328,
      "epoch": 0.35676492819349964,
      "fcm_dpo/beta": 0.01174501795321703,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 32.69788360595703,
      "fcm_dpo/q_t": 0.41291993856430054,
      "grad_norm": 12.297943115234375,
      "learning_rate": 4.076381667711306e-07,
      "logits/chosen": 0.6182401180267334,
      "logits/rejected": 0.60513836145401,
      "logps/chosen": -153.14764404296875,
      "logps/ref_chosen": -71.15473937988281,
      "logps/ref_rejected": -84.88541412353516,
      "logps/rejected": -199.57620239257812,
      "loss": 1.1524,
      "margin_dpo/margin_mean": 32.69788360595703,
      "margin_dpo/margin_std": 58.897560119628906,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -69.26779174804688,
      "KL/mean": -89.37890625,
      "KL/rejected_KL_mean": -109.49002075195312,
      "KL/std": 51.652854919433594,
      "epoch": 0.35827664399092973,
      "fcm_dpo/beta": 0.011675616726279259,
      "fcm_dpo/delta": -0.05944126471877098,
      "fcm_dpo/margin": 40.22222137451172,
      "fcm_dpo/q_t": 0.39383262395858765,
      "grad_norm": 16.05605697631836,
      "learning_rate": 4.066097311132753e-07,
      "logits/chosen": 0.6465665102005005,
      "logits/rejected": 0.6383150815963745,
      "logps/chosen": -145.40980529785156,
      "logps/ref_chosen": -76.14201354980469,
      "logps/ref_rejected": -80.88479614257812,
      "logps/rejected": -190.37481689453125,
      "loss": 1.0856,
      "margin_dpo/margin_mean": 40.22222137451172,
      "margin_dpo/margin_std": 59.51573181152344,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -62.64219665527344,
      "KL/mean": -81.81443786621094,
      "KL/rejected_KL_mean": -100.98668670654297,
      "KL/std": 53.077049255371094,
      "epoch": 0.35978835978835977,
      "fcm_dpo/beta": 0.011450215242803097,
      "fcm_dpo/delta": -0.0676615908741951,
      "fcm_dpo/margin": 38.34449005126953,
      "fcm_dpo/q_t": 0.4004179835319519,
      "grad_norm": 16.575794219970703,
      "learning_rate": 4.0557691474458414e-07,
      "logits/chosen": 0.6054675579071045,
      "logits/rejected": 0.5951860547065735,
      "logps/chosen": -131.52703857421875,
      "logps/ref_chosen": -68.88484954833984,
      "logps/ref_rejected": -75.8946304321289,
      "logps/rejected": -176.88131713867188,
      "loss": 1.0933,
      "margin_dpo/margin_mean": 38.34449005126953,
      "margin_dpo/margin_std": 55.98069763183594,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -75.8132553100586,
      "KL/mean": -95.68144989013672,
      "KL/rejected_KL_mean": -115.54964447021484,
      "KL/std": 57.8829231262207,
      "epoch": 0.36130007558578986,
      "fcm_dpo/beta": 0.011365259066224098,
      "fcm_dpo/delta": -0.07475205510854721,
      "fcm_dpo/margin": 39.73638153076172,
      "fcm_dpo/q_t": 0.39894479513168335,
      "grad_norm": 18.736330032348633,
      "learning_rate": 4.045397465551513e-07,
      "logits/chosen": 0.761664867401123,
      "logits/rejected": 0.6149318814277649,
      "logps/chosen": -132.5850830078125,
      "logps/ref_chosen": -56.771827697753906,
      "logps/ref_rejected": -116.23050689697266,
      "logps/rejected": -231.7801513671875,
      "loss": 1.1029,
      "margin_dpo/margin_mean": 39.73638153076172,
      "margin_dpo/margin_std": 61.89457702636719,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -73.51741027832031,
      "KL/mean": -98.4951171875,
      "KL/rejected_KL_mean": -123.47282409667969,
      "KL/std": 58.34437561035156,
      "epoch": 0.36281179138321995,
      "fcm_dpo/beta": 0.011012222617864609,
      "fcm_dpo/delta": -0.15893766283988953,
      "fcm_dpo/margin": 49.955413818359375,
      "fcm_dpo/q_t": 0.3772236406803131,
      "grad_norm": 12.825221061706543,
      "learning_rate": 4.0349825555680045e-07,
      "logits/chosen": 0.6706682443618774,
      "logits/rejected": 0.5711009502410889,
      "logps/chosen": -126.87152099609375,
      "logps/ref_chosen": -53.35411071777344,
      "logps/ref_rejected": -80.12019348144531,
      "logps/rejected": -203.593017578125,
      "loss": 1.017,
      "margin_dpo/margin_mean": 49.955413818359375,
      "margin_dpo/margin_std": 62.52080154418945,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -69.59007263183594,
      "KL/mean": -86.24327087402344,
      "KL/rejected_KL_mean": -102.89646911621094,
      "KL/std": 55.32313537597656,
      "epoch": 0.36432350718065004,
      "fcm_dpo/beta": 0.010927368886768818,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 33.3064079284668,
      "fcm_dpo/q_t": 0.41905292868614197,
      "grad_norm": 17.20731544494629,
      "learning_rate": 4.0245247088227377e-07,
      "logits/chosen": 0.6281282901763916,
      "logits/rejected": 0.591729998588562,
      "logps/chosen": -141.4854736328125,
      "logps/ref_chosen": -71.89541625976562,
      "logps/ref_rejected": -83.03492736816406,
      "logps/rejected": -185.931396484375,
      "loss": 1.1626,
      "margin_dpo/margin_mean": 33.3064079284668,
      "margin_dpo/margin_std": 62.025535583496094,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -71.19725036621094,
      "KL/mean": -91.92439270019531,
      "KL/rejected_KL_mean": -112.65153503417969,
      "KL/std": 57.12744140625,
      "epoch": 0.36583522297808013,
      "fcm_dpo/beta": 0.0107469093054533,
      "fcm_dpo/delta": -0.08326168358325958,
      "fcm_dpo/margin": 41.45427703857422,
      "fcm_dpo/q_t": 0.4003368616104126,
      "grad_norm": 12.280982971191406,
      "learning_rate": 4.0140242178441665e-07,
      "logits/chosen": 0.6384247541427612,
      "logits/rejected": 0.6186807155609131,
      "logps/chosen": -129.1246795654297,
      "logps/ref_chosen": -57.927433013916016,
      "logps/ref_rejected": -67.838623046875,
      "logps/rejected": -180.49014282226562,
      "loss": 1.0906,
      "margin_dpo/margin_mean": 41.45427703857422,
      "margin_dpo/margin_std": 61.07661056518555,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -70.12564086914062,
      "KL/mean": -89.19436645507812,
      "KL/rejected_KL_mean": -108.26309204101562,
      "KL/std": 56.41188430786133,
      "epoch": 0.3673469387755102,
      "fcm_dpo/beta": 0.0107469093054533,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.137451171875,
      "fcm_dpo/q_t": 0.40642350912094116,
      "grad_norm": 16.068456649780273,
      "learning_rate": 4.003481376353596e-07,
      "logits/chosen": 0.6034530401229858,
      "logits/rejected": 0.6172385215759277,
      "logps/chosen": -144.40231323242188,
      "logps/ref_chosen": -74.27667236328125,
      "logps/ref_rejected": -73.24340057373047,
      "logps/rejected": -181.50650024414062,
      "loss": 1.1101,
      "margin_dpo/margin_mean": 38.137451171875,
      "margin_dpo/margin_std": 58.296730041503906,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -64.46190643310547,
      "KL/mean": -89.28399658203125,
      "KL/rejected_KL_mean": -114.10609436035156,
      "KL/std": 56.00457000732422,
      "epoch": 0.3688586545729403,
      "fcm_dpo/beta": 0.010579448193311691,
      "fcm_dpo/delta": -0.13194507360458374,
      "fcm_dpo/margin": 49.64418029785156,
      "fcm_dpo/q_t": 0.3805602192878723,
      "grad_norm": 12.418740272521973,
      "learning_rate": 3.9928964792569654e-07,
      "logits/chosen": 0.6857927441596985,
      "logits/rejected": 0.5967106819152832,
      "logps/chosen": -117.82581329345703,
      "logps/ref_chosen": -53.36390686035156,
      "logps/ref_rejected": -71.10276794433594,
      "logps/rejected": -185.2088623046875,
      "loss": 1.0158,
      "margin_dpo/margin_mean": 49.64418029785156,
      "margin_dpo/margin_std": 57.57759094238281,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -71.72976684570312,
      "KL/mean": -100.52909851074219,
      "KL/rejected_KL_mean": -129.32843017578125,
      "KL/std": 58.295223236083984,
      "epoch": 0.37037037037037035,
      "fcm_dpo/beta": 0.010150259360671043,
      "fcm_dpo/delta": -0.19647106528282166,
      "fcm_dpo/margin": 57.598663330078125,
      "fcm_dpo/q_t": 0.3648463785648346,
      "grad_norm": 13.76476001739502,
      "learning_rate": 3.982269822636601e-07,
      "logits/chosen": 0.7182765007019043,
      "logits/rejected": 0.6913450956344604,
      "logps/chosen": -142.92486572265625,
      "logps/ref_chosen": -71.19510650634766,
      "logps/ref_rejected": -80.76235961914062,
      "logps/rejected": -210.09078979492188,
      "loss": 0.9685,
      "margin_dpo/margin_mean": 57.59866714477539,
      "margin_dpo/margin_std": 59.257904052734375,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -85.19328308105469,
      "KL/mean": -106.13648986816406,
      "KL/rejected_KL_mean": -127.07968139648438,
      "KL/std": 57.94656753540039,
      "epoch": 0.37188208616780044,
      "fcm_dpo/beta": 0.010008112527430058,
      "fcm_dpo/delta": -0.05554331839084625,
      "fcm_dpo/margin": 41.88639831542969,
      "fcm_dpo/q_t": 0.4062495231628418,
      "grad_norm": 12.851189613342285,
      "learning_rate": 3.971601703742932e-07,
      "logits/chosen": 0.7253904342651367,
      "logits/rejected": 0.6600919961929321,
      "logps/chosen": -156.8143310546875,
      "logps/ref_chosen": -71.62104797363281,
      "logps/ref_rejected": -94.03392028808594,
      "logps/rejected": -221.11361694335938,
      "loss": 1.1361,
      "margin_dpo/margin_mean": 41.88640213012695,
      "margin_dpo/margin_std": 73.20356750488281,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -87.87586212158203,
      "KL/mean": -101.99429321289062,
      "KL/rejected_KL_mean": -116.11273193359375,
      "KL/std": 58.836448669433594,
      "epoch": 0.37339380196523053,
      "fcm_dpo/beta": 0.009952524676918983,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.236865997314453,
      "fcm_dpo/q_t": 0.43647515773773193,
      "grad_norm": 15.817756652832031,
      "learning_rate": 3.960892420986177e-07,
      "logits/chosen": 0.657268762588501,
      "logits/rejected": 0.6478973627090454,
      "logps/chosen": -167.89840698242188,
      "logps/ref_chosen": -80.02254486083984,
      "logps/ref_rejected": -89.22705841064453,
      "logps/rejected": -205.33978271484375,
      "loss": 1.2344,
      "margin_dpo/margin_mean": 28.236865997314453,
      "margin_dpo/margin_std": 67.63511657714844,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -73.84143829345703,
      "KL/mean": -96.81367492675781,
      "KL/rejected_KL_mean": -119.78591918945312,
      "KL/std": 61.485862731933594,
      "epoch": 0.3749055177626606,
      "fcm_dpo/beta": 0.00990164466202259,
      "fcm_dpo/delta": -0.0513855516910553,
      "fcm_dpo/margin": 45.94447326660156,
      "fcm_dpo/q_t": 0.3986445665359497,
      "grad_norm": 16.848495483398438,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 0.656964898109436,
      "logits/rejected": 0.714156985282898,
      "logps/chosen": -139.21939086914062,
      "logps/ref_chosen": -65.37796020507812,
      "logps/ref_rejected": -61.365787506103516,
      "logps/rejected": -181.15170288085938,
      "loss": 1.0949,
      "margin_dpo/margin_mean": 45.94447326660156,
      "margin_dpo/margin_std": 70.1961898803711,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -93.00102233886719,
      "KL/mean": -101.233154296875,
      "KL/rejected_KL_mean": -109.46527099609375,
      "KL/std": 57.36132049560547,
      "epoch": 0.3764172335600907,
      "fcm_dpo/beta": 0.009850765578448772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 16.464248657226562,
      "fcm_dpo/q_t": 0.46179455518722534,
      "grad_norm": 16.948257446289062,
      "learning_rate": 3.9393515632731094e-07,
      "logits/chosen": 0.658106803894043,
      "logits/rejected": 0.7000288963317871,
      "logps/chosen": -167.60247802734375,
      "logps/ref_chosen": -74.60145568847656,
      "logps/ref_rejected": -63.79338455200195,
      "logps/rejected": -173.2586669921875,
      "loss": 1.3486,
      "margin_dpo/margin_mean": 16.464248657226562,
      "margin_dpo/margin_std": 69.59953308105469,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -81.48941040039062,
      "KL/mean": -105.01419830322266,
      "KL/rejected_KL_mean": -128.53900146484375,
      "KL/std": 64.33500671386719,
      "epoch": 0.3779289493575208,
      "fcm_dpo/beta": 0.009752372279763222,
      "fcm_dpo/delta": -0.10089530050754547,
      "fcm_dpo/margin": 47.049591064453125,
      "fcm_dpo/q_t": 0.3958304524421692,
      "grad_norm": 14.456826210021973,
      "learning_rate": 3.9285205908608934e-07,
      "logits/chosen": 0.7627512216567993,
      "logits/rejected": 0.7224296927452087,
      "logps/chosen": -143.4276123046875,
      "logps/ref_chosen": -61.938209533691406,
      "logps/ref_rejected": -72.21602630615234,
      "logps/rejected": -200.75502014160156,
      "loss": 1.0794,
      "margin_dpo/margin_mean": 47.04959487915039,
      "margin_dpo/margin_std": 66.09559631347656,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -87.73074340820312,
      "KL/mean": -103.30741882324219,
      "KL/rejected_KL_mean": -118.88409423828125,
      "KL/std": 56.92936706542969,
      "epoch": 0.3794406651549509,
      "fcm_dpo/beta": 0.009653978981077671,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.153358459472656,
      "fcm_dpo/q_t": 0.43081212043762207,
      "grad_norm": 22.772136688232422,
      "learning_rate": 3.9176496596569265e-07,
      "logits/chosen": 0.7472172975540161,
      "logits/rejected": 0.7043584585189819,
      "logps/chosen": -154.58767700195312,
      "logps/ref_chosen": -66.85694885253906,
      "logps/ref_rejected": -84.83396911621094,
      "logps/rejected": -203.71807861328125,
      "loss": 1.2011,
      "margin_dpo/margin_mean": 31.153358459472656,
      "margin_dpo/margin_std": 64.73333740234375,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -82.26924133300781,
      "KL/mean": -94.54788208007812,
      "KL/rejected_KL_mean": -106.82652282714844,
      "KL/std": 59.7686653137207,
      "epoch": 0.38095238095238093,
      "fcm_dpo/beta": 0.009653978981077671,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.557275772094727,
      "fcm_dpo/q_t": 0.4452328681945801,
      "grad_norm": 19.71927833557129,
      "learning_rate": 3.9067390737445254e-07,
      "logits/chosen": 0.6775693893432617,
      "logits/rejected": 0.6212589144706726,
      "logps/chosen": -138.4931640625,
      "logps/ref_chosen": -56.22393035888672,
      "logps/ref_rejected": -77.1136245727539,
      "logps/rejected": -183.94015502929688,
      "loss": 1.2756,
      "margin_dpo/margin_mean": 24.557275772094727,
      "margin_dpo/margin_std": 68.90715026855469,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -80.07911682128906,
      "KL/mean": -97.68270874023438,
      "KL/rejected_KL_mean": -115.28628540039062,
      "KL/std": 57.1429443359375,
      "epoch": 0.382464096749811,
      "fcm_dpo/beta": 0.009601515717804432,
      "fcm_dpo/delta": -0.027246130630373955,
      "fcm_dpo/margin": 35.207183837890625,
      "fcm_dpo/q_t": 0.42204049229621887,
      "grad_norm": 14.635730743408203,
      "learning_rate": 3.8957891383162304e-07,
      "logits/chosen": 0.7735403776168823,
      "logits/rejected": 0.7356526851654053,
      "logps/chosen": -132.28912353515625,
      "logps/ref_chosen": -52.21001434326172,
      "logps/ref_rejected": -58.75764846801758,
      "logps/rejected": -174.0439453125,
      "loss": 1.1572,
      "margin_dpo/margin_mean": 35.207183837890625,
      "margin_dpo/margin_std": 59.34178924560547,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -88.13926696777344,
      "KL/mean": -106.71817016601562,
      "KL/rejected_KL_mean": -125.29707336425781,
      "KL/std": 62.50201416015625,
      "epoch": 0.3839758125472411,
      "fcm_dpo/beta": 0.009601515717804432,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.157798767089844,
      "fcm_dpo/q_t": 0.41981834173202515,
      "grad_norm": 13.173120498657227,
      "learning_rate": 3.884800159665276e-07,
      "logits/chosen": 0.6648178100585938,
      "logits/rejected": 0.6116340160369873,
      "logps/chosen": -153.77560424804688,
      "logps/ref_chosen": -65.63632202148438,
      "logps/ref_rejected": -82.34425354003906,
      "logps/rejected": -207.64132690429688,
      "loss": 1.1467,
      "margin_dpo/margin_mean": 37.157798767089844,
      "margin_dpo/margin_std": 62.577247619628906,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -77.47877502441406,
      "KL/mean": -100.01842498779297,
      "KL/rejected_KL_mean": -122.55805969238281,
      "KL/std": 61.58515167236328,
      "epoch": 0.3854875283446712,
      "fcm_dpo/beta": 0.009527172893285751,
      "fcm_dpo/delta": -0.03886480629444122,
      "fcm_dpo/margin": 45.079288482666016,
      "fcm_dpo/q_t": 0.4024481773376465,
      "grad_norm": 15.716689109802246,
      "learning_rate": 3.873772445177015e-07,
      "logits/chosen": 0.6487714052200317,
      "logits/rejected": 0.6173020005226135,
      "logps/chosen": -145.38986206054688,
      "logps/ref_chosen": -67.91108703613281,
      "logps/ref_rejected": -83.89114379882812,
      "logps/rejected": -206.44921875,
      "loss": 1.0994,
      "margin_dpo/margin_mean": 45.07928466796875,
      "margin_dpo/margin_std": 67.75491333007812,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -94.20085144042969,
      "KL/mean": -115.31936645507812,
      "KL/rejected_KL_mean": -136.43788146972656,
      "KL/std": 59.03213119506836,
      "epoch": 0.3869992441421013,
      "fcm_dpo/beta": 0.009447208605706692,
      "fcm_dpo/delta": -0.042143430560827255,
      "fcm_dpo/margin": 42.237030029296875,
      "fcm_dpo/q_t": 0.4108726680278778,
      "grad_norm": 14.595268249511719,
      "learning_rate": 3.862706303320329e-07,
      "logits/chosen": 0.676605761051178,
      "logits/rejected": 0.6072751879692078,
      "logps/chosen": -157.70083618164062,
      "logps/ref_chosen": -63.49998474121094,
      "logps/ref_rejected": -90.77104187011719,
      "logps/rejected": -227.20892333984375,
      "loss": 1.1483,
      "margin_dpo/margin_mean": 42.237030029296875,
      "margin_dpo/margin_std": 76.56491088867188,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -89.88215637207031,
      "KL/mean": -116.78739166259766,
      "KL/rejected_KL_mean": -143.692626953125,
      "KL/std": 65.77367401123047,
      "epoch": 0.3885109599395314,
      "fcm_dpo/beta": 0.009294034913182259,
      "fcm_dpo/delta": -0.08173276484012604,
      "fcm_dpo/margin": 53.810462951660156,
      "fcm_dpo/q_t": 0.38952571153640747,
      "grad_norm": 11.852346420288086,
      "learning_rate": 3.851602043638994e-07,
      "logits/chosen": 0.672225296497345,
      "logits/rejected": 0.5996080636978149,
      "logps/chosen": -160.48281860351562,
      "logps/ref_chosen": -70.60064697265625,
      "logps/ref_rejected": -108.58313751220703,
      "logps/rejected": -252.2757568359375,
      "loss": 1.0582,
      "margin_dpo/margin_mean": 53.810462951660156,
      "margin_dpo/margin_std": 75.61648559570312,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -90.14016723632812,
      "KL/mean": -110.83567810058594,
      "KL/rejected_KL_mean": -131.53118896484375,
      "KL/std": 57.87552261352539,
      "epoch": 0.3900226757369615,
      "fcm_dpo/beta": 0.009294034913182259,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.391021728515625,
      "fcm_dpo/q_t": 0.41010379791259766,
      "grad_norm": 13.612093925476074,
      "learning_rate": 3.840459976743023e-07,
      "logits/chosen": 0.7125341892242432,
      "logits/rejected": 0.6583952307701111,
      "logps/chosen": -149.3943328857422,
      "logps/ref_chosen": -59.25416564941406,
      "logps/ref_rejected": -85.58709716796875,
      "logps/rejected": -217.1182861328125,
      "loss": 1.1076,
      "margin_dpo/margin_mean": 41.391021728515625,
      "margin_dpo/margin_std": 58.5115966796875,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -79.61314392089844,
      "KL/mean": -109.55941772460938,
      "KL/rejected_KL_mean": -139.50570678710938,
      "KL/std": 64.32805633544922,
      "epoch": 0.3915343915343915,
      "fcm_dpo/beta": 0.009043391793966293,
      "fcm_dpo/delta": -0.15097783505916595,
      "fcm_dpo/margin": 59.89255905151367,
      "fcm_dpo/q_t": 0.3776322901248932,
      "grad_norm": 13.222972869873047,
      "learning_rate": 3.8292804142999796e-07,
      "logits/chosen": 0.6543328166007996,
      "logits/rejected": 0.5453038215637207,
      "logps/chosen": -145.0480194091797,
      "logps/ref_chosen": -65.43487548828125,
      "logps/ref_rejected": -95.41731262207031,
      "logps/rejected": -234.92300415039062,
      "loss": 1.0328,
      "margin_dpo/margin_mean": 59.89255905151367,
      "margin_dpo/margin_std": 77.88516235351562,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -78.07093811035156,
      "KL/mean": -102.1653823852539,
      "KL/rejected_KL_mean": -126.25981903076172,
      "KL/std": 58.916969299316406,
      "epoch": 0.3930461073318216,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": -0.05486639216542244,
      "fcm_dpo/margin": 48.18886947631836,
      "fcm_dpo/q_t": 0.4026397168636322,
      "grad_norm": 19.17668914794922,
      "learning_rate": 3.818063669026256e-07,
      "logits/chosen": 0.6947526931762695,
      "logits/rejected": 0.5949761271476746,
      "logps/chosen": -127.1605224609375,
      "logps/ref_chosen": -49.08958435058594,
      "logps/ref_rejected": -79.01708221435547,
      "logps/rejected": -205.2769012451172,
      "loss": 1.1036,
      "margin_dpo/margin_mean": 48.188865661621094,
      "margin_dpo/margin_std": 73.38361358642578,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -78.03056335449219,
      "KL/mean": -98.04486846923828,
      "KL/rejected_KL_mean": -118.0591812133789,
      "KL/std": 57.562782287597656,
      "epoch": 0.3945578231292517,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 40.02862548828125,
      "fcm_dpo/q_t": 0.4189288318157196,
      "grad_norm": 16.82101058959961,
      "learning_rate": 3.806810054678331e-07,
      "logits/chosen": 0.5842745304107666,
      "logits/rejected": 0.6270730495452881,
      "logps/chosen": -148.9029541015625,
      "logps/ref_chosen": -70.87239074707031,
      "logps/ref_rejected": -65.01522064208984,
      "logps/rejected": -183.07440185546875,
      "loss": 1.1465,
      "margin_dpo/margin_mean": 40.02862548828125,
      "margin_dpo/margin_std": 67.19842529296875,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -85.85145568847656,
      "KL/mean": -104.21426391601562,
      "KL/rejected_KL_mean": -122.57707214355469,
      "KL/std": 58.654212951660156,
      "epoch": 0.3960695389266818,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 36.725616455078125,
      "fcm_dpo/q_t": 0.4247691035270691,
      "grad_norm": 16.46879005432129,
      "learning_rate": 3.7955198860439887e-07,
      "logits/chosen": 0.710350751876831,
      "logits/rejected": 0.6420841217041016,
      "logps/chosen": -153.72207641601562,
      "logps/ref_chosen": -67.8706283569336,
      "logps/ref_rejected": -88.7205810546875,
      "logps/rejected": -211.2976531982422,
      "loss": 1.16,
      "margin_dpo/margin_mean": 36.725616455078125,
      "margin_dpo/margin_std": 62.674476623535156,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -79.6908187866211,
      "KL/mean": -100.17828369140625,
      "KL/rejected_KL_mean": -120.66574096679688,
      "KL/std": 59.07019805908203,
      "epoch": 0.3975812547241119,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 40.97491455078125,
      "fcm_dpo/q_t": 0.41486692428588867,
      "grad_norm": 15.303149223327637,
      "learning_rate": 3.784193478933516e-07,
      "logits/chosen": 0.6577154397964478,
      "logits/rejected": 0.5451149940490723,
      "logps/chosen": -134.88540649414062,
      "logps/ref_chosen": -55.194583892822266,
      "logps/ref_rejected": -80.54048156738281,
      "logps/rejected": -201.2062225341797,
      "loss": 1.1287,
      "margin_dpo/margin_mean": 40.97491455078125,
      "margin_dpo/margin_std": 62.79846954345703,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -84.52911376953125,
      "KL/mean": -106.55697631835938,
      "KL/rejected_KL_mean": -128.5848388671875,
      "KL/std": 65.0322494506836,
      "epoch": 0.39909297052154197,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.05571746826172,
      "fcm_dpo/q_t": 0.41048991680145264,
      "grad_norm": 13.230753898620605,
      "learning_rate": 3.7728311501708674e-07,
      "logits/chosen": 0.5887368321418762,
      "logits/rejected": 0.5512345433235168,
      "logps/chosen": -167.69979858398438,
      "logps/ref_chosen": -83.17068481445312,
      "logps/ref_rejected": -88.33625793457031,
      "logps/rejected": -216.9210968017578,
      "loss": 1.1262,
      "margin_dpo/margin_mean": 44.055721282958984,
      "margin_dpo/margin_std": 70.82036590576172,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -80.12649536132812,
      "KL/mean": -102.91234588623047,
      "KL/rejected_KL_mean": -125.69821166992188,
      "KL/std": 62.13268280029297,
      "epoch": 0.40060468631897206,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.57171630859375,
      "fcm_dpo/q_t": 0.4080004394054413,
      "grad_norm": 11.275876998901367,
      "learning_rate": 3.7614332175848027e-07,
      "logits/chosen": 0.7446998357772827,
      "logits/rejected": 0.6785590052604675,
      "logps/chosen": -131.78933715820312,
      "logps/ref_chosen": -51.66284942626953,
      "logps/ref_rejected": -67.1720962524414,
      "logps/rejected": -192.87030029296875,
      "loss": 1.1342,
      "margin_dpo/margin_mean": 45.57171630859375,
      "margin_dpo/margin_std": 77.82930755615234,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -77.90547180175781,
      "KL/mean": -99.6781005859375,
      "KL/rejected_KL_mean": -121.45071411132812,
      "KL/std": 62.47583770751953,
      "epoch": 0.4021164021164021,
      "fcm_dpo/beta": 0.008919179439544678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.54524612426758,
      "fcm_dpo/q_t": 0.41210731863975525,
      "grad_norm": 19.963960647583008,
      "learning_rate": 3.75e-07,
      "logits/chosen": 0.6973221302032471,
      "logits/rejected": 0.6187717914581299,
      "logps/chosen": -135.35597229003906,
      "logps/ref_chosen": -57.45049285888672,
      "logps/ref_rejected": -77.60826110839844,
      "logps/rejected": -199.05897521972656,
      "loss": 1.1361,
      "margin_dpo/margin_mean": 43.54524612426758,
      "margin_dpo/margin_std": 73.66079711914062,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -69.62604522705078,
      "KL/mean": -87.49578857421875,
      "KL/rejected_KL_mean": -105.36552429199219,
      "KL/std": 60.37127685546875,
      "epoch": 0.4036281179138322,
      "fcm_dpo/beta": 0.00886848010122776,
      "fcm_dpo/delta": -0.05716828629374504,
      "fcm_dpo/margin": 35.739471435546875,
      "fcm_dpo/q_t": 0.42751142382621765,
      "grad_norm": 17.76947021484375,
      "learning_rate": 3.738531817228131e-07,
      "logits/chosen": 0.7317204475402832,
      "logits/rejected": 0.712775468826294,
      "logps/chosen": -124.66139221191406,
      "logps/ref_chosen": -55.03535079956055,
      "logps/ref_rejected": -66.0953369140625,
      "logps/rejected": -171.4608612060547,
      "loss": 1.2038,
      "margin_dpo/margin_mean": 35.739471435546875,
      "margin_dpo/margin_std": 72.42359924316406,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -69.93362426757812,
      "KL/mean": -91.20822143554688,
      "KL/rejected_KL_mean": -112.48282623291016,
      "KL/std": 63.411277770996094,
      "epoch": 0.4051398337112623,
      "fcm_dpo/beta": 0.008803716860711575,
      "fcm_dpo/delta": -0.015976281836628914,
      "fcm_dpo/margin": 42.549198150634766,
      "fcm_dpo/q_t": 0.414595365524292,
      "grad_norm": 16.74564552307129,
      "learning_rate": 3.7270289900589204e-07,
      "logits/chosen": 0.5760444402694702,
      "logits/rejected": 0.5645999312400818,
      "logps/chosen": -135.00537109375,
      "logps/ref_chosen": -65.07174682617188,
      "logps/ref_rejected": -71.42485809326172,
      "logps/rejected": -183.90768432617188,
      "loss": 1.1182,
      "margin_dpo/margin_mean": 42.549198150634766,
      "margin_dpo/margin_std": 62.1130485534668,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -81.66681671142578,
      "KL/mean": -105.6706314086914,
      "KL/rejected_KL_mean": -129.6744384765625,
      "KL/std": 67.39564514160156,
      "epoch": 0.40665154950869237,
      "fcm_dpo/beta": 0.008736472576856613,
      "fcm_dpo/delta": -0.06087144464254379,
      "fcm_dpo/margin": 48.00763702392578,
      "fcm_dpo/q_t": 0.4043356776237488,
      "grad_norm": 13.045381546020508,
      "learning_rate": 3.7154918402511714e-07,
      "logits/chosen": 0.7855197191238403,
      "logits/rejected": 0.7380328178405762,
      "logps/chosen": -148.8030242919922,
      "logps/ref_chosen": -67.1362075805664,
      "logps/ref_rejected": -82.55778503417969,
      "logps/rejected": -212.23223876953125,
      "loss": 1.0931,
      "margin_dpo/margin_mean": 48.00763702392578,
      "margin_dpo/margin_std": 67.01888275146484,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -79.60834503173828,
      "KL/mean": -100.95454406738281,
      "KL/rejected_KL_mean": -122.3007583618164,
      "KL/std": 65.51744079589844,
      "epoch": 0.40816326530612246,
      "fcm_dpo/beta": 0.008683293126523495,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.692413330078125,
      "fcm_dpo/q_t": 0.414852499961853,
      "grad_norm": 14.931079864501953,
      "learning_rate": 3.7039206905237656e-07,
      "logits/chosen": 0.7250645756721497,
      "logits/rejected": 0.6401588320732117,
      "logps/chosen": -146.2970428466797,
      "logps/ref_chosen": -66.6886978149414,
      "logps/ref_rejected": -85.16129302978516,
      "logps/rejected": -207.46205139160156,
      "loss": 1.1352,
      "margin_dpo/margin_mean": 42.692413330078125,
      "margin_dpo/margin_std": 69.67828369140625,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -81.57511138916016,
      "KL/mean": -97.54861450195312,
      "KL/rejected_KL_mean": -113.5221176147461,
      "KL/std": 70.79106140136719,
      "epoch": 0.40967498110355255,
      "fcm_dpo/beta": 0.008683293126523495,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.947006225585938,
      "fcm_dpo/q_t": 0.44066426157951355,
      "grad_norm": 20.921737670898438,
      "learning_rate": 3.692315864546635e-07,
      "logits/chosen": 0.6996304988861084,
      "logits/rejected": 0.6347111463546753,
      "logps/chosen": -153.982666015625,
      "logps/ref_chosen": -72.40754699707031,
      "logps/ref_rejected": -92.06311798095703,
      "logps/rejected": -205.58523559570312,
      "loss": 1.2502,
      "margin_dpo/margin_mean": 31.94700813293457,
      "margin_dpo/margin_std": 83.21861267089844,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -67.67726135253906,
      "KL/mean": -98.39691925048828,
      "KL/rejected_KL_mean": -129.1165771484375,
      "KL/std": 62.144187927246094,
      "epoch": 0.41118669690098264,
      "fcm_dpo/beta": 0.008535921573638916,
      "fcm_dpo/delta": -0.13113445043563843,
      "fcm_dpo/margin": 61.4393310546875,
      "fcm_dpo/q_t": 0.37742963433265686,
      "grad_norm": 12.506288528442383,
      "learning_rate": 3.6806776869317067e-07,
      "logits/chosen": 0.742793083190918,
      "logits/rejected": 0.7653173208236694,
      "logps/chosen": -134.27865600585938,
      "logps/ref_chosen": -66.60140228271484,
      "logps/ref_rejected": -67.74340057373047,
      "logps/rejected": -196.8599853515625,
      "loss": 0.9911,
      "margin_dpo/margin_mean": 61.4393310546875,
      "margin_dpo/margin_std": 59.98362731933594,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -88.57992553710938,
      "KL/mean": -110.85275268554688,
      "KL/rejected_KL_mean": -133.12557983398438,
      "KL/std": 67.68528747558594,
      "epoch": 0.4126984126984127,
      "fcm_dpo/beta": 0.008458517491817474,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.54566192626953,
      "fcm_dpo/q_t": 0.41627636551856995,
      "grad_norm": 20.072294235229492,
      "learning_rate": 3.669006483223828e-07,
      "logits/chosen": 0.7203218936920166,
      "logits/rejected": 0.6446723937988281,
      "logps/chosen": -145.934814453125,
      "logps/ref_chosen": -57.35487747192383,
      "logps/ref_rejected": -84.17168426513672,
      "logps/rejected": -217.29727172851562,
      "loss": 1.1725,
      "margin_dpo/margin_mean": 44.545658111572266,
      "margin_dpo/margin_std": 86.9574203491211,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -85.86259460449219,
      "KL/mean": -110.53076171875,
      "KL/rejected_KL_mean": -135.1989288330078,
      "KL/std": 66.234619140625,
      "epoch": 0.41421012849584277,
      "fcm_dpo/beta": 0.00843963585793972,
      "fcm_dpo/delta": -0.02237236686050892,
      "fcm_dpo/margin": 49.33633804321289,
      "fcm_dpo/q_t": 0.4048211872577667,
      "grad_norm": 14.553647994995117,
      "learning_rate": 3.657302579891656e-07,
      "logits/chosen": 0.5977407693862915,
      "logits/rejected": 0.581523060798645,
      "logps/chosen": -145.50408935546875,
      "logps/ref_chosen": -59.64149475097656,
      "logps/ref_rejected": -68.29348754882812,
      "logps/rejected": -203.49240112304688,
      "loss": 1.1142,
      "margin_dpo/margin_mean": 49.33633804321289,
      "margin_dpo/margin_std": 77.90882110595703,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -85.02987670898438,
      "KL/mean": -108.27204895019531,
      "KL/rejected_KL_mean": -131.51422119140625,
      "KL/std": 63.815208435058594,
      "epoch": 0.41572184429327286,
      "fcm_dpo/beta": 0.008404111489653587,
      "fcm_dpo/delta": -0.019802536815404892,
      "fcm_dpo/margin": 46.48435974121094,
      "fcm_dpo/q_t": 0.41008520126342773,
      "grad_norm": 21.007122039794922,
      "learning_rate": 3.645566304318526e-07,
      "logits/chosen": 0.6761637926101685,
      "logits/rejected": 0.5931113958358765,
      "logps/chosen": -138.2965087890625,
      "logps/ref_chosen": -53.26664352416992,
      "logps/ref_rejected": -73.84062194824219,
      "logps/rejected": -205.3548583984375,
      "loss": 1.1075,
      "margin_dpo/margin_mean": 46.4843635559082,
      "margin_dpo/margin_std": 66.59793090820312,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -79.12834167480469,
      "KL/mean": -102.84222412109375,
      "KL/rejected_KL_mean": -126.55609893798828,
      "KL/std": 59.876068115234375,
      "epoch": 0.41723356009070295,
      "fcm_dpo/beta": 0.008387469686567783,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 47.427764892578125,
      "fcm_dpo/q_t": 0.4078884720802307,
      "grad_norm": 16.673648834228516,
      "learning_rate": 3.633797984793294e-07,
      "logits/chosen": 0.6499893069267273,
      "logits/rejected": 0.6180636882781982,
      "logps/chosen": -132.14913940429688,
      "logps/ref_chosen": -53.02079772949219,
      "logps/ref_rejected": -61.56678771972656,
      "logps/rejected": -188.12289428710938,
      "loss": 1.1034,
      "margin_dpo/margin_mean": 47.427764892578125,
      "margin_dpo/margin_std": 67.45655822753906,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -92.67013549804688,
      "KL/mean": -104.78817749023438,
      "KL/rejected_KL_mean": -116.90620422363281,
      "KL/std": 67.29153442382812,
      "epoch": 0.41874527588813304,
      "fcm_dpo/beta": 0.008387469686567783,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 24.23607635498047,
      "fcm_dpo/q_t": 0.4531954526901245,
      "grad_norm": 19.32560157775879,
      "learning_rate": 3.6219979505011555e-07,
      "logits/chosen": 0.7294178009033203,
      "logits/rejected": 0.7562111616134644,
      "logps/chosen": -164.10311889648438,
      "logps/ref_chosen": -71.43299102783203,
      "logps/ref_rejected": -67.65852355957031,
      "logps/rejected": -184.56472778320312,
      "loss": 1.2928,
      "margin_dpo/margin_mean": 24.23607635498047,
      "margin_dpo/margin_std": 76.65855407714844,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -98.54222106933594,
      "KL/mean": -121.556884765625,
      "KL/rejected_KL_mean": -144.57154846191406,
      "KL/std": 69.13101196289062,
      "epoch": 0.42025699168556313,
      "fcm_dpo/beta": 0.00834178552031517,
      "fcm_dpo/delta": -0.05476619303226471,
      "fcm_dpo/margin": 46.029335021972656,
      "fcm_dpo/q_t": 0.41048651933670044,
      "grad_norm": 19.770599365234375,
      "learning_rate": 3.6101665315144353e-07,
      "logits/chosen": 0.6561414003372192,
      "logits/rejected": 0.5932800769805908,
      "logps/chosen": -165.65298461914062,
      "logps/ref_chosen": -67.11076354980469,
      "logps/ref_rejected": -88.74851989746094,
      "logps/rejected": -233.320068359375,
      "loss": 1.1265,
      "margin_dpo/margin_mean": 46.029335021972656,
      "margin_dpo/margin_std": 71.37066650390625,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -77.33357238769531,
      "KL/mean": -108.97832489013672,
      "KL/rejected_KL_mean": -140.62307739257812,
      "KL/std": 66.38153076171875,
      "epoch": 0.4217687074829932,
      "fcm_dpo/beta": 0.008170300163328648,
      "fcm_dpo/delta": -0.12329418212175369,
      "fcm_dpo/margin": 63.28950500488281,
      "fcm_dpo/q_t": 0.3793160319328308,
      "grad_norm": 18.47074317932129,
      "learning_rate": 3.5983040587833563e-07,
      "logits/chosen": 0.684441089630127,
      "logits/rejected": 0.6455780267715454,
      "logps/chosen": -131.8310546875,
      "logps/ref_chosen": -54.49748611450195,
      "logps/ref_rejected": -70.42373657226562,
      "logps/rejected": -211.04681396484375,
      "loss": 1.0028,
      "margin_dpo/margin_mean": 63.28950500488281,
      "margin_dpo/margin_std": 64.86318969726562,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -84.39732360839844,
      "KL/mean": -117.23088836669922,
      "KL/rejected_KL_mean": -150.064453125,
      "KL/std": 69.86494445800781,
      "epoch": 0.42328042328042326,
      "fcm_dpo/beta": 0.007942087948322296,
      "fcm_dpo/delta": -0.1282101720571518,
      "fcm_dpo/margin": 65.66712951660156,
      "fcm_dpo/q_t": 0.3792378902435303,
      "grad_norm": 13.688976287841797,
      "learning_rate": 3.586410864126781e-07,
      "logits/chosen": 0.7099773287773132,
      "logits/rejected": 0.6641367673873901,
      "logps/chosen": -144.83013916015625,
      "logps/ref_chosen": -60.43281173706055,
      "logps/ref_rejected": -78.39051818847656,
      "logps/rejected": -228.45497131347656,
      "loss": 0.9951,
      "margin_dpo/margin_mean": 65.66712951660156,
      "margin_dpo/margin_std": 66.24174499511719,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -87.24712371826172,
      "KL/mean": -113.78001403808594,
      "KL/rejected_KL_mean": -140.31289672851562,
      "KL/std": 68.34732818603516,
      "epoch": 0.42479213907785335,
      "fcm_dpo/beta": 0.007860523648560047,
      "fcm_dpo/delta": -0.03638084977865219,
      "fcm_dpo/margin": 53.0657844543457,
      "fcm_dpo/q_t": 0.4040064811706543,
      "grad_norm": 15.016488075256348,
      "learning_rate": 3.574487280222929e-07,
      "logits/chosen": 0.678945779800415,
      "logits/rejected": 0.7068662643432617,
      "logps/chosen": -147.52920532226562,
      "logps/ref_chosen": -60.2820930480957,
      "logps/ref_rejected": -62.04009246826172,
      "logps/rejected": -202.35299682617188,
      "loss": 1.0984,
      "margin_dpo/margin_mean": 53.0657844543457,
      "margin_dpo/margin_std": 76.41177368164062,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -98.00955200195312,
      "KL/mean": -122.0562973022461,
      "KL/rejected_KL_mean": -146.10305786132812,
      "KL/std": 69.0963134765625,
      "epoch": 0.42630385487528344,
      "fcm_dpo/beta": 0.007776046171784401,
      "fcm_dpo/delta": -0.07186315953731537,
      "fcm_dpo/margin": 48.0934944152832,
      "fcm_dpo/q_t": 0.41511061787605286,
      "grad_norm": 18.2562313079834,
      "learning_rate": 3.562533640600075e-07,
      "logits/chosen": 0.615809440612793,
      "logits/rejected": 0.5732920169830322,
      "logps/chosen": -158.63348388671875,
      "logps/ref_chosen": -60.623924255371094,
      "logps/ref_rejected": -68.67400360107422,
      "logps/rejected": -214.7770538330078,
      "loss": 1.144,
      "margin_dpo/margin_mean": 48.0934944152832,
      "margin_dpo/margin_std": 79.40950775146484,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -95.66755676269531,
      "KL/mean": -121.100830078125,
      "KL/rejected_KL_mean": -146.53411865234375,
      "KL/std": 72.49221801757812,
      "epoch": 0.42781557067271353,
      "fcm_dpo/beta": 0.007720166351646185,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 50.866546630859375,
      "fcm_dpo/q_t": 0.410746693611145,
      "grad_norm": 13.532066345214844,
      "learning_rate": 3.550550279627215e-07,
      "logits/chosen": 0.6475073099136353,
      "logits/rejected": 0.5460326671600342,
      "logps/chosen": -163.3153076171875,
      "logps/ref_chosen": -67.64775085449219,
      "logps/ref_rejected": -99.96835327148438,
      "logps/rejected": -246.50247192382812,
      "loss": 1.125,
      "margin_dpo/margin_mean": 50.866546630859375,
      "margin_dpo/margin_std": 81.53480529785156,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -92.60566711425781,
      "KL/mean": -117.02914428710938,
      "KL/rejected_KL_mean": -141.45263671875,
      "KL/std": 71.92315673828125,
      "epoch": 0.4293272864701436,
      "fcm_dpo/beta": 0.007683092262595892,
      "fcm_dpo/delta": -0.024069122970104218,
      "fcm_dpo/margin": 48.846961975097656,
      "fcm_dpo/q_t": 0.4131779670715332,
      "grad_norm": 11.925021171569824,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 0.7429874539375305,
      "logits/rejected": 0.6754733324050903,
      "logps/chosen": -149.57308959960938,
      "logps/ref_chosen": -56.96742630004883,
      "logps/ref_rejected": -86.36236572265625,
      "logps/rejected": -227.81500244140625,
      "loss": 1.1139,
      "margin_dpo/margin_mean": 48.846961975097656,
      "margin_dpo/margin_std": 69.17724609375,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -108.53469848632812,
      "KL/mean": -128.46315002441406,
      "KL/rejected_KL_mean": -148.39158630371094,
      "KL/std": 65.19612121582031,
      "epoch": 0.4308390022675737,
      "fcm_dpo/beta": 0.007683092262595892,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.85688781738281,
      "fcm_dpo/q_t": 0.4297737181186676,
      "grad_norm": 16.518638610839844,
      "learning_rate": 3.5264957352549375e-07,
      "logits/chosen": 0.7107409238815308,
      "logits/rejected": 0.6831285953521729,
      "logps/chosen": -180.19081115722656,
      "logps/ref_chosen": -71.65611267089844,
      "logps/ref_rejected": -81.63829803466797,
      "logps/rejected": -230.02987670898438,
      "loss": 1.1817,
      "margin_dpo/margin_mean": 39.85688400268555,
      "margin_dpo/margin_std": 73.79049682617188,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -103.9982681274414,
      "KL/mean": -136.58309936523438,
      "KL/rejected_KL_mean": -169.1679229736328,
      "KL/std": 70.7746810913086,
      "epoch": 0.4323507180650038,
      "fcm_dpo/beta": 0.007573241367936134,
      "fcm_dpo/delta": -0.0983816385269165,
      "fcm_dpo/margin": 65.16966247558594,
      "fcm_dpo/q_t": 0.3875123858451843,
      "grad_norm": 14.638473510742188,
      "learning_rate": 3.514425224712835e-07,
      "logits/chosen": 0.635071873664856,
      "logits/rejected": 0.5436767339706421,
      "logps/chosen": -165.07778930664062,
      "logps/ref_chosen": -61.07952117919922,
      "logps/ref_rejected": -91.28128051757812,
      "logps/rejected": -260.44921875,
      "loss": 1.0267,
      "margin_dpo/margin_mean": 65.16966247558594,
      "margin_dpo/margin_std": 75.55731964111328,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -82.97677612304688,
      "KL/mean": -115.36837768554688,
      "KL/rejected_KL_mean": -147.7599639892578,
      "KL/std": 70.15164184570312,
      "epoch": 0.43386243386243384,
      "fcm_dpo/beta": 0.007446384988725185,
      "fcm_dpo/delta": -0.08650145679712296,
      "fcm_dpo/margin": 64.78318786621094,
      "fcm_dpo/q_t": 0.3885365128517151,
      "grad_norm": 10.52198314666748,
      "learning_rate": 3.502326338516534e-07,
      "logits/chosen": 0.7257726192474365,
      "logits/rejected": 0.6863799095153809,
      "logps/chosen": -129.0125732421875,
      "logps/ref_chosen": -46.035789489746094,
      "logps/ref_rejected": -59.95293426513672,
      "logps/rejected": -207.712890625,
      "loss": 1.0308,
      "margin_dpo/margin_mean": 64.78318786621094,
      "margin_dpo/margin_std": 73.37957763671875,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -107.4090805053711,
      "KL/mean": -132.0665740966797,
      "KL/rejected_KL_mean": -156.72406005859375,
      "KL/std": 69.27061462402344,
      "epoch": 0.43537414965986393,
      "fcm_dpo/beta": 0.0074041858315467834,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.314971923828125,
      "fcm_dpo/q_t": 0.41598302125930786,
      "grad_norm": 12.507437705993652,
      "learning_rate": 3.490199415097892e-07,
      "logits/chosen": 0.5906128883361816,
      "logits/rejected": 0.5325485467910767,
      "logps/chosen": -172.7999267578125,
      "logps/ref_chosen": -65.3908462524414,
      "logps/ref_rejected": -88.53607940673828,
      "logps/rejected": -245.2601318359375,
      "loss": 1.1293,
      "margin_dpo/margin_mean": 49.314971923828125,
      "margin_dpo/margin_std": 75.95675659179688,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -108.69920349121094,
      "KL/mean": -132.19676208496094,
      "KL/rejected_KL_mean": -155.69430541992188,
      "KL/std": 69.22113800048828,
      "epoch": 0.436885865457294,
      "fcm_dpo/beta": 0.0074041858315467834,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.99510192871094,
      "fcm_dpo/q_t": 0.4210740029811859,
      "grad_norm": 11.996780395507812,
      "learning_rate": 3.4780447936730247e-07,
      "logits/chosen": 0.7544581294059753,
      "logits/rejected": 0.7194222807884216,
      "logps/chosen": -163.2928466796875,
      "logps/ref_chosen": -54.5936279296875,
      "logps/ref_rejected": -67.20855712890625,
      "logps/rejected": -222.90286254882812,
      "loss": 1.1505,
      "margin_dpo/margin_mean": 46.99510192871094,
      "margin_dpo/margin_std": 79.12466430664062,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -122.24220275878906,
      "KL/mean": -147.22738647460938,
      "KL/rejected_KL_mean": -172.21255493164062,
      "KL/std": 67.15545654296875,
      "epoch": 0.4383975812547241,
      "fcm_dpo/beta": 0.007365147117525339,
      "fcm_dpo/delta": -0.02643224596977234,
      "fcm_dpo/margin": 49.970359802246094,
      "fcm_dpo/q_t": 0.41537922620773315,
      "grad_norm": 15.518068313598633,
      "learning_rate": 3.465862814232821e-07,
      "logits/chosen": 0.7712986469268799,
      "logits/rejected": 0.6948202252388,
      "logps/chosen": -183.62677001953125,
      "logps/ref_chosen": -61.38457489013672,
      "logps/ref_rejected": -91.92778015136719,
      "logps/rejected": -264.14031982421875,
      "loss": 1.1284,
      "margin_dpo/margin_mean": 49.97035598754883,
      "margin_dpo/margin_std": 76.6805648803711,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -111.89765167236328,
      "KL/mean": -143.10159301757812,
      "KL/rejected_KL_mean": -174.3055419921875,
      "KL/std": 69.66302490234375,
      "epoch": 0.4399092970521542,
      "fcm_dpo/beta": 0.007289406843483448,
      "fcm_dpo/delta": -0.10390862822532654,
      "fcm_dpo/margin": 62.407894134521484,
      "fcm_dpo/q_t": 0.39798957109451294,
      "grad_norm": 12.921642303466797,
      "learning_rate": 3.4536538175334343e-07,
      "logits/chosen": 0.8233385682106018,
      "logits/rejected": 0.7497340440750122,
      "logps/chosen": -162.76068115234375,
      "logps/ref_chosen": -50.863037109375,
      "logps/ref_rejected": -82.20868682861328,
      "logps/rejected": -256.51422119140625,
      "loss": 1.0685,
      "margin_dpo/margin_mean": 62.40789031982422,
      "margin_dpo/margin_std": 82.35305786132812,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -113.44772338867188,
      "KL/mean": -134.71461486816406,
      "KL/rejected_KL_mean": -155.9815216064453,
      "KL/std": 73.57888793945312,
      "epoch": 0.4414210128495843,
      "fcm_dpo/beta": 0.00721366610378027,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.53379821777344,
      "fcm_dpo/q_t": 0.4306395351886749,
      "grad_norm": 15.100018501281738,
      "learning_rate": 3.4414181450867465e-07,
      "logits/chosen": 0.720421552658081,
      "logits/rejected": 0.6704583764076233,
      "logps/chosen": -177.79660034179688,
      "logps/ref_chosen": -64.34888458251953,
      "logps/ref_rejected": -72.86434173583984,
      "logps/rejected": -228.84585571289062,
      "loss": 1.2032,
      "margin_dpo/margin_mean": 42.53379821777344,
      "margin_dpo/margin_std": 90.44749450683594,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -105.96971130371094,
      "KL/mean": -140.18777465820312,
      "KL/rejected_KL_mean": -174.40582275390625,
      "KL/std": 74.28352355957031,
      "epoch": 0.4429327286470144,
      "fcm_dpo/beta": 0.007149941753596067,
      "fcm_dpo/delta": -0.08912809938192368,
      "fcm_dpo/margin": 68.43610382080078,
      "fcm_dpo/q_t": 0.38950157165527344,
      "grad_norm": 10.191673278808594,
      "learning_rate": 3.4291561391508185e-07,
      "logits/chosen": 0.822028398513794,
      "logits/rejected": 0.7336381673812866,
      "logps/chosen": -160.83917236328125,
      "logps/ref_chosen": -54.869468688964844,
      "logps/ref_rejected": -81.858642578125,
      "logps/rejected": -256.26446533203125,
      "loss": 1.0537,
      "margin_dpo/margin_mean": 68.43610382080078,
      "margin_dpo/margin_std": 89.89541625976562,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -113.77799987792969,
      "KL/mean": -136.8584442138672,
      "KL/rejected_KL_mean": -159.93887329101562,
      "KL/std": 76.31533813476562,
      "epoch": 0.4444444444444444,
      "fcm_dpo/beta": 0.007086217403411865,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.1608772277832,
      "fcm_dpo/q_t": 0.4244413375854492,
      "grad_norm": 11.190646171569824,
      "learning_rate": 3.4168681427203153e-07,
      "logits/chosen": 0.7264614105224609,
      "logits/rejected": 0.6801228523254395,
      "logps/chosen": -170.44891357421875,
      "logps/ref_chosen": -56.670902252197266,
      "logps/ref_rejected": -70.32819366455078,
      "logps/rejected": -230.26707458496094,
      "loss": 1.1415,
      "margin_dpo/margin_mean": 46.16087341308594,
      "margin_dpo/margin_std": 68.03617095947266,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -117.91046142578125,
      "KL/mean": -139.72238159179688,
      "KL/rejected_KL_mean": -161.53427124023438,
      "KL/std": 71.26055908203125,
      "epoch": 0.4459561602418745,
      "fcm_dpo/beta": 0.007086217403411865,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 43.623809814453125,
      "fcm_dpo/q_t": 0.4286356568336487,
      "grad_norm": 16.999357223510742,
      "learning_rate": 3.4045544995169125e-07,
      "logits/chosen": 0.7363671660423279,
      "logits/rejected": 0.6302077174186707,
      "logps/chosen": -168.3113555908203,
      "logps/ref_chosen": -50.40088653564453,
      "logps/ref_rejected": -83.43521881103516,
      "logps/rejected": -244.96949768066406,
      "loss": 1.1704,
      "margin_dpo/margin_mean": 43.623809814453125,
      "margin_dpo/margin_std": 76.41813659667969,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -121.54795837402344,
      "KL/mean": -148.98709106445312,
      "KL/rejected_KL_mean": -176.4262237548828,
      "KL/std": 78.2703857421875,
      "epoch": 0.4474678760393046,
      "fcm_dpo/beta": 0.007052321918308735,
      "fcm_dpo/delta": -0.023973939940333366,
      "fcm_dpo/margin": 54.87827682495117,
      "fcm_dpo/q_t": 0.4116702377796173,
      "grad_norm": 12.077364921569824,
      "learning_rate": 3.392215553979679e-07,
      "logits/chosen": 0.7017396688461304,
      "logits/rejected": 0.6508808135986328,
      "logps/chosen": -190.69830322265625,
      "logps/ref_chosen": -69.15034484863281,
      "logps/ref_rejected": -89.60166931152344,
      "logps/rejected": -266.02789306640625,
      "loss": 1.1304,
      "margin_dpo/margin_mean": 54.878273010253906,
      "margin_dpo/margin_std": 89.46514892578125,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -124.2834243774414,
      "KL/mean": -151.585693359375,
      "KL/rejected_KL_mean": -178.887939453125,
      "KL/std": 68.32637023925781,
      "epoch": 0.4489795918367347,
      "fcm_dpo/beta": 0.007041130214929581,
      "fcm_dpo/delta": -0.01589428447186947,
      "fcm_dpo/margin": 54.60452651977539,
      "fcm_dpo/q_t": 0.41009995341300964,
      "grad_norm": 12.764442443847656,
      "learning_rate": 3.3798516512554485e-07,
      "logits/chosen": 0.7037564516067505,
      "logits/rejected": 0.6517189741134644,
      "logps/chosen": -182.2997283935547,
      "logps/ref_chosen": -58.01630401611328,
      "logps/ref_rejected": -69.95780944824219,
      "logps/rejected": -248.84576416015625,
      "loss": 1.1013,
      "margin_dpo/margin_mean": 54.60452651977539,
      "margin_dpo/margin_std": 72.62581634521484,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -121.14657592773438,
      "KL/mean": -141.3595733642578,
      "KL/rejected_KL_mean": -161.57257080078125,
      "KL/std": 72.325927734375,
      "epoch": 0.4504913076341648,
      "fcm_dpo/beta": 0.007029938977211714,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 40.426002502441406,
      "fcm_dpo/q_t": 0.4347230792045593,
      "grad_norm": 11.478727340698242,
      "learning_rate": 3.367463137189156e-07,
      "logits/chosen": 0.8037296533584595,
      "logits/rejected": 0.7485308647155762,
      "logps/chosen": -177.31588745117188,
      "logps/ref_chosen": -56.1693115234375,
      "logps/ref_rejected": -68.55052185058594,
      "logps/rejected": -230.1230926513672,
      "loss": 1.2023,
      "margin_dpo/margin_mean": 40.426002502441406,
      "margin_dpo/margin_std": 82.11355590820312,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -127.74474334716797,
      "KL/mean": -144.35882568359375,
      "KL/rejected_KL_mean": -160.972900390625,
      "KL/std": 71.02047729492188,
      "epoch": 0.4520030234315949,
      "fcm_dpo/beta": 0.007029938977211714,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 33.2281494140625,
      "fcm_dpo/q_t": 0.44710543751716614,
      "grad_norm": 19.625972747802734,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": 0.5966833829879761,
      "logits/rejected": 0.5654438734054565,
      "logps/chosen": -190.0625457763672,
      "logps/ref_chosen": -62.31780242919922,
      "logps/ref_rejected": -72.60028839111328,
      "logps/rejected": -233.57318115234375,
      "loss": 1.2661,
      "margin_dpo/margin_mean": 33.228145599365234,
      "margin_dpo/margin_std": 90.59477996826172,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -126.5094223022461,
      "KL/mean": -148.61163330078125,
      "KL/rejected_KL_mean": -170.7138214111328,
      "KL/std": 72.69538879394531,
      "epoch": 0.45351473922902497,
      "fcm_dpo/beta": 0.007029938977211714,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.20439147949219,
      "fcm_dpo/q_t": 0.428915411233902,
      "grad_norm": 13.045540809631348,
      "learning_rate": 3.3426136618426043e-07,
      "logits/chosen": 0.7217717170715332,
      "logits/rejected": 0.6547795534133911,
      "logps/chosen": -186.89100646972656,
      "logps/ref_chosen": -60.38157653808594,
      "logps/ref_rejected": -75.45442199707031,
      "logps/rejected": -246.16824340820312,
      "loss": 1.185,
      "margin_dpo/margin_mean": 44.20439147949219,
      "margin_dpo/margin_std": 84.41621398925781,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -126.29975891113281,
      "KL/mean": -145.92193603515625,
      "KL/rejected_KL_mean": -165.54409790039062,
      "KL/std": 71.30606079101562,
      "epoch": 0.455026455026455,
      "fcm_dpo/beta": 0.007029938977211714,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 39.244346618652344,
      "fcm_dpo/q_t": 0.4370271563529968,
      "grad_norm": 12.925996780395508,
      "learning_rate": 3.3301533956555885e-07,
      "logits/chosen": 0.7587097883224487,
      "logits/rejected": 0.7291955351829529,
      "logps/chosen": -179.15066528320312,
      "logps/ref_chosen": -52.85089111328125,
      "logps/ref_rejected": -69.97584533691406,
      "logps/rejected": -235.51995849609375,
      "loss": 1.2112,
      "margin_dpo/margin_mean": 39.244346618652344,
      "margin_dpo/margin_std": 83.38739013671875,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -133.72024536132812,
      "KL/mean": -148.84934997558594,
      "KL/rejected_KL_mean": -163.9784698486328,
      "KL/std": 75.41315460205078,
      "epoch": 0.4565381708238851,
      "fcm_dpo/beta": 0.007029938977211714,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 30.258220672607422,
      "fcm_dpo/q_t": 0.45121005177497864,
      "grad_norm": 13.880743980407715,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 0.5911428928375244,
      "logits/rejected": 0.5351113677024841,
      "logps/chosen": -200.686767578125,
      "logps/ref_chosen": -66.96650695800781,
      "logps/ref_rejected": -88.09510803222656,
      "logps/rejected": -252.07357788085938,
      "loss": 1.2674,
      "margin_dpo/margin_mean": 30.258224487304688,
      "margin_dpo/margin_std": 83.85945129394531,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -114.63668823242188,
      "KL/mean": -145.92343139648438,
      "KL/rejected_KL_mean": -177.21014404296875,
      "KL/std": 77.03577423095703,
      "epoch": 0.4580498866213152,
      "fcm_dpo/beta": 0.007015762850642204,
      "fcm_dpo/delta": -0.020205684006214142,
      "fcm_dpo/margin": 62.5734748840332,
      "fcm_dpo/q_t": 0.40118786692619324,
      "grad_norm": 10.651202201843262,
      "learning_rate": 3.3051635489464793e-07,
      "logits/chosen": 0.6889491081237793,
      "logits/rejected": 0.6172829270362854,
      "logps/chosen": -176.75820922851562,
      "logps/ref_chosen": -62.12152862548828,
      "logps/ref_rejected": -90.31204223632812,
      "logps/rejected": -267.5221862792969,
      "loss": 1.1054,
      "margin_dpo/margin_mean": 62.57347106933594,
      "margin_dpo/margin_std": 97.65402221679688,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -105.85643005371094,
      "KL/mean": -137.82760620117188,
      "KL/rejected_KL_mean": -169.7987518310547,
      "KL/std": 80.725830078125,
      "epoch": 0.4595616024187453,
      "fcm_dpo/beta": 0.006954976357519627,
      "fcm_dpo/delta": -0.033397287130355835,
      "fcm_dpo/margin": 63.942317962646484,
      "fcm_dpo/q_t": 0.3968748450279236,
      "grad_norm": 11.252235412597656,
      "learning_rate": 3.292634667444117e-07,
      "logits/chosen": 0.7025733590126038,
      "logits/rejected": 0.6516116261482239,
      "logps/chosen": -166.551513671875,
      "logps/ref_chosen": -60.695091247558594,
      "logps/ref_rejected": -78.2525405883789,
      "logps/rejected": -248.05130004882812,
      "loss": 1.055,
      "margin_dpo/margin_mean": 63.942317962646484,
      "margin_dpo/margin_std": 75.22644805908203,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -121.06056213378906,
      "KL/mean": -141.9627685546875,
      "KL/rejected_KL_mean": -162.86495971679688,
      "KL/std": 72.43345642089844,
      "epoch": 0.46107331821617537,
      "fcm_dpo/beta": 0.006954976357519627,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 41.804405212402344,
      "fcm_dpo/q_t": 0.43231460452079773,
      "grad_norm": 14.43248176574707,
      "learning_rate": 3.280083614246217e-07,
      "logits/chosen": 0.6767128705978394,
      "logits/rejected": 0.7134938836097717,
      "logps/chosen": -193.75970458984375,
      "logps/ref_chosen": -72.69914245605469,
      "logps/ref_rejected": -65.65670776367188,
      "logps/rejected": -228.52166748046875,
      "loss": 1.2077,
      "margin_dpo/margin_mean": 41.804405212402344,
      "margin_dpo/margin_std": 87.34265899658203,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -112.66262817382812,
      "KL/mean": -136.23704528808594,
      "KL/rejected_KL_mean": -159.81146240234375,
      "KL/std": 70.7812728881836,
      "epoch": 0.46258503401360546,
      "fcm_dpo/beta": 0.006920297164469957,
      "fcm_dpo/delta": -0.024993740022182465,
      "fcm_dpo/margin": 47.14885330200195,
      "fcm_dpo/q_t": 0.42409563064575195,
      "grad_norm": 11.079917907714844,
      "learning_rate": 3.267510740432719e-07,
      "logits/chosen": 0.7989003658294678,
      "logits/rejected": 0.6930177211761475,
      "logps/chosen": -166.63314819335938,
      "logps/ref_chosen": -53.97052764892578,
      "logps/ref_rejected": -71.02423095703125,
      "logps/rejected": -230.835693359375,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 47.14885330200195,
      "margin_dpo/margin_std": 72.32952117919922,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -109.00520324707031,
      "KL/mean": -120.71246337890625,
      "KL/rejected_KL_mean": -132.4197235107422,
      "KL/std": 71.84052276611328,
      "epoch": 0.46409674981103555,
      "fcm_dpo/beta": 0.006920297164469957,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 23.41451644897461,
      "fcm_dpo/q_t": 0.4631454348564148,
      "grad_norm": 17.977399826049805,
      "learning_rate": 3.2549163976939285e-07,
      "logits/chosen": 0.7276726961135864,
      "logits/rejected": 0.6820414066314697,
      "logps/chosen": -166.41830444335938,
      "logps/ref_chosen": -57.413108825683594,
      "logps/ref_rejected": -68.68010711669922,
      "logps/rejected": -201.09982299804688,
      "loss": 1.326,
      "margin_dpo/margin_mean": 23.41451644897461,
      "margin_dpo/margin_std": 90.96375274658203,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -110.5123291015625,
      "KL/mean": -133.32980346679688,
      "KL/rejected_KL_mean": -156.14727783203125,
      "KL/std": 74.56875610351562,
      "epoch": 0.4656084656084656,
      "fcm_dpo/beta": 0.006920297164469957,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.63495635986328,
      "fcm_dpo/q_t": 0.42745688557624817,
      "grad_norm": 10.50518798828125,
      "learning_rate": 3.2423009383206874e-07,
      "logits/chosen": 0.6881338953971863,
      "logits/rejected": 0.6755229234695435,
      "logps/chosen": -177.11111450195312,
      "logps/ref_chosen": -66.59879302978516,
      "logps/ref_rejected": -74.337158203125,
      "logps/rejected": -230.48443603515625,
      "loss": 1.1838,
      "margin_dpo/margin_mean": 45.63495635986328,
      "margin_dpo/margin_std": 87.10369873046875,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -123.3994140625,
      "KL/mean": -147.40701293945312,
      "KL/rejected_KL_mean": -171.41461181640625,
      "KL/std": 72.49065399169922,
      "epoch": 0.4671201814058957,
      "fcm_dpo/beta": 0.006920297164469957,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.01519012451172,
      "fcm_dpo/q_t": 0.424016535282135,
      "grad_norm": 11.415412902832031,
      "learning_rate": 3.229664715194511e-07,
      "logits/chosen": 0.7687605619430542,
      "logits/rejected": 0.7142656445503235,
      "logps/chosen": -188.79415893554688,
      "logps/ref_chosen": -65.39474487304688,
      "logps/ref_rejected": -75.70930480957031,
      "logps/rejected": -247.12391662597656,
      "loss": 1.1677,
      "margin_dpo/margin_mean": 48.01519012451172,
      "margin_dpo/margin_std": 85.55278015136719,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -130.60060119628906,
      "KL/mean": -144.18211364746094,
      "KL/rejected_KL_mean": -157.76361083984375,
      "KL/std": 66.98379516601562,
      "epoch": 0.46863189720332576,
      "fcm_dpo/beta": 0.006920297164469957,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 27.163013458251953,
      "fcm_dpo/q_t": 0.4576417803764343,
      "grad_norm": 12.827436447143555,
      "learning_rate": 3.2170080817777257e-07,
      "logits/chosen": 0.7497519254684448,
      "logits/rejected": 0.7340184450149536,
      "logps/chosen": -205.26889038085938,
      "logps/ref_chosen": -74.66827392578125,
      "logps/ref_rejected": -80.5689697265625,
      "logps/rejected": -238.33258056640625,
      "loss": 1.2808,
      "margin_dpo/margin_mean": 27.163015365600586,
      "margin_dpo/margin_std": 79.86006164550781,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -102.02139282226562,
      "KL/mean": -128.64895629882812,
      "KL/rejected_KL_mean": -155.27651977539062,
      "KL/std": 84.87614440917969,
      "epoch": 0.47014361300075586,
      "fcm_dpo/beta": 0.006893772166222334,
      "fcm_dpo/delta": -0.019201569259166718,
      "fcm_dpo/margin": 53.25511169433594,
      "fcm_dpo/q_t": 0.4192228317260742,
      "grad_norm": 11.669011116027832,
      "learning_rate": 3.204331392103574e-07,
      "logits/chosen": 0.6330962777137756,
      "logits/rejected": 0.49181753396987915,
      "logps/chosen": -161.75942993164062,
      "logps/ref_chosen": -59.738033294677734,
      "logps/ref_rejected": -93.60757446289062,
      "logps/rejected": -248.88409423828125,
      "loss": 1.1386,
      "margin_dpo/margin_mean": 53.25511169433594,
      "margin_dpo/margin_std": 89.1590576171875,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -105.78706359863281,
      "KL/mean": -135.12808227539062,
      "KL/rejected_KL_mean": -164.46908569335938,
      "KL/std": 74.64009094238281,
      "epoch": 0.47165532879818595,
      "fcm_dpo/beta": 0.006878282874822617,
      "fcm_dpo/delta": -0.02251959592103958,
      "fcm_dpo/margin": 58.682029724121094,
      "fcm_dpo/q_t": 0.40607959032058716,
      "grad_norm": 12.168621063232422,
      "learning_rate": 3.1916350007663176e-07,
      "logits/chosen": 0.7204309701919556,
      "logits/rejected": 0.6345335245132446,
      "logps/chosen": -159.60350036621094,
      "logps/ref_chosen": -53.816436767578125,
      "logps/ref_rejected": -68.6575698852539,
      "logps/rejected": -233.12664794921875,
      "loss": 1.0835,
      "margin_dpo/margin_mean": 58.682029724121094,
      "margin_dpo/margin_std": 73.37113952636719,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -112.03815460205078,
      "KL/mean": -127.57806396484375,
      "KL/rejected_KL_mean": -143.11798095703125,
      "KL/std": 73.82379150390625,
      "epoch": 0.47316704459561604,
      "fcm_dpo/beta": 0.006862793117761612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.079830169677734,
      "fcm_dpo/q_t": 0.45053255558013916,
      "grad_norm": 13.539533615112305,
      "learning_rate": 3.178919262911314e-07,
      "logits/chosen": 0.7814310789108276,
      "logits/rejected": 0.7609319686889648,
      "logps/chosen": -171.99551391601562,
      "logps/ref_chosen": -59.957359313964844,
      "logps/ref_rejected": -69.31729888916016,
      "logps/rejected": -212.43527221679688,
      "loss": 1.2604,
      "margin_dpo/margin_mean": 31.07982635498047,
      "margin_dpo/margin_std": 82.09465026855469,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -102.45433807373047,
      "KL/mean": -132.56947326660156,
      "KL/rejected_KL_mean": -162.68460083007812,
      "KL/std": 69.83338928222656,
      "epoch": 0.47467876039304613,
      "fcm_dpo/beta": 0.006821155082434416,
      "fcm_dpo/delta": -0.030428625643253326,
      "fcm_dpo/margin": 60.230255126953125,
      "fcm_dpo/q_t": 0.4053559899330139,
      "grad_norm": 12.13412094116211,
      "learning_rate": 3.166184534225087e-07,
      "logits/chosen": 0.7317984104156494,
      "logits/rejected": 0.7632255554199219,
      "logps/chosen": -172.72250366210938,
      "logps/ref_chosen": -70.26815795898438,
      "logps/ref_rejected": -69.23971557617188,
      "logps/rejected": -231.92431640625,
      "loss": 1.082,
      "margin_dpo/margin_mean": 60.23025131225586,
      "margin_dpo/margin_std": 75.92375183105469,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -108.26924896240234,
      "KL/mean": -130.59422302246094,
      "KL/rejected_KL_mean": -152.91920471191406,
      "KL/std": 71.26750183105469,
      "epoch": 0.47619047619047616,
      "fcm_dpo/beta": 0.006821155082434416,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 44.64996337890625,
      "fcm_dpo/q_t": 0.428594708442688,
      "grad_norm": 11.18031120300293,
      "learning_rate": 3.1534311709253723e-07,
      "logits/chosen": 0.6409457921981812,
      "logits/rejected": 0.6099203824996948,
      "logps/chosen": -176.06393432617188,
      "logps/ref_chosen": -67.79469299316406,
      "logps/ref_rejected": -74.55148315429688,
      "logps/rejected": -227.47068786621094,
      "loss": 1.1727,
      "margin_dpo/margin_mean": 44.64996337890625,
      "margin_dpo/margin_std": 77.41693115234375,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -101.49205017089844,
      "KL/mean": -135.0659942626953,
      "KL/rejected_KL_mean": -168.63992309570312,
      "KL/std": 76.8759765625,
      "epoch": 0.47770219198790626,
      "fcm_dpo/beta": 0.006735288072377443,
      "fcm_dpo/delta": -0.12749513983726501,
      "fcm_dpo/margin": 67.14787292480469,
      "fcm_dpo/q_t": 0.39548853039741516,
      "grad_norm": 11.79714298248291,
      "learning_rate": 3.1406595297511564e-07,
      "logits/chosen": 0.617354691028595,
      "logits/rejected": 0.480925977230072,
      "logps/chosen": -156.78053283691406,
      "logps/ref_chosen": -55.288482666015625,
      "logps/ref_rejected": -96.15723419189453,
      "logps/rejected": -264.79718017578125,
      "loss": 1.0558,
      "margin_dpo/margin_mean": 67.14787292480469,
      "margin_dpo/margin_std": 76.20933532714844,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -91.71333312988281,
      "KL/mean": -122.70863342285156,
      "KL/rejected_KL_mean": -153.70391845703125,
      "KL/std": 68.813720703125,
      "epoch": 0.47921390778533635,
      "fcm_dpo/beta": 0.006600792054086924,
      "fcm_dpo/delta": -0.03670065850019455,
      "fcm_dpo/margin": 61.99058532714844,
      "fcm_dpo/q_t": 0.4032962918281555,
      "grad_norm": 11.325054168701172,
      "learning_rate": 3.1278699679526975e-07,
      "logits/chosen": 0.7354048490524292,
      "logits/rejected": 0.6916720271110535,
      "logps/chosen": -146.29470825195312,
      "logps/ref_chosen": -54.58137512207031,
      "logps/ref_rejected": -72.77232360839844,
      "logps/rejected": -226.47625732421875,
      "loss": 1.0793,
      "margin_dpo/margin_mean": 61.99058532714844,
      "margin_dpo/margin_std": 76.21528625488281,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -103.02191162109375,
      "KL/mean": -125.77468872070312,
      "KL/rejected_KL_mean": -148.5274658203125,
      "KL/std": 74.2537841796875,
      "epoch": 0.48072562358276644,
      "fcm_dpo/beta": 0.006600792054086924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.50556182861328,
      "fcm_dpo/q_t": 0.4332531690597534,
      "grad_norm": 11.361948013305664,
      "learning_rate": 3.1150628432815336e-07,
      "logits/chosen": 0.7207176685333252,
      "logits/rejected": 0.6528332233428955,
      "logps/chosen": -155.91014099121094,
      "logps/ref_chosen": -52.88822937011719,
      "logps/ref_rejected": -80.63988494873047,
      "logps/rejected": -229.1673583984375,
      "loss": 1.2132,
      "margin_dpo/margin_mean": 45.50556182861328,
      "margin_dpo/margin_std": 101.61717224121094,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -101.49418640136719,
      "KL/mean": -133.5969696044922,
      "KL/rejected_KL_mean": -165.69973754882812,
      "KL/std": 78.28202056884766,
      "epoch": 0.48223733938019653,
      "fcm_dpo/beta": 0.006600792054086924,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.20555877685547,
      "fcm_dpo/q_t": 0.40408360958099365,
      "grad_norm": 10.197602272033691,
      "learning_rate": 3.1022385139804707e-07,
      "logits/chosen": 0.6491215229034424,
      "logits/rejected": 0.6349847316741943,
      "logps/chosen": -165.85751342773438,
      "logps/ref_chosen": -64.36333465576172,
      "logps/ref_rejected": -79.47296142578125,
      "logps/rejected": -245.17269897460938,
      "loss": 1.0897,
      "margin_dpo/margin_mean": 64.20555877685547,
      "margin_dpo/margin_std": 90.96015930175781,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -91.69598388671875,
      "KL/mean": -118.007568359375,
      "KL/rejected_KL_mean": -144.3191680908203,
      "KL/std": 79.62506103515625,
      "epoch": 0.4837490551776266,
      "fcm_dpo/beta": 0.0065584946423769,
      "fcm_dpo/delta": -0.0644935593008995,
      "fcm_dpo/margin": 52.6231803894043,
      "fcm_dpo/q_t": 0.4222339987754822,
      "grad_norm": 11.063246726989746,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 0.6024397611618042,
      "logits/rejected": 0.5657603144645691,
      "logps/chosen": -141.25473022460938,
      "logps/ref_chosen": -49.558746337890625,
      "logps/ref_rejected": -71.23444366455078,
      "logps/rejected": -215.55360412597656,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 52.6231803894043,
      "margin_dpo/margin_std": 84.94557189941406,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -95.55155944824219,
      "KL/mean": -123.66300201416016,
      "KL/rejected_KL_mean": -151.77444458007812,
      "KL/std": 67.8209228515625,
      "epoch": 0.4852607709750567,
      "fcm_dpo/beta": 0.006455008406192064,
      "fcm_dpo/delta": -0.04717318341135979,
      "fcm_dpo/margin": 56.22286605834961,
      "fcm_dpo/q_t": 0.4145912230014801,
      "grad_norm": 15.530342102050781,
      "learning_rate": 3.0765396768561004e-07,
      "logits/chosen": 0.6982482671737671,
      "logits/rejected": 0.6813958883285522,
      "logps/chosen": -147.6368408203125,
      "logps/ref_chosen": -52.08526611328125,
      "logps/ref_rejected": -55.58674621582031,
      "logps/rejected": -207.36117553710938,
      "loss": 1.1252,
      "margin_dpo/margin_mean": 56.222869873046875,
      "margin_dpo/margin_std": 80.80537414550781,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -109.41558837890625,
      "KL/mean": -143.95343017578125,
      "KL/rejected_KL_mean": -178.49130249023438,
      "KL/std": 77.21576690673828,
      "epoch": 0.48677248677248675,
      "fcm_dpo/beta": 0.006419917568564415,
      "fcm_dpo/delta": -0.05466010794043541,
      "fcm_dpo/margin": 69.07571411132812,
      "fcm_dpo/q_t": 0.3954285979270935,
      "grad_norm": 9.539613723754883,
      "learning_rate": 3.063665887884511e-07,
      "logits/chosen": 0.7315066456794739,
      "logits/rejected": 0.6500815749168396,
      "logps/chosen": -156.8197021484375,
      "logps/ref_chosen": -47.404109954833984,
      "logps/ref_rejected": -73.4260025024414,
      "logps/rejected": -251.91729736328125,
      "loss": 1.0462,
      "margin_dpo/margin_mean": 69.07571411132812,
      "margin_dpo/margin_std": 74.04450988769531,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -110.42620849609375,
      "KL/mean": -132.9568634033203,
      "KL/rejected_KL_mean": -155.48751831054688,
      "KL/std": 73.69956970214844,
      "epoch": 0.48828420256991684,
      "fcm_dpo/beta": 0.006384826265275478,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.061317443847656,
      "fcm_dpo/q_t": 0.43496036529541016,
      "grad_norm": 12.915155410766602,
      "learning_rate": 3.0507763319663517e-07,
      "logits/chosen": 0.6116001605987549,
      "logits/rejected": 0.5418746471405029,
      "logps/chosen": -180.4324951171875,
      "logps/ref_chosen": -70.00630187988281,
      "logps/ref_rejected": -86.96690368652344,
      "logps/rejected": -242.4544219970703,
      "loss": 1.2044,
      "margin_dpo/margin_mean": 45.061317443847656,
      "margin_dpo/margin_std": 94.1461181640625,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -98.17274475097656,
      "KL/mean": -127.4241943359375,
      "KL/rejected_KL_mean": -156.67564392089844,
      "KL/std": 83.87608337402344,
      "epoch": 0.4897959183673469,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": -0.02678520791232586,
      "fcm_dpo/margin": 58.502906799316406,
      "fcm_dpo/q_t": 0.41296952962875366,
      "grad_norm": 14.295312881469727,
      "learning_rate": 3.0378713696502097e-07,
      "logits/chosen": 0.7219698429107666,
      "logits/rejected": 0.665217399597168,
      "logps/chosen": -154.06155395507812,
      "logps/ref_chosen": -55.88882064819336,
      "logps/ref_rejected": -75.23088073730469,
      "logps/rejected": -231.90652465820312,
      "loss": 1.1068,
      "margin_dpo/margin_mean": 58.502906799316406,
      "margin_dpo/margin_std": 76.33798217773438,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -118.92147827148438,
      "KL/mean": -148.49241638183594,
      "KL/rejected_KL_mean": -178.0633544921875,
      "KL/std": 82.98800659179688,
      "epoch": 0.491307634164777,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.141883850097656,
      "fcm_dpo/q_t": 0.41542884707450867,
      "grad_norm": 10.648490905761719,
      "learning_rate": 3.0249513619156206e-07,
      "logits/chosen": 0.6718249320983887,
      "logits/rejected": 0.6107696294784546,
      "logps/chosen": -183.06849670410156,
      "logps/ref_chosen": -64.14701843261719,
      "logps/ref_rejected": -79.91143798828125,
      "logps/rejected": -257.97479248046875,
      "loss": 1.1377,
      "margin_dpo/margin_mean": 59.141883850097656,
      "margin_dpo/margin_std": 100.32028198242188,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -140.14584350585938,
      "KL/mean": -151.37425231933594,
      "KL/rejected_KL_mean": -162.6026611328125,
      "KL/std": 75.11209869384766,
      "epoch": 0.4928193499622071,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 22.45681381225586,
      "fcm_dpo/q_t": 0.46714678406715393,
      "grad_norm": 11.448811531066895,
      "learning_rate": 3.012016670162977e-07,
      "logits/chosen": 0.6767762899398804,
      "logits/rejected": 0.6858989596366882,
      "logps/chosen": -215.67715454101562,
      "logps/ref_chosen": -75.53131103515625,
      "logps/ref_rejected": -76.5898666381836,
      "logps/rejected": -239.19252014160156,
      "loss": 1.3198,
      "margin_dpo/margin_mean": 22.45681381225586,
      "margin_dpo/margin_std": 85.329833984375,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -123.46060180664062,
      "KL/mean": -146.76063537597656,
      "KL/rejected_KL_mean": -170.06063842773438,
      "KL/std": 80.32379913330078,
      "epoch": 0.4943310657596372,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.600040435791016,
      "fcm_dpo/q_t": 0.4317762851715088,
      "grad_norm": 11.476740837097168,
      "learning_rate": 2.99906765620341e-07,
      "logits/chosen": 0.6259051561355591,
      "logits/rejected": 0.6025689840316772,
      "logps/chosen": -192.7977752685547,
      "logps/ref_chosen": -69.33717346191406,
      "logps/ref_rejected": -73.37751770019531,
      "logps/rejected": -243.4381561279297,
      "loss": 1.1923,
      "margin_dpo/margin_mean": 46.60003662109375,
      "margin_dpo/margin_std": 91.36933135986328,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -108.52227783203125,
      "KL/mean": -137.43905639648438,
      "KL/rejected_KL_mean": -166.35581970214844,
      "KL/std": 80.34719848632812,
      "epoch": 0.4958427815570673,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.833534240722656,
      "fcm_dpo/q_t": 0.4146382510662079,
      "grad_norm": 10.102066993713379,
      "learning_rate": 2.9861046822486766e-07,
      "logits/chosen": 0.5871948003768921,
      "logits/rejected": 0.5468528270721436,
      "logps/chosen": -170.228515625,
      "logps/ref_chosen": -61.70623016357422,
      "logps/ref_rejected": -83.73808288574219,
      "logps/rejected": -250.09390258789062,
      "loss": 1.1094,
      "margin_dpo/margin_mean": 57.833534240722656,
      "margin_dpo/margin_std": 76.56809997558594,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -120.30358123779297,
      "KL/mean": -149.27682495117188,
      "KL/rejected_KL_mean": -178.25009155273438,
      "KL/std": 86.12091064453125,
      "epoch": 0.4973544973544973,
      "fcm_dpo/beta": 0.006350713782012463,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.946502685546875,
      "fcm_dpo/q_t": 0.41649869084358215,
      "grad_norm": 12.654784202575684,
      "learning_rate": 2.9731281109010253e-07,
      "logits/chosen": 0.729865550994873,
      "logits/rejected": 0.6701647043228149,
      "logps/chosen": -184.802001953125,
      "logps/ref_chosen": -64.4984130859375,
      "logps/ref_rejected": -83.6591796875,
      "logps/rejected": -261.9092712402344,
      "loss": 1.1267,
      "margin_dpo/margin_mean": 57.94650650024414,
      "margin_dpo/margin_std": 89.17796325683594,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -103.599609375,
      "KL/mean": -132.35592651367188,
      "KL/rejected_KL_mean": -161.11224365234375,
      "KL/std": 77.50123596191406,
      "epoch": 0.4988662131519274,
      "fcm_dpo/beta": 0.006339719519019127,
      "fcm_dpo/delta": -0.01734209805727005,
      "fcm_dpo/margin": 57.51264190673828,
      "fcm_dpo/q_t": 0.416418194770813,
      "grad_norm": 11.395146369934082,
      "learning_rate": 2.9601383051430505e-07,
      "logits/chosen": 0.7071419954299927,
      "logits/rejected": 0.6409908533096313,
      "logps/chosen": -158.4042510986328,
      "logps/ref_chosen": -54.80464172363281,
      "logps/ref_rejected": -75.3194351196289,
      "logps/rejected": -236.43167114257812,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 57.51264572143555,
      "margin_dpo/margin_std": 98.8066635131836,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -112.81170654296875,
      "KL/mean": -148.69761657714844,
      "KL/rejected_KL_mean": -184.58352661132812,
      "KL/std": 81.27599334716797,
      "epoch": 0.5003779289493575,
      "fcm_dpo/beta": 0.006279571447521448,
      "fcm_dpo/delta": -0.053083788603544235,
      "fcm_dpo/margin": 71.77181243896484,
      "fcm_dpo/q_t": 0.39528989791870117,
      "grad_norm": 11.644200325012207,
      "learning_rate": 2.947135628327544e-07,
      "logits/chosen": 0.7728036642074585,
      "logits/rejected": 0.7491520643234253,
      "logps/chosen": -172.05429077148438,
      "logps/ref_chosen": -59.242584228515625,
      "logps/ref_rejected": -69.87483215332031,
      "logps/rejected": -254.45835876464844,
      "loss": 1.0565,
      "margin_dpo/margin_mean": 71.77180480957031,
      "margin_dpo/margin_std": 87.65985107421875,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -117.23881530761719,
      "KL/mean": -147.25274658203125,
      "KL/rejected_KL_mean": -177.26669311523438,
      "KL/std": 79.46280670166016,
      "epoch": 0.5018896447467877,
      "fcm_dpo/beta": 0.00623284000903368,
      "fcm_dpo/delta": -0.04660751298069954,
      "fcm_dpo/margin": 60.02788543701172,
      "fcm_dpo/q_t": 0.41367125511169434,
      "grad_norm": 10.473052978515625,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": 0.6207424402236938,
      "logits/rejected": 0.5803670883178711,
      "logps/chosen": -184.34857177734375,
      "logps/ref_chosen": -67.10975646972656,
      "logps/ref_rejected": -77.11839294433594,
      "logps/rejected": -254.3850860595703,
      "loss": 1.1111,
      "margin_dpo/margin_mean": 60.02788162231445,
      "margin_dpo/margin_std": 81.86831665039062,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -125.28157043457031,
      "KL/mean": -159.71221923828125,
      "KL/rejected_KL_mean": -194.1428680419922,
      "KL/std": 81.67819213867188,
      "epoch": 0.5034013605442177,
      "fcm_dpo/beta": 0.006189778447151184,
      "fcm_dpo/delta": -0.022637760266661644,
      "fcm_dpo/margin": 68.86128234863281,
      "fcm_dpo/q_t": 0.4018930196762085,
      "grad_norm": 9.133350372314453,
      "learning_rate": 2.921093116725076e-07,
      "logits/chosen": 0.712902307510376,
      "logits/rejected": 0.6342633962631226,
      "logps/chosen": -183.66270446777344,
      "logps/ref_chosen": -58.381134033203125,
      "logps/ref_rejected": -85.02839660644531,
      "logps/rejected": -279.1712646484375,
      "loss": 1.0697,
      "margin_dpo/margin_mean": 68.86128234863281,
      "margin_dpo/margin_std": 85.3521728515625,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -119.81755828857422,
      "KL/mean": -144.76309204101562,
      "KL/rejected_KL_mean": -169.7086181640625,
      "KL/std": 80.81599426269531,
      "epoch": 0.5049130763416477,
      "fcm_dpo/beta": 0.006175766233354807,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 49.89105987548828,
      "fcm_dpo/q_t": 0.4292786717414856,
      "grad_norm": 12.244359016418457,
      "learning_rate": 2.9080540104031484e-07,
      "logits/chosen": 0.7428555488586426,
      "logits/rejected": 0.6925406455993652,
      "logps/chosen": -186.70956420898438,
      "logps/ref_chosen": -66.89199829101562,
      "logps/ref_rejected": -91.83695220947266,
      "logps/rejected": -261.5455627441406,
      "loss": 1.1793,
      "margin_dpo/margin_mean": 49.89106369018555,
      "margin_dpo/margin_std": 92.7274169921875,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -120.25526428222656,
      "KL/mean": -147.18856811523438,
      "KL/rejected_KL_mean": -174.12188720703125,
      "KL/std": 80.76295471191406,
      "epoch": 0.5064247921390779,
      "fcm_dpo/beta": 0.006175766233354807,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 53.86663055419922,
      "fcm_dpo/q_t": 0.4242987036705017,
      "grad_norm": 14.082091331481934,
      "learning_rate": 2.895003489933375e-07,
      "logits/chosen": 0.7074752449989319,
      "logits/rejected": 0.6703487634658813,
      "logps/chosen": -181.76971435546875,
      "logps/ref_chosen": -61.51445770263672,
      "logps/ref_rejected": -75.68916320800781,
      "logps/rejected": -249.81105041503906,
      "loss": 1.1612,
      "margin_dpo/margin_mean": 53.86663055419922,
      "margin_dpo/margin_std": 93.7869873046875,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -135.12388610839844,
      "KL/mean": -164.06655883789062,
      "KL/rejected_KL_mean": -193.00921630859375,
      "KL/std": 87.16038513183594,
      "epoch": 0.5079365079365079,
      "fcm_dpo/beta": 0.006122264079749584,
      "fcm_dpo/delta": -0.043504975736141205,
      "fcm_dpo/margin": 57.88533401489258,
      "fcm_dpo/q_t": 0.4214656949043274,
      "grad_norm": 11.543611526489258,
      "learning_rate": 2.8819419203668675e-07,
      "logits/chosen": 0.6747065186500549,
      "logits/rejected": 0.6493879556655884,
      "logps/chosen": -203.9739532470703,
      "logps/ref_chosen": -68.85006713867188,
      "logps/ref_rejected": -92.99603271484375,
      "logps/rejected": -286.0052490234375,
      "loss": 1.1491,
      "margin_dpo/margin_mean": 57.88533401489258,
      "margin_dpo/margin_std": 97.946044921875,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -143.09075927734375,
      "KL/mean": -166.30104064941406,
      "KL/rejected_KL_mean": -189.51132202148438,
      "KL/std": 86.70985412597656,
      "epoch": 0.509448223733938,
      "fcm_dpo/beta": 0.006122264079749584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.420562744140625,
      "fcm_dpo/q_t": 0.4345945119857788,
      "grad_norm": 10.347283363342285,
      "learning_rate": 2.8688696670638053e-07,
      "logits/chosen": 0.596039891242981,
      "logits/rejected": 0.568123459815979,
      "logps/chosen": -216.27859497070312,
      "logps/ref_chosen": -73.18783569335938,
      "logps/ref_rejected": -86.89118957519531,
      "logps/rejected": -276.40252685546875,
      "loss": 1.1964,
      "margin_dpo/margin_mean": 46.420562744140625,
      "margin_dpo/margin_std": 91.3853759765625,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -139.84075927734375,
      "KL/mean": -164.1468505859375,
      "KL/rejected_KL_mean": -188.4529266357422,
      "KL/std": 78.78346252441406,
      "epoch": 0.5109599395313681,
      "fcm_dpo/beta": 0.006122264079749584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.612159729003906,
      "fcm_dpo/q_t": 0.4303521513938904,
      "grad_norm": 11.543200492858887,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 0.6580469608306885,
      "logits/rejected": 0.6333959102630615,
      "logps/chosen": -203.78038024902344,
      "logps/ref_chosen": -63.939613342285156,
      "logps/ref_rejected": -75.34243774414062,
      "logps/rejected": -263.79534912109375,
      "loss": 1.1781,
      "margin_dpo/margin_mean": 48.612159729003906,
      "margin_dpo/margin_std": 86.77371215820312,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -114.18165588378906,
      "KL/mean": -143.21810913085938,
      "KL/rejected_KL_mean": -172.2545928955078,
      "KL/std": 81.62635803222656,
      "epoch": 0.5124716553287982,
      "fcm_dpo/beta": 0.006122264079749584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.07294464111328,
      "fcm_dpo/q_t": 0.41747236251831055,
      "grad_norm": 12.970744132995605,
      "learning_rate": 2.842694572172736e-07,
      "logits/chosen": 0.8276938199996948,
      "logits/rejected": 0.7435949444770813,
      "logps/chosen": -159.73077392578125,
      "logps/ref_chosen": -45.54913330078125,
      "logps/ref_rejected": -67.0482177734375,
      "logps/rejected": -239.3028106689453,
      "loss": 1.1311,
      "margin_dpo/margin_mean": 58.07294464111328,
      "margin_dpo/margin_std": 86.94050598144531,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -130.46449279785156,
      "KL/mean": -157.4834747314453,
      "KL/rejected_KL_mean": -184.50247192382812,
      "KL/std": 86.1899642944336,
      "epoch": 0.5139833711262283,
      "fcm_dpo/beta": 0.006122264079749584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.03797149658203,
      "fcm_dpo/q_t": 0.42464643716812134,
      "grad_norm": 11.059535026550293,
      "learning_rate": 2.8295924627584004e-07,
      "logits/chosen": 0.7042380571365356,
      "logits/rejected": 0.6846690773963928,
      "logps/chosen": -184.4701385498047,
      "logps/ref_chosen": -54.00564956665039,
      "logps/ref_rejected": -61.314430236816406,
      "logps/rejected": -245.81689453125,
      "loss": 1.1738,
      "margin_dpo/margin_mean": 54.03797149658203,
      "margin_dpo/margin_std": 100.92936706542969,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -133.33224487304688,
      "KL/mean": -164.9116668701172,
      "KL/rejected_KL_mean": -196.4910888671875,
      "KL/std": 87.80421447753906,
      "epoch": 0.5154950869236583,
      "fcm_dpo/beta": 0.006004424300044775,
      "fcm_dpo/delta": -0.0971769466996193,
      "fcm_dpo/margin": 63.15885925292969,
      "fcm_dpo/q_t": 0.4124714434146881,
      "grad_norm": 12.540910720825195,
      "learning_rate": 2.816481133934373e-07,
      "logits/chosen": 0.724452555179596,
      "logits/rejected": 0.6779919266700745,
      "logps/chosen": -196.72732543945312,
      "logps/ref_chosen": -63.39509582519531,
      "logps/ref_rejected": -76.20973205566406,
      "logps/rejected": -272.7008056640625,
      "loss": 1.1267,
      "margin_dpo/margin_mean": 63.158851623535156,
      "margin_dpo/margin_std": 93.57270812988281,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -130.59140014648438,
      "KL/mean": -160.70944213867188,
      "KL/rejected_KL_mean": -190.82748413085938,
      "KL/std": 92.62532043457031,
      "epoch": 0.5170068027210885,
      "fcm_dpo/beta": 0.006004424300044775,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.236087799072266,
      "fcm_dpo/q_t": 0.41768157482147217,
      "grad_norm": 9.833003044128418,
      "learning_rate": 2.8033609524527046e-07,
      "logits/chosen": 0.7610205411911011,
      "logits/rejected": 0.7204849123954773,
      "logps/chosen": -183.6392059326172,
      "logps/ref_chosen": -53.047813415527344,
      "logps/ref_rejected": -68.2854232788086,
      "logps/rejected": -259.1129150390625,
      "loss": 1.1352,
      "margin_dpo/margin_mean": 60.236083984375,
      "margin_dpo/margin_std": 95.39666748046875,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -125.97956085205078,
      "KL/mean": -149.21401977539062,
      "KL/rejected_KL_mean": -172.448486328125,
      "KL/std": 85.8572006225586,
      "epoch": 0.5185185185185185,
      "fcm_dpo/beta": 0.006004424300044775,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.46892166137695,
      "fcm_dpo/q_t": 0.4357026219367981,
      "grad_norm": 9.35943603515625,
      "learning_rate": 2.7902322853130753e-07,
      "logits/chosen": 0.5916262865066528,
      "logits/rejected": 0.5824925899505615,
      "logps/chosen": -196.55809020996094,
      "logps/ref_chosen": -70.57852935791016,
      "logps/ref_rejected": -84.73873901367188,
      "logps/rejected": -257.1872253417969,
      "loss": 1.1976,
      "margin_dpo/margin_mean": 46.46892547607422,
      "margin_dpo/margin_std": 89.41838073730469,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -138.49673461914062,
      "KL/mean": -171.74130249023438,
      "KL/rejected_KL_mean": -204.98583984375,
      "KL/std": 87.63685607910156,
      "epoch": 0.5200302343159486,
      "fcm_dpo/beta": 0.006004424300044775,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.48912048339844,
      "fcm_dpo/q_t": 0.4096273183822632,
      "grad_norm": 11.909037590026855,
      "learning_rate": 2.7770954997525274e-07,
      "logits/chosen": 0.7216250896453857,
      "logits/rejected": 0.6522601842880249,
      "logps/chosen": -194.3077392578125,
      "logps/ref_chosen": -55.811004638671875,
      "logps/ref_rejected": -84.77637481689453,
      "logps/rejected": -289.76220703125,
      "loss": 1.1151,
      "margin_dpo/margin_mean": 66.48912048339844,
      "margin_dpo/margin_std": 102.74324798583984,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -114.75037384033203,
      "KL/mean": -144.64398193359375,
      "KL/rejected_KL_mean": -174.53756713867188,
      "KL/std": 80.88868713378906,
      "epoch": 0.5215419501133787,
      "fcm_dpo/beta": 0.006004424300044775,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.787208557128906,
      "fcm_dpo/q_t": 0.4173477292060852,
      "grad_norm": 11.631264686584473,
      "learning_rate": 2.7639509632351927e-07,
      "logits/chosen": 0.7710617780685425,
      "logits/rejected": 0.7289673686027527,
      "logps/chosen": -172.53646850585938,
      "logps/ref_chosen": -57.78609848022461,
      "logps/ref_rejected": -78.91847229003906,
      "logps/rejected": -253.4560546875,
      "loss": 1.1366,
      "margin_dpo/margin_mean": 59.787208557128906,
      "margin_dpo/margin_std": 95.0912094116211,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -120.35940551757812,
      "KL/mean": -156.9630126953125,
      "KL/rejected_KL_mean": -193.56661987304688,
      "KL/std": 94.83656311035156,
      "epoch": 0.5230536659108088,
      "fcm_dpo/beta": 0.005976714193820953,
      "fcm_dpo/delta": -0.04636390507221222,
      "fcm_dpo/margin": 73.20722198486328,
      "fcm_dpo/q_t": 0.3988511264324188,
      "grad_norm": 11.821297645568848,
      "learning_rate": 2.7507990434420123e-07,
      "logits/chosen": 0.7447773218154907,
      "logits/rejected": 0.6635010242462158,
      "logps/chosen": -176.64453125,
      "logps/ref_chosen": -56.285125732421875,
      "logps/ref_rejected": -91.15303039550781,
      "logps/rejected": -284.71966552734375,
      "loss": 1.0786,
      "margin_dpo/margin_mean": 73.20722961425781,
      "margin_dpo/margin_std": 97.56934356689453,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -133.1365966796875,
      "KL/mean": -161.14346313476562,
      "KL/rejected_KL_mean": -189.15032958984375,
      "KL/std": 87.99459838867188,
      "epoch": 0.5245653817082389,
      "fcm_dpo/beta": 0.005949004087597132,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.01373291015625,
      "fcm_dpo/q_t": 0.42295533418655396,
      "grad_norm": 10.870003700256348,
      "learning_rate": 2.737640108260456e-07,
      "logits/chosen": 0.8249450922012329,
      "logits/rejected": 0.7794303894042969,
      "logps/chosen": -186.63613891601562,
      "logps/ref_chosen": -53.499542236328125,
      "logps/ref_rejected": -72.52565002441406,
      "logps/rejected": -261.6759948730469,
      "loss": 1.146,
      "margin_dpo/margin_mean": 56.01373291015625,
      "margin_dpo/margin_std": 88.20159149169922,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -120.46863555908203,
      "KL/mean": -151.01657104492188,
      "KL/rejected_KL_mean": -181.5644989013672,
      "KL/std": 82.77992248535156,
      "epoch": 0.5260770975056689,
      "fcm_dpo/beta": 0.005949004087597132,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.095863342285156,
      "fcm_dpo/q_t": 0.4175592064857483,
      "grad_norm": 12.275740623474121,
      "learning_rate": 2.724474525774229e-07,
      "logits/chosen": 0.7986717224121094,
      "logits/rejected": 0.7687997817993164,
      "logps/chosen": -171.2554931640625,
      "logps/ref_chosen": -50.78684997558594,
      "logps/ref_rejected": -68.63732147216797,
      "logps/rejected": -250.20181274414062,
      "loss": 1.1344,
      "margin_dpo/margin_mean": 61.095863342285156,
      "margin_dpo/margin_std": 97.12753295898438,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -122.85971069335938,
      "KL/mean": -155.31509399414062,
      "KL/rejected_KL_mean": -187.77047729492188,
      "KL/std": 93.95545196533203,
      "epoch": 0.527588813303099,
      "fcm_dpo/beta": 0.005926626734435558,
      "fcm_dpo/delta": -0.03775714710354805,
      "fcm_dpo/margin": 64.9107666015625,
      "fcm_dpo/q_t": 0.41291543841362,
      "grad_norm": 10.789410591125488,
      "learning_rate": 2.711302664252973e-07,
      "logits/chosen": 0.8072630167007446,
      "logits/rejected": 0.7103030681610107,
      "logps/chosen": -176.18472290039062,
      "logps/ref_chosen": -53.325008392333984,
      "logps/ref_rejected": -83.21236419677734,
      "logps/rejected": -270.98284912109375,
      "loss": 1.1198,
      "margin_dpo/margin_mean": 64.9107666015625,
      "margin_dpo/margin_std": 98.733154296875,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -125.94990539550781,
      "KL/mean": -169.85733032226562,
      "KL/rejected_KL_mean": -213.7647705078125,
      "KL/std": 99.48991394042969,
      "epoch": 0.5291005291005291,
      "fcm_dpo/beta": 0.005794004071503878,
      "fcm_dpo/delta": -0.11490027606487274,
      "fcm_dpo/margin": 87.81488037109375,
      "fcm_dpo/q_t": 0.3824026882648468,
      "grad_norm": 13.445771217346191,
      "learning_rate": 2.698124892141971e-07,
      "logits/chosen": 0.7362804412841797,
      "logits/rejected": 0.6689400672912598,
      "logps/chosen": -187.57566833496094,
      "logps/ref_chosen": -61.625770568847656,
      "logps/ref_rejected": -87.63627624511719,
      "logps/rejected": -301.4010314941406,
      "loss": 1.0084,
      "margin_dpo/margin_mean": 87.81488037109375,
      "margin_dpo/margin_std": 93.23359680175781,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -121.92890930175781,
      "KL/mean": -152.43846130371094,
      "KL/rejected_KL_mean": -182.94802856445312,
      "KL/std": 83.26048278808594,
      "epoch": 0.5306122448979592,
      "fcm_dpo/beta": 0.0057701170444488525,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.019126892089844,
      "fcm_dpo/q_t": 0.41859257221221924,
      "grad_norm": 11.20499324798584,
      "learning_rate": 2.6849415780518357e-07,
      "logits/chosen": 0.690535306930542,
      "logits/rejected": 0.6209006905555725,
      "logps/chosen": -178.18524169921875,
      "logps/ref_chosen": -56.2563362121582,
      "logps/ref_rejected": -79.11589813232422,
      "logps/rejected": -262.06390380859375,
      "loss": 1.1553,
      "margin_dpo/margin_mean": 61.019126892089844,
      "margin_dpo/margin_std": 106.53633880615234,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -126.20106506347656,
      "KL/mean": -160.4630889892578,
      "KL/rejected_KL_mean": -194.72511291503906,
      "KL/std": 94.95686340332031,
      "epoch": 0.5321239606953893,
      "fcm_dpo/beta": 0.005735332146286964,
      "fcm_dpo/delta": -0.03023368865251541,
      "fcm_dpo/margin": 68.52404022216797,
      "fcm_dpo/q_t": 0.4111440181732178,
      "grad_norm": 10.544001579284668,
      "learning_rate": 2.6717530907482024e-07,
      "logits/chosen": 0.763157069683075,
      "logits/rejected": 0.7131781578063965,
      "logps/chosen": -189.25302124023438,
      "logps/ref_chosen": -63.05195236206055,
      "logps/ref_rejected": -85.52035522460938,
      "logps/rejected": -280.2454833984375,
      "loss": 1.1116,
      "margin_dpo/margin_mean": 68.52404022216797,
      "margin_dpo/margin_std": 102.7152099609375,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -124.33663940429688,
      "KL/mean": -155.11465454101562,
      "KL/rejected_KL_mean": -185.89266967773438,
      "KL/std": 85.38116455078125,
      "epoch": 0.5336356764928194,
      "fcm_dpo/beta": 0.005735332146286964,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.55604934692383,
      "fcm_dpo/q_t": 0.41740575432777405,
      "grad_norm": 11.076839447021484,
      "learning_rate": 2.658559799141411e-07,
      "logits/chosen": 0.7371421456336975,
      "logits/rejected": 0.7465870976448059,
      "logps/chosen": -193.3458251953125,
      "logps/ref_chosen": -69.00918579101562,
      "logps/ref_rejected": -72.65840148925781,
      "logps/rejected": -258.55108642578125,
      "loss": 1.1249,
      "margin_dpo/margin_mean": 61.556053161621094,
      "margin_dpo/margin_std": 87.54644012451172,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -121.45633697509766,
      "KL/mean": -158.8951873779297,
      "KL/rejected_KL_mean": -196.33404541015625,
      "KL/std": 80.35975646972656,
      "epoch": 0.5351473922902494,
      "fcm_dpo/beta": 0.005694113671779633,
      "fcm_dpo/delta": -0.036063361912965775,
      "fcm_dpo/margin": 74.8777084350586,
      "fcm_dpo/q_t": 0.40101897716522217,
      "grad_norm": 14.158194541931152,
      "learning_rate": 2.6453620722761895e-07,
      "logits/chosen": 0.8556749820709229,
      "logits/rejected": 0.7295167446136475,
      "logps/chosen": -161.2446746826172,
      "logps/ref_chosen": -39.78833770751953,
      "logps/ref_rejected": -69.56885528564453,
      "logps/rejected": -265.90289306640625,
      "loss": 1.0787,
      "margin_dpo/margin_mean": 74.87770080566406,
      "margin_dpo/margin_std": 97.76545715332031,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -127.52540588378906,
      "KL/mean": -169.996826171875,
      "KL/rejected_KL_mean": -212.46826171875,
      "KL/std": 97.32823181152344,
      "epoch": 0.5366591080876795,
      "fcm_dpo/beta": 0.005657237023115158,
      "fcm_dpo/delta": -0.06518547236919403,
      "fcm_dpo/margin": 84.94284057617188,
      "fcm_dpo/q_t": 0.39050179719924927,
      "grad_norm": 12.261187553405762,
      "learning_rate": 2.632160279321328e-07,
      "logits/chosen": 0.778215765953064,
      "logits/rejected": 0.65799880027771,
      "logps/chosen": -173.78079223632812,
      "logps/ref_chosen": -46.25537872314453,
      "logps/ref_rejected": -78.20236206054688,
      "logps/rejected": -290.67059326171875,
      "loss": 1.0487,
      "margin_dpo/margin_mean": 84.94284057617188,
      "margin_dpo/margin_std": 106.72587585449219,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -122.94532775878906,
      "KL/mean": -155.99130249023438,
      "KL/rejected_KL_mean": -189.03729248046875,
      "KL/std": 89.70805358886719,
      "epoch": 0.5381708238851096,
      "fcm_dpo/beta": 0.005620360840111971,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.09196472167969,
      "fcm_dpo/q_t": 0.4156641364097595,
      "grad_norm": 10.317228317260742,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 0.7800395488739014,
      "logits/rejected": 0.7022900581359863,
      "logps/chosen": -170.8514862060547,
      "logps/ref_chosen": -47.906158447265625,
      "logps/ref_rejected": -74.29397583007812,
      "logps/rejected": -263.3312683105469,
      "loss": 1.1421,
      "margin_dpo/margin_mean": 66.09196472167969,
      "margin_dpo/margin_std": 111.29681396484375,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -145.76248168945312,
      "KL/mean": -171.95567321777344,
      "KL/rejected_KL_mean": -198.1488800048828,
      "KL/std": 91.05963134765625,
      "epoch": 0.5396825396825397,
      "fcm_dpo/beta": 0.005574680399149656,
      "fcm_dpo/delta": -0.04080449417233467,
      "fcm_dpo/margin": 52.38639831542969,
      "fcm_dpo/q_t": 0.4316297769546509,
      "grad_norm": 11.417281150817871,
      "learning_rate": 2.6057459723762076e-07,
      "logits/chosen": 0.778508186340332,
      "logits/rejected": 0.7620171308517456,
      "logps/chosen": -208.39747619628906,
      "logps/ref_chosen": -62.63500213623047,
      "logps/ref_rejected": -65.11399841308594,
      "logps/rejected": -263.26287841796875,
      "loss": 1.1986,
      "margin_dpo/margin_mean": 52.38639831542969,
      "margin_dpo/margin_std": 100.94227600097656,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -145.61180114746094,
      "KL/mean": -179.46226501464844,
      "KL/rejected_KL_mean": -213.31271362304688,
      "KL/std": 97.12767028808594,
      "epoch": 0.5411942554799698,
      "fcm_dpo/beta": 0.005523651838302612,
      "fcm_dpo/delta": -0.0923849493265152,
      "fcm_dpo/margin": 67.7009048461914,
      "fcm_dpo/q_t": 0.415596067905426,
      "grad_norm": 16.69784164428711,
      "learning_rate": 2.5925341972508954e-07,
      "logits/chosen": 0.673900306224823,
      "logits/rejected": 0.6875008940696716,
      "logps/chosen": -212.8214111328125,
      "logps/ref_chosen": -67.20960998535156,
      "logps/ref_rejected": -69.34715270996094,
      "logps/rejected": -282.65985107421875,
      "loss": 1.1401,
      "margin_dpo/margin_mean": 67.70091247558594,
      "margin_dpo/margin_std": 108.23062896728516,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -161.5914306640625,
      "KL/mean": -180.28622436523438,
      "KL/rejected_KL_mean": -198.98104858398438,
      "KL/std": 95.16166687011719,
      "epoch": 0.5427059712773998,
      "fcm_dpo/beta": 0.005472622811794281,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.389610290527344,
      "fcm_dpo/q_t": 0.4529466927051544,
      "grad_norm": 13.044179916381836,
      "learning_rate": 2.579319833745169e-07,
      "logits/chosen": 0.74629145860672,
      "logits/rejected": 0.71300208568573,
      "logps/chosen": -224.11721801757812,
      "logps/ref_chosen": -62.52578353881836,
      "logps/ref_rejected": -76.63114929199219,
      "logps/rejected": -275.6121826171875,
      "loss": 1.276,
      "margin_dpo/margin_mean": 37.389610290527344,
      "margin_dpo/margin_std": 105.70441436767578,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -153.78167724609375,
      "KL/mean": -187.89193725585938,
      "KL/rejected_KL_mean": -222.002197265625,
      "KL/std": 100.96775817871094,
      "epoch": 0.54421768707483,
      "fcm_dpo/beta": 0.005472622811794281,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.22052001953125,
      "fcm_dpo/q_t": 0.41469377279281616,
      "grad_norm": 12.181764602661133,
      "learning_rate": 2.5661032514931834e-07,
      "logits/chosen": 0.7016806602478027,
      "logits/rejected": 0.618013858795166,
      "logps/chosen": -217.26937866210938,
      "logps/ref_chosen": -63.48772048950195,
      "logps/ref_rejected": -90.6891098022461,
      "logps/rejected": -312.6912841796875,
      "loss": 1.1184,
      "margin_dpo/margin_mean": 68.22052001953125,
      "margin_dpo/margin_std": 100.18309020996094,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -144.5960693359375,
      "KL/mean": -181.52798461914062,
      "KL/rejected_KL_mean": -218.45993041992188,
      "KL/std": 93.3544921875,
      "epoch": 0.54572940287226,
      "fcm_dpo/beta": 0.0054628076031804085,
      "fcm_dpo/delta": -0.017967604100704193,
      "fcm_dpo/margin": 73.8638687133789,
      "fcm_dpo/q_t": 0.4064033031463623,
      "grad_norm": 10.888641357421875,
      "learning_rate": 2.552884820191154e-07,
      "logits/chosen": 0.8262885808944702,
      "logits/rejected": 0.786286473274231,
      "logps/chosen": -202.51321411132812,
      "logps/ref_chosen": -57.917144775390625,
      "logps/ref_rejected": -72.39089965820312,
      "logps/rejected": -290.850830078125,
      "loss": 1.0897,
      "margin_dpo/margin_mean": 73.8638687133789,
      "margin_dpo/margin_std": 96.90747833251953,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -146.7781982421875,
      "KL/mean": -189.61180114746094,
      "KL/rejected_KL_mean": -232.44540405273438,
      "KL/std": 110.26019287109375,
      "epoch": 0.54724111866969,
      "fcm_dpo/beta": 0.005410117097198963,
      "fcm_dpo/delta": -0.07925108075141907,
      "fcm_dpo/margin": 85.66720581054688,
      "fcm_dpo/q_t": 0.3988887071609497,
      "grad_norm": 12.250260353088379,
      "learning_rate": 2.53966490958702e-07,
      "logits/chosen": 0.8019309043884277,
      "logits/rejected": 0.7083509564399719,
      "logps/chosen": -210.2216796875,
      "logps/ref_chosen": -63.4434700012207,
      "logps/ref_rejected": -103.45516967773438,
      "logps/rejected": -335.90057373046875,
      "loss": 1.0825,
      "margin_dpo/margin_mean": 85.66720581054688,
      "margin_dpo/margin_std": 129.54876708984375,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -149.93971252441406,
      "KL/mean": -188.626220703125,
      "KL/rejected_KL_mean": -227.31271362304688,
      "KL/std": 94.37256622314453,
      "epoch": 0.5487528344671202,
      "fcm_dpo/beta": 0.005367242265492678,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 77.3730239868164,
      "fcm_dpo/q_t": 0.40640437602996826,
      "grad_norm": 12.255396842956543,
      "learning_rate": 2.526443889470099e-07,
      "logits/chosen": 0.8176679611206055,
      "logits/rejected": 0.6928176879882812,
      "logps/chosen": -198.59153747558594,
      "logps/ref_chosen": -48.65182876586914,
      "logps/ref_rejected": -88.65904235839844,
      "logps/rejected": -315.9717712402344,
      "loss": 1.0943,
      "margin_dpo/margin_mean": 77.3730239868164,
      "margin_dpo/margin_std": 110.87226867675781,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -132.52880859375,
      "KL/mean": -176.2803497314453,
      "KL/rejected_KL_mean": -220.03187561035156,
      "KL/std": 104.11674499511719,
      "epoch": 0.5502645502645502,
      "fcm_dpo/beta": 0.005317722912877798,
      "fcm_dpo/delta": -0.0463450662791729,
      "fcm_dpo/margin": 87.50308227539062,
      "fcm_dpo/q_t": 0.3970663845539093,
      "grad_norm": 10.543391227722168,
      "learning_rate": 2.513222129660744e-07,
      "logits/chosen": 0.7445929050445557,
      "logits/rejected": 0.6653020977973938,
      "logps/chosen": -190.39987182617188,
      "logps/ref_chosen": -57.87107467651367,
      "logps/ref_rejected": -80.95503234863281,
      "logps/rejected": -300.9869079589844,
      "loss": 1.082,
      "margin_dpo/margin_mean": 87.50308227539062,
      "margin_dpo/margin_std": 129.79031372070312,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -126.51123046875,
      "KL/mean": -165.5495147705078,
      "KL/rejected_KL_mean": -204.58779907226562,
      "KL/std": 111.03411865234375,
      "epoch": 0.5517762660619804,
      "fcm_dpo/beta": 0.005268987733870745,
      "fcm_dpo/delta": -0.04603463038802147,
      "fcm_dpo/margin": 78.07655334472656,
      "fcm_dpo/q_t": 0.40557482838630676,
      "grad_norm": 10.389727592468262,
      "learning_rate": 2.5e-07,
      "logits/chosen": 0.7515792846679688,
      "logits/rejected": 0.7503241300582886,
      "logps/chosen": -191.45339965820312,
      "logps/ref_chosen": -64.94217681884766,
      "logps/ref_rejected": -74.8599853515625,
      "logps/rejected": -279.4477844238281,
      "loss": 1.0758,
      "margin_dpo/margin_mean": 78.07655334472656,
      "margin_dpo/margin_std": 97.56391906738281,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -125.47860717773438,
      "KL/mean": -157.73670959472656,
      "KL/rejected_KL_mean": -189.99481201171875,
      "KL/std": 97.81672668457031,
      "epoch": 0.5532879818594104,
      "fcm_dpo/beta": 0.005268987733870745,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.5162124633789,
      "fcm_dpo/q_t": 0.4233565330505371,
      "grad_norm": 14.451668739318848,
      "learning_rate": 2.486777870339255e-07,
      "logits/chosen": 0.7044786810874939,
      "logits/rejected": 0.7007114887237549,
      "logps/chosen": -180.64459228515625,
      "logps/ref_chosen": -55.16598129272461,
      "logps/ref_rejected": -65.26121520996094,
      "logps/rejected": -255.25601196289062,
      "loss": 1.1763,
      "margin_dpo/margin_mean": 64.5162124633789,
      "margin_dpo/margin_std": 124.49813842773438,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -144.489990234375,
      "KL/mean": -176.97052001953125,
      "KL/rejected_KL_mean": -209.45106506347656,
      "KL/std": 96.73851776123047,
      "epoch": 0.5547996976568406,
      "fcm_dpo/beta": 0.005268987733870745,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.96107482910156,
      "fcm_dpo/q_t": 0.4195973873138428,
      "grad_norm": 12.064628601074219,
      "learning_rate": 2.4735561105299014e-07,
      "logits/chosen": 0.7793039083480835,
      "logits/rejected": 0.6833162307739258,
      "logps/chosen": -200.50045776367188,
      "logps/ref_chosen": -56.01046371459961,
      "logps/ref_rejected": -77.31010437011719,
      "logps/rejected": -286.76116943359375,
      "loss": 1.1418,
      "margin_dpo/margin_mean": 64.96107482910156,
      "margin_dpo/margin_std": 101.67163848876953,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -155.61111450195312,
      "KL/mean": -184.02906799316406,
      "KL/rejected_KL_mean": -212.447021484375,
      "KL/std": 94.99668884277344,
      "epoch": 0.5563114134542706,
      "fcm_dpo/beta": 0.005268987733870745,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.83592224121094,
      "fcm_dpo/q_t": 0.43033045530319214,
      "grad_norm": 14.06124496459961,
      "learning_rate": 2.46033509041298e-07,
      "logits/chosen": 0.6116068959236145,
      "logits/rejected": 0.6141465306282043,
      "logps/chosen": -230.44039916992188,
      "logps/ref_chosen": -74.82927703857422,
      "logps/ref_rejected": -76.11680603027344,
      "logps/rejected": -288.5638427734375,
      "loss": 1.1803,
      "margin_dpo/margin_mean": 56.83592224121094,
      "margin_dpo/margin_std": 102.63706970214844,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -142.20578002929688,
      "KL/mean": -169.76429748535156,
      "KL/rejected_KL_mean": -197.32281494140625,
      "KL/std": 95.49687194824219,
      "epoch": 0.5578231292517006,
      "fcm_dpo/beta": 0.005268987733870745,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.11702346801758,
      "fcm_dpo/q_t": 0.43302732706069946,
      "grad_norm": 11.773307800292969,
      "learning_rate": 2.447115179808846e-07,
      "logits/chosen": 0.7620031833648682,
      "logits/rejected": 0.7175683379173279,
      "logps/chosen": -200.53199768066406,
      "logps/ref_chosen": -58.32621765136719,
      "logps/ref_rejected": -80.92183685302734,
      "logps/rejected": -278.2446594238281,
      "loss": 1.1847,
      "margin_dpo/margin_mean": 55.11701965332031,
      "margin_dpo/margin_std": 100.87909698486328,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -135.1417694091797,
      "KL/mean": -175.1524200439453,
      "KL/rejected_KL_mean": -215.16307067871094,
      "KL/std": 99.09980010986328,
      "epoch": 0.5593348450491308,
      "fcm_dpo/beta": 0.00522764865309,
      "fcm_dpo/delta": -0.03938331827521324,
      "fcm_dpo/margin": 80.02131652832031,
      "fcm_dpo/q_t": 0.40278932452201843,
      "grad_norm": 12.446624755859375,
      "learning_rate": 2.4338967485068164e-07,
      "logits/chosen": 0.8438280820846558,
      "logits/rejected": 0.7817610502243042,
      "logps/chosen": -188.02548217773438,
      "logps/ref_chosen": -52.88372039794922,
      "logps/ref_rejected": -79.43692016601562,
      "logps/rejected": -294.5999755859375,
      "loss": 1.0979,
      "margin_dpo/margin_mean": 80.02131652832031,
      "margin_dpo/margin_std": 115.83177185058594,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -131.9373016357422,
      "KL/mean": -166.3912811279297,
      "KL/rejected_KL_mean": -200.8452606201172,
      "KL/std": 92.92374420166016,
      "epoch": 0.5608465608465608,
      "fcm_dpo/beta": 0.005213094875216484,
      "fcm_dpo/delta": -0.027917178347706795,
      "fcm_dpo/margin": 68.90794372558594,
      "fcm_dpo/q_t": 0.41668015718460083,
      "grad_norm": 15.715535163879395,
      "learning_rate": 2.420680166254831e-07,
      "logits/chosen": 0.8548682928085327,
      "logits/rejected": 0.8279607892036438,
      "logps/chosen": -181.16151428222656,
      "logps/ref_chosen": -49.224212646484375,
      "logps/ref_rejected": -63.348472595214844,
      "logps/rejected": -264.1937255859375,
      "loss": 1.1289,
      "margin_dpo/margin_mean": 68.90794372558594,
      "margin_dpo/margin_std": 102.37510681152344,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -140.001708984375,
      "KL/mean": -161.2264404296875,
      "KL/rejected_KL_mean": -182.451171875,
      "KL/std": 93.77758026123047,
      "epoch": 0.562358276643991,
      "fcm_dpo/beta": 0.005198541563004255,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.44948959350586,
      "fcm_dpo/q_t": 0.4500351548194885,
      "grad_norm": 12.793850898742676,
      "learning_rate": 2.4074658027491044e-07,
      "logits/chosen": 0.7945922613143921,
      "logits/rejected": 0.7056927680969238,
      "logps/chosen": -192.271240234375,
      "logps/ref_chosen": -52.269554138183594,
      "logps/ref_rejected": -72.99522399902344,
      "logps/rejected": -255.4464111328125,
      "loss": 1.2767,
      "margin_dpo/margin_mean": 42.44948959350586,
      "margin_dpo/margin_std": 121.45559692382812,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -161.46475219726562,
      "KL/mean": -188.8463134765625,
      "KL/rejected_KL_mean": -216.2279052734375,
      "KL/std": 101.70494079589844,
      "epoch": 0.563869992441421,
      "fcm_dpo/beta": 0.005198541563004255,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.76313781738281,
      "fcm_dpo/q_t": 0.43512850999832153,
      "grad_norm": 11.343805313110352,
      "learning_rate": 2.394254027623792e-07,
      "logits/chosen": 0.7745212316513062,
      "logits/rejected": 0.7118136882781982,
      "logps/chosen": -222.57774353027344,
      "logps/ref_chosen": -61.112998962402344,
      "logps/ref_rejected": -76.24851989746094,
      "logps/rejected": -292.4764099121094,
      "loss": 1.2211,
      "margin_dpo/margin_mean": 54.76313781738281,
      "margin_dpo/margin_std": 124.361572265625,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -137.03231811523438,
      "KL/mean": -180.58938598632812,
      "KL/rejected_KL_mean": -224.1464385986328,
      "KL/std": 104.06997680664062,
      "epoch": 0.5653817082388511,
      "fcm_dpo/beta": 0.005156625993549824,
      "fcm_dpo/delta": -0.051555268466472626,
      "fcm_dpo/margin": 87.11412048339844,
      "fcm_dpo/q_t": 0.39748048782348633,
      "grad_norm": 14.498461723327637,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 0.7031779289245605,
      "logits/rejected": 0.7090506553649902,
      "logps/chosen": -209.7015380859375,
      "logps/ref_chosen": -72.66920471191406,
      "logps/ref_rejected": -76.83158874511719,
      "logps/rejected": -300.97802734375,
      "loss": 1.0878,
      "margin_dpo/margin_mean": 87.11412048339844,
      "margin_dpo/margin_std": 128.58489990234375,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -133.54660034179688,
      "KL/mean": -166.88844299316406,
      "KL/rejected_KL_mean": -200.23028564453125,
      "KL/std": 113.39869689941406,
      "epoch": 0.5668934240362812,
      "fcm_dpo/beta": 0.00514521449804306,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.68367004394531,
      "fcm_dpo/q_t": 0.42260217666625977,
      "grad_norm": 15.06658935546875,
      "learning_rate": 2.3678397206786715e-07,
      "logits/chosen": 0.7743487358093262,
      "logits/rejected": 0.7246922850608826,
      "logps/chosen": -191.22991943359375,
      "logps/ref_chosen": -57.68330383300781,
      "logps/ref_rejected": -79.34097290039062,
      "logps/rejected": -279.5712585449219,
      "loss": 1.1673,
      "margin_dpo/margin_mean": 66.68367004394531,
      "margin_dpo/margin_std": 123.28602600097656,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -140.264892578125,
      "KL/mean": -181.48529052734375,
      "KL/rejected_KL_mean": -222.70571899414062,
      "KL/std": 114.37265014648438,
      "epoch": 0.5684051398337112,
      "fcm_dpo/beta": 0.005105261690914631,
      "fcm_dpo/delta": -0.03897655010223389,
      "fcm_dpo/margin": 82.44081115722656,
      "fcm_dpo/q_t": 0.407854825258255,
      "grad_norm": 12.342073440551758,
      "learning_rate": 2.3546379277238103e-07,
      "logits/chosen": 0.8240178823471069,
      "logits/rejected": 0.7641883492469788,
      "logps/chosen": -191.93896484375,
      "logps/ref_chosen": -51.674072265625,
      "logps/ref_rejected": -75.69713592529297,
      "logps/rejected": -298.40283203125,
      "loss": 1.1138,
      "margin_dpo/margin_mean": 82.44081115722656,
      "margin_dpo/margin_std": 133.641357421875,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -143.00613403320312,
      "KL/mean": -170.45001220703125,
      "KL/rejected_KL_mean": -197.89390563964844,
      "KL/std": 89.45880126953125,
      "epoch": 0.5699168556311414,
      "fcm_dpo/beta": 0.005105261690914631,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 54.88779067993164,
      "fcm_dpo/q_t": 0.43406882882118225,
      "grad_norm": 12.48110294342041,
      "learning_rate": 2.3414402008585886e-07,
      "logits/chosen": 0.7774357795715332,
      "logits/rejected": 0.7551122903823853,
      "logps/chosen": -189.18466186523438,
      "logps/ref_chosen": -46.17853546142578,
      "logps/ref_rejected": -57.756500244140625,
      "logps/rejected": -255.65040588378906,
      "loss": 1.1898,
      "margin_dpo/margin_mean": 54.887786865234375,
      "margin_dpo/margin_std": 100.91114807128906,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -139.11705017089844,
      "KL/mean": -167.0404052734375,
      "KL/rejected_KL_mean": -194.96377563476562,
      "KL/std": 105.56818389892578,
      "epoch": 0.5714285714285714,
      "fcm_dpo/beta": 0.005105261690914631,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 55.846744537353516,
      "fcm_dpo/q_t": 0.43387657403945923,
      "grad_norm": 9.497807502746582,
      "learning_rate": 2.3282469092517977e-07,
      "logits/chosen": 0.817177414894104,
      "logits/rejected": 0.7751537561416626,
      "logps/chosen": -198.33592224121094,
      "logps/ref_chosen": -59.21887969970703,
      "logps/ref_rejected": -71.24818420410156,
      "logps/rejected": -266.21197509765625,
      "loss": 1.1906,
      "margin_dpo/margin_mean": 55.84674072265625,
      "margin_dpo/margin_std": 102.57722473144531,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -144.64332580566406,
      "KL/mean": -184.85391235351562,
      "KL/rejected_KL_mean": -225.06451416015625,
      "KL/std": 111.22518157958984,
      "epoch": 0.5729402872260015,
      "fcm_dpo/beta": 0.005105261690914631,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.42117309570312,
      "fcm_dpo/q_t": 0.40791893005371094,
      "grad_norm": 11.734951972961426,
      "learning_rate": 2.3150584219481643e-07,
      "logits/chosen": 0.7828426361083984,
      "logits/rejected": 0.7149155139923096,
      "logps/chosen": -220.95989990234375,
      "logps/ref_chosen": -76.31658935546875,
      "logps/ref_rejected": -104.26200103759766,
      "logps/rejected": -329.3265075683594,
      "loss": 1.1054,
      "margin_dpo/margin_mean": 80.42117309570312,
      "margin_dpo/margin_std": 121.62455749511719,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -119.44280242919922,
      "KL/mean": -162.35813903808594,
      "KL/rejected_KL_mean": -205.2735137939453,
      "KL/std": 97.02486419677734,
      "epoch": 0.5744520030234316,
      "fcm_dpo/beta": 0.005090559832751751,
      "fcm_dpo/delta": -0.028881238773465157,
      "fcm_dpo/margin": 85.83067321777344,
      "fcm_dpo/q_t": 0.39921677112579346,
      "grad_norm": 12.701099395751953,
      "learning_rate": 2.3018751078580283e-07,
      "logits/chosen": 0.7823134660720825,
      "logits/rejected": 0.7494254112243652,
      "logps/chosen": -180.72596740722656,
      "logps/ref_chosen": -61.283164978027344,
      "logps/ref_rejected": -72.38892364501953,
      "logps/rejected": -277.66241455078125,
      "loss": 1.0761,
      "margin_dpo/margin_mean": 85.83068084716797,
      "margin_dpo/margin_std": 114.44374084472656,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -146.58526611328125,
      "KL/mean": -163.72341918945312,
      "KL/rejected_KL_mean": -180.861572265625,
      "KL/std": 102.95851135253906,
      "epoch": 0.5759637188208617,
      "fcm_dpo/beta": 0.005075857508927584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 34.27632141113281,
      "fcm_dpo/q_t": 0.4604954719543457,
      "grad_norm": 10.866552352905273,
      "learning_rate": 2.288697335747027e-07,
      "logits/chosen": 0.7479392290115356,
      "logits/rejected": 0.7347009181976318,
      "logps/chosen": -204.79925537109375,
      "logps/ref_chosen": -58.2139892578125,
      "logps/ref_rejected": -60.78669357299805,
      "logps/rejected": -241.6482696533203,
      "loss": 1.3144,
      "margin_dpo/margin_mean": 34.27631759643555,
      "margin_dpo/margin_std": 124.18953704833984,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -147.87542724609375,
      "KL/mean": -179.83062744140625,
      "KL/rejected_KL_mean": -211.78582763671875,
      "KL/std": 103.192138671875,
      "epoch": 0.5774754346182918,
      "fcm_dpo/beta": 0.005075857508927584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.91041564941406,
      "fcm_dpo/q_t": 0.4254421591758728,
      "grad_norm": 12.20398998260498,
      "learning_rate": 2.2755254742257706e-07,
      "logits/chosen": 0.7671145796775818,
      "logits/rejected": 0.7206964492797852,
      "logps/chosen": -209.70074462890625,
      "logps/ref_chosen": -61.82532501220703,
      "logps/ref_rejected": -83.0452880859375,
      "logps/rejected": -294.83111572265625,
      "loss": 1.1512,
      "margin_dpo/margin_mean": 63.91041564941406,
      "margin_dpo/margin_std": 100.8181381225586,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -153.29888916015625,
      "KL/mean": -179.7510986328125,
      "KL/rejected_KL_mean": -206.20333862304688,
      "KL/std": 102.09304809570312,
      "epoch": 0.5789871504157218,
      "fcm_dpo/beta": 0.005075857508927584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.904457092285156,
      "fcm_dpo/q_t": 0.4390997290611267,
      "grad_norm": 13.56281566619873,
      "learning_rate": 2.2623598917395436e-07,
      "logits/chosen": 0.6700069904327393,
      "logits/rejected": 0.7050174474716187,
      "logps/chosen": -233.86215209960938,
      "logps/ref_chosen": -80.56326293945312,
      "logps/ref_rejected": -74.62922668457031,
      "logps/rejected": -280.83258056640625,
      "loss": 1.2353,
      "margin_dpo/margin_mean": 52.90446472167969,
      "margin_dpo/margin_std": 127.63932800292969,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -138.74447631835938,
      "KL/mean": -171.95449829101562,
      "KL/rejected_KL_mean": -205.16452026367188,
      "KL/std": 96.96101379394531,
      "epoch": 0.5804988662131519,
      "fcm_dpo/beta": 0.005075857508927584,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.4200439453125,
      "fcm_dpo/q_t": 0.420096755027771,
      "grad_norm": 13.154545783996582,
      "learning_rate": 2.2492009565579875e-07,
      "logits/chosen": 0.7832565307617188,
      "logits/rejected": 0.7525622844696045,
      "logps/chosen": -204.21963500976562,
      "logps/ref_chosen": -65.47514343261719,
      "logps/ref_rejected": -79.67378234863281,
      "logps/rejected": -284.83831787109375,
      "loss": 1.1424,
      "margin_dpo/margin_mean": 66.4200439453125,
      "margin_dpo/margin_std": 103.07756042480469,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -141.26541137695312,
      "KL/mean": -183.92938232421875,
      "KL/rejected_KL_mean": -226.5933380126953,
      "KL/std": 103.0797119140625,
      "epoch": 0.582010582010582,
      "fcm_dpo/beta": 0.005049902945756912,
      "fcm_dpo/delta": -0.051395926624536514,
      "fcm_dpo/margin": 85.32792663574219,
      "fcm_dpo/q_t": 0.40050801634788513,
      "grad_norm": 9.880163192749023,
      "learning_rate": 2.2360490367648084e-07,
      "logits/chosen": 0.7219782471656799,
      "logits/rejected": 0.6834163069725037,
      "logps/chosen": -207.32192993164062,
      "logps/ref_chosen": -66.0565185546875,
      "logps/ref_rejected": -86.68023681640625,
      "logps/rejected": -313.2735595703125,
      "loss": 1.0692,
      "margin_dpo/margin_mean": 85.32792663574219,
      "margin_dpo/margin_std": 105.10133361816406,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -165.14895629882812,
      "KL/mean": -193.8024139404297,
      "KL/rejected_KL_mean": -222.45584106445312,
      "KL/std": 109.71835327148438,
      "epoch": 0.5835222978080121,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.306884765625,
      "fcm_dpo/q_t": 0.4323265552520752,
      "grad_norm": 12.290263175964355,
      "learning_rate": 2.2229045002474724e-07,
      "logits/chosen": 0.7119057178497314,
      "logits/rejected": 0.664527177810669,
      "logps/chosen": -240.7726287841797,
      "logps/ref_chosen": -75.6236572265625,
      "logps/ref_rejected": -92.62330627441406,
      "logps/rejected": -315.07916259765625,
      "loss": 1.1928,
      "margin_dpo/margin_mean": 57.306884765625,
      "margin_dpo/margin_std": 110.36276245117188,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -143.9866485595703,
      "KL/mean": -184.1669921875,
      "KL/rejected_KL_mean": -224.3473663330078,
      "KL/std": 99.10783386230469,
      "epoch": 0.5850340136054422,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.36071014404297,
      "fcm_dpo/q_t": 0.4071784019470215,
      "grad_norm": 12.432520866394043,
      "learning_rate": 2.209767714686924e-07,
      "logits/chosen": 0.787395715713501,
      "logits/rejected": 0.6864792704582214,
      "logps/chosen": -191.20835876464844,
      "logps/ref_chosen": -47.22170639038086,
      "logps/ref_rejected": -87.338134765625,
      "logps/rejected": -311.68548583984375,
      "loss": 1.0866,
      "margin_dpo/margin_mean": 80.3607177734375,
      "margin_dpo/margin_std": 103.79031372070312,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -151.07595825195312,
      "KL/mean": -175.32736206054688,
      "KL/rejected_KL_mean": -199.5787811279297,
      "KL/std": 105.66417694091797,
      "epoch": 0.5865457294028723,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 48.50281524658203,
      "fcm_dpo/q_t": 0.4453752636909485,
      "grad_norm": 12.267721176147461,
      "learning_rate": 2.1966390475472954e-07,
      "logits/chosen": 0.7400516271591187,
      "logits/rejected": 0.7385213375091553,
      "logps/chosen": -225.65542602539062,
      "logps/ref_chosen": -74.5794677734375,
      "logps/ref_rejected": -79.92558288574219,
      "logps/rejected": -279.5043640136719,
      "loss": 1.2521,
      "margin_dpo/margin_mean": 48.5028190612793,
      "margin_dpo/margin_std": 125.29054260253906,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -140.3387451171875,
      "KL/mean": -180.39773559570312,
      "KL/rejected_KL_mean": -220.45672607421875,
      "KL/std": 98.77024841308594,
      "epoch": 0.5880574452003023,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.11798095703125,
      "fcm_dpo/q_t": 0.40777429938316345,
      "grad_norm": 20.287439346313477,
      "learning_rate": 2.1835188660656265e-07,
      "logits/chosen": 0.7831531763076782,
      "logits/rejected": 0.7533621788024902,
      "logps/chosen": -201.96310424804688,
      "logps/ref_chosen": -61.624366760253906,
      "logps/ref_rejected": -76.50978088378906,
      "logps/rejected": -296.96649169921875,
      "loss": 1.0962,
      "margin_dpo/margin_mean": 80.11798095703125,
      "margin_dpo/margin_std": 110.59423828125,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -124.74932098388672,
      "KL/mean": -155.60525512695312,
      "KL/rejected_KL_mean": -186.46119689941406,
      "KL/std": 94.935791015625,
      "epoch": 0.5895691609977324,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.71186828613281,
      "fcm_dpo/q_t": 0.42818349599838257,
      "grad_norm": 11.866067886352539,
      "learning_rate": 2.170407537241599e-07,
      "logits/chosen": 0.8283019065856934,
      "logits/rejected": 0.7651022672653198,
      "logps/chosen": -170.62118530273438,
      "logps/ref_chosen": -45.871864318847656,
      "logps/ref_rejected": -61.305999755859375,
      "logps/rejected": -247.76719665527344,
      "loss": 1.1634,
      "margin_dpo/margin_mean": 61.71186828613281,
      "margin_dpo/margin_std": 102.22355651855469,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -140.8014678955078,
      "KL/mean": -179.24481201171875,
      "KL/rejected_KL_mean": -217.68814086914062,
      "KL/std": 99.41610717773438,
      "epoch": 0.5910808767951625,
      "fcm_dpo/beta": 0.005023948848247528,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.8866958618164,
      "fcm_dpo/q_t": 0.41153138875961304,
      "grad_norm": 11.932952880859375,
      "learning_rate": 2.1573054278272636e-07,
      "logits/chosen": 0.7904618978500366,
      "logits/rejected": 0.7304722666740417,
      "logps/chosen": -198.98846435546875,
      "logps/ref_chosen": -58.18701171875,
      "logps/ref_rejected": -83.63442993164062,
      "logps/rejected": -301.32257080078125,
      "loss": 1.1213,
      "margin_dpo/margin_mean": 76.8866958618164,
      "margin_dpo/margin_std": 118.32135772705078,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -131.2482147216797,
      "KL/mean": -170.39854431152344,
      "KL/rejected_KL_mean": -209.5488739013672,
      "KL/std": 109.20217895507812,
      "epoch": 0.5925925925925926,
      "fcm_dpo/beta": 0.004997436888515949,
      "fcm_dpo/delta": -0.053051162511110306,
      "fcm_dpo/margin": 78.3006591796875,
      "fcm_dpo/q_t": 0.41144829988479614,
      "grad_norm": 11.570497512817383,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 0.8129167556762695,
      "logits/rejected": 0.7619023323059082,
      "logps/chosen": -200.99273681640625,
      "logps/ref_chosen": -69.7445297241211,
      "logps/ref_rejected": -94.05877685546875,
      "logps/rejected": -303.607666015625,
      "loss": 1.1248,
      "margin_dpo/margin_mean": 78.3006591796875,
      "margin_dpo/margin_std": 125.41221618652344,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -146.885498046875,
      "KL/mean": -188.21755981445312,
      "KL/rejected_KL_mean": -229.54962158203125,
      "KL/std": 106.22329711914062,
      "epoch": 0.5941043083900227,
      "fcm_dpo/beta": 0.004906511399894953,
      "fcm_dpo/delta": -0.06521414965391159,
      "fcm_dpo/margin": 82.66412353515625,
      "fcm_dpo/q_t": 0.4055265784263611,
      "grad_norm": 9.672101020812988,
      "learning_rate": 2.131130332936195e-07,
      "logits/chosen": 0.8200071454048157,
      "logits/rejected": 0.7858361601829529,
      "logps/chosen": -199.22039794921875,
      "logps/ref_chosen": -52.33489990234375,
      "logps/ref_rejected": -74.33809661865234,
      "logps/rejected": -303.8877258300781,
      "loss": 1.0866,
      "margin_dpo/margin_mean": 82.66412353515625,
      "margin_dpo/margin_std": 104.7359619140625,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -140.14215087890625,
      "KL/mean": -176.29550170898438,
      "KL/rejected_KL_mean": -212.44886779785156,
      "KL/std": 98.92924499511719,
      "epoch": 0.5956160241874527,
      "fcm_dpo/beta": 0.004906511399894953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 72.30670166015625,
      "fcm_dpo/q_t": 0.41654276847839355,
      "grad_norm": 11.078004837036133,
      "learning_rate": 2.1180580796331323e-07,
      "logits/chosen": 0.8135772943496704,
      "logits/rejected": 0.7879060506820679,
      "logps/chosen": -200.81829833984375,
      "logps/ref_chosen": -60.6761360168457,
      "logps/ref_rejected": -71.36074829101562,
      "logps/rejected": -283.80963134765625,
      "loss": 1.1207,
      "margin_dpo/margin_mean": 72.30670166015625,
      "margin_dpo/margin_std": 98.23634338378906,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -141.8544158935547,
      "KL/mean": -177.10147094726562,
      "KL/rejected_KL_mean": -212.34854125976562,
      "KL/std": 103.47579193115234,
      "epoch": 0.5971277399848829,
      "fcm_dpo/beta": 0.004906511399894953,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.49411010742188,
      "fcm_dpo/q_t": 0.4212576150894165,
      "grad_norm": 12.0609769821167,
      "learning_rate": 2.104996510066625e-07,
      "logits/chosen": 0.8366742134094238,
      "logits/rejected": 0.7517580389976501,
      "logps/chosen": -192.458740234375,
      "logps/ref_chosen": -50.60432434082031,
      "logps/ref_rejected": -77.08731079101562,
      "logps/rejected": -289.43585205078125,
      "loss": 1.1396,
      "margin_dpo/margin_mean": 70.4941177368164,
      "margin_dpo/margin_std": 111.34257507324219,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -137.52777099609375,
      "KL/mean": -172.69790649414062,
      "KL/rejected_KL_mean": -207.8680419921875,
      "KL/std": 110.33338928222656,
      "epoch": 0.5986394557823129,
      "fcm_dpo/beta": 0.004860267508774996,
      "fcm_dpo/delta": -0.04734856262803078,
      "fcm_dpo/margin": 70.34027862548828,
      "fcm_dpo/q_t": 0.420335054397583,
      "grad_norm": 9.277693748474121,
      "learning_rate": 2.0919459895968517e-07,
      "logits/chosen": 0.8274677991867065,
      "logits/rejected": 0.7451637387275696,
      "logps/chosen": -188.8873748779297,
      "logps/ref_chosen": -51.35961151123047,
      "logps/ref_rejected": -79.89360046386719,
      "logps/rejected": -287.76165771484375,
      "loss": 1.1321,
      "margin_dpo/margin_mean": 70.34027862548828,
      "margin_dpo/margin_std": 98.32467651367188,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -158.9486846923828,
      "KL/mean": -179.14877319335938,
      "KL/rejected_KL_mean": -199.3488311767578,
      "KL/std": 103.03988647460938,
      "epoch": 0.600151171579743,
      "fcm_dpo/beta": 0.004860267508774996,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 40.400169372558594,
      "fcm_dpo/q_t": 0.45476654171943665,
      "grad_norm": 10.52593994140625,
      "learning_rate": 2.078906883274924e-07,
      "logits/chosen": 0.7486104369163513,
      "logits/rejected": 0.7011691331863403,
      "logps/chosen": -225.4049072265625,
      "logps/ref_chosen": -66.45622253417969,
      "logps/ref_rejected": -85.74736785888672,
      "logps/rejected": -285.09619140625,
      "loss": 1.2768,
      "margin_dpo/margin_mean": 40.400169372558594,
      "margin_dpo/margin_std": 115.52510833740234,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -131.5105438232422,
      "KL/mean": -179.10305786132812,
      "KL/rejected_KL_mean": -226.69557189941406,
      "KL/std": 106.75947570800781,
      "epoch": 0.6016628873771731,
      "fcm_dpo/beta": 0.004792422521859407,
      "fcm_dpo/delta": -0.07028739899396896,
      "fcm_dpo/margin": 95.18501281738281,
      "fcm_dpo/q_t": 0.396945595741272,
      "grad_norm": 9.588266372680664,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": 0.8046476244926453,
      "logits/rejected": 0.7484967708587646,
      "logps/chosen": -180.75479125976562,
      "logps/ref_chosen": -49.244239807128906,
      "logps/ref_rejected": -75.18949127197266,
      "logps/rejected": -301.88507080078125,
      "loss": 1.0527,
      "margin_dpo/margin_mean": 95.18502044677734,
      "margin_dpo/margin_std": 117.26836395263672,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -157.90802001953125,
      "KL/mean": -212.11856079101562,
      "KL/rejected_KL_mean": -266.3291015625,
      "KL/std": 123.38093566894531,
      "epoch": 0.6031746031746031,
      "fcm_dpo/beta": 0.004697504919022322,
      "fcm_dpo/delta": -0.10002284497022629,
      "fcm_dpo/margin": 108.42111206054688,
      "fcm_dpo/q_t": 0.387840211391449,
      "grad_norm": 10.69479751586914,
      "learning_rate": 2.052864371672457e-07,
      "logits/chosen": 0.750247597694397,
      "logits/rejected": 0.6243202090263367,
      "logps/chosen": -226.21481323242188,
      "logps/ref_chosen": -68.30679321289062,
      "logps/ref_rejected": -113.2708511352539,
      "logps/rejected": -379.5999755859375,
      "loss": 1.036,
      "margin_dpo/margin_mean": 108.42111206054688,
      "margin_dpo/margin_std": 141.37326049804688,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -174.25674438476562,
      "KL/mean": -206.14834594726562,
      "KL/rejected_KL_mean": -238.03994750976562,
      "KL/std": 111.63092041015625,
      "epoch": 0.6046863189720333,
      "fcm_dpo/beta": 0.004666075576096773,
      "fcm_dpo/delta": -0.03356543183326721,
      "fcm_dpo/margin": 63.78321075439453,
      "fcm_dpo/q_t": 0.4290245771408081,
      "grad_norm": 14.130722045898438,
      "learning_rate": 2.0398616948569493e-07,
      "logits/chosen": 0.7988376617431641,
      "logits/rejected": 0.74619060754776,
      "logps/chosen": -245.88323974609375,
      "logps/ref_chosen": -71.62649536132812,
      "logps/ref_rejected": -90.98765563964844,
      "logps/rejected": -329.027587890625,
      "loss": 1.1741,
      "margin_dpo/margin_mean": 63.78321075439453,
      "margin_dpo/margin_std": 103.69830322265625,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -130.93389892578125,
      "KL/mean": -174.3590545654297,
      "KL/rejected_KL_mean": -217.78421020507812,
      "KL/std": 116.89300537109375,
      "epoch": 0.6061980347694633,
      "fcm_dpo/beta": 0.004630550276488066,
      "fcm_dpo/delta": -0.03821335360407829,
      "fcm_dpo/margin": 86.85031127929688,
      "fcm_dpo/q_t": 0.4078829884529114,
      "grad_norm": 7.8426055908203125,
      "learning_rate": 2.0268718890989752e-07,
      "logits/chosen": 0.8150224685668945,
      "logits/rejected": 0.7409332990646362,
      "logps/chosen": -184.6588592529297,
      "logps/ref_chosen": -53.72495651245117,
      "logps/ref_rejected": -75.06304931640625,
      "logps/rejected": -292.8472595214844,
      "loss": 1.0919,
      "margin_dpo/margin_mean": 86.85031127929688,
      "margin_dpo/margin_std": 116.00386047363281,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -145.6488037109375,
      "KL/mean": -178.26016235351562,
      "KL/rejected_KL_mean": -210.87152099609375,
      "KL/std": 97.66880798339844,
      "epoch": 0.6077097505668935,
      "fcm_dpo/beta": 0.004630550276488066,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 65.22270965576172,
      "fcm_dpo/q_t": 0.4299997091293335,
      "grad_norm": 11.507543563842773,
      "learning_rate": 2.013895317751323e-07,
      "logits/chosen": 0.7916992902755737,
      "logits/rejected": 0.785792350769043,
      "logps/chosen": -207.52273559570312,
      "logps/ref_chosen": -61.873931884765625,
      "logps/ref_rejected": -66.15198516845703,
      "logps/rejected": -277.02349853515625,
      "loss": 1.1841,
      "margin_dpo/margin_mean": 65.22270965576172,
      "margin_dpo/margin_std": 120.50914001464844,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -151.57379150390625,
      "KL/mean": -194.6015167236328,
      "KL/rejected_KL_mean": -237.6292724609375,
      "KL/std": 114.89691162109375,
      "epoch": 0.6092214663643235,
      "fcm_dpo/beta": 0.004630550276488066,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 86.05549621582031,
      "fcm_dpo/q_t": 0.409629762172699,
      "grad_norm": 8.590554237365723,
      "learning_rate": 2.0009323437965898e-07,
      "logits/chosen": 0.9000512957572937,
      "logits/rejected": 0.8217411041259766,
      "logps/chosen": -202.89527893066406,
      "logps/ref_chosen": -51.321502685546875,
      "logps/ref_rejected": -86.54010772705078,
      "logps/rejected": -324.16937255859375,
      "loss": 1.0996,
      "margin_dpo/margin_mean": 86.05550384521484,
      "margin_dpo/margin_std": 118.92399597167969,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -145.5100860595703,
      "KL/mean": -190.7388153076172,
      "KL/rejected_KL_mean": -235.96755981445312,
      "KL/std": 120.02464294433594,
      "epoch": 0.6107331821617535,
      "fcm_dpo/beta": 0.004570495802909136,
      "fcm_dpo/delta": -0.06526995450258255,
      "fcm_dpo/margin": 90.45745849609375,
      "fcm_dpo/q_t": 0.4066828787326813,
      "grad_norm": 12.769746780395508,
      "learning_rate": 1.9879833298370237e-07,
      "logits/chosen": 0.7784748077392578,
      "logits/rejected": 0.6955345869064331,
      "logps/chosen": -207.77297973632812,
      "logps/ref_chosen": -62.26288604736328,
      "logps/ref_rejected": -95.19029998779297,
      "logps/rejected": -331.1578369140625,
      "loss": 1.105,
      "margin_dpo/margin_mean": 90.45745849609375,
      "margin_dpo/margin_std": 134.48211669921875,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -147.45518493652344,
      "KL/mean": -180.65243530273438,
      "KL/rejected_KL_mean": -213.84967041015625,
      "KL/std": 103.51240539550781,
      "epoch": 0.6122448979591837,
      "fcm_dpo/beta": 0.004570495802909136,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.39448547363281,
      "fcm_dpo/q_t": 0.42996299266815186,
      "grad_norm": 9.430486679077148,
      "learning_rate": 1.975048638084379e-07,
      "logits/chosen": 0.8736664056777954,
      "logits/rejected": 0.8355212211608887,
      "logps/chosen": -198.03952026367188,
      "logps/ref_chosen": -50.5843391418457,
      "logps/ref_rejected": -65.43156433105469,
      "logps/rejected": -279.2812194824219,
      "loss": 1.1645,
      "margin_dpo/margin_mean": 66.39448547363281,
      "margin_dpo/margin_std": 103.71170043945312,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -136.18429565429688,
      "KL/mean": -185.00283813476562,
      "KL/rejected_KL_mean": -233.82138061523438,
      "KL/std": 112.89027404785156,
      "epoch": 0.6137566137566137,
      "fcm_dpo/beta": 0.00454317219555378,
      "fcm_dpo/delta": -0.06014251708984375,
      "fcm_dpo/margin": 97.63706970214844,
      "fcm_dpo/q_t": 0.397684782743454,
      "grad_norm": 9.497514724731445,
      "learning_rate": 1.9621286303497914e-07,
      "logits/chosen": 0.8508515357971191,
      "logits/rejected": 0.71308434009552,
      "logps/chosen": -185.17990112304688,
      "logps/ref_chosen": -48.99560546875,
      "logps/ref_rejected": -92.47774505615234,
      "logps/rejected": -326.29913330078125,
      "loss": 1.0675,
      "margin_dpo/margin_mean": 97.63706970214844,
      "margin_dpo/margin_std": 124.28600311279297,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -183.27198791503906,
      "KL/mean": -220.62786865234375,
      "KL/rejected_KL_mean": -257.98370361328125,
      "KL/std": 134.5868377685547,
      "epoch": 0.6152683295540439,
      "fcm_dpo/beta": 0.004515849053859711,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.71171569824219,
      "fcm_dpo/q_t": 0.4221992492675781,
      "grad_norm": 11.74845027923584,
      "learning_rate": 1.9492236680336483e-07,
      "logits/chosen": 0.7110066413879395,
      "logits/rejected": 0.6630129814147949,
      "logps/chosen": -272.67254638671875,
      "logps/ref_chosen": -89.40056610107422,
      "logps/ref_rejected": -99.28775024414062,
      "logps/rejected": -357.271484375,
      "loss": 1.1663,
      "margin_dpo/margin_mean": 74.71171569824219,
      "margin_dpo/margin_std": 135.02276611328125,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -133.894775390625,
      "KL/mean": -185.35739135742188,
      "KL/rejected_KL_mean": -236.8200225830078,
      "KL/std": 104.712646484375,
      "epoch": 0.6167800453514739,
      "fcm_dpo/beta": 0.004481324460357428,
      "fcm_dpo/delta": -0.07704263180494308,
      "fcm_dpo/margin": 102.92523956298828,
      "fcm_dpo/q_t": 0.3919832408428192,
      "grad_norm": 9.033692359924316,
      "learning_rate": 1.9363341121154895e-07,
      "logits/chosen": 0.8178368806838989,
      "logits/rejected": 0.7567524909973145,
      "logps/chosen": -188.59869384765625,
      "logps/ref_chosen": -54.70391845703125,
      "logps/ref_rejected": -73.98648834228516,
      "logps/rejected": -310.8065185546875,
      "loss": 1.0345,
      "margin_dpo/margin_mean": 102.92523956298828,
      "margin_dpo/margin_std": 108.17862701416016,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -166.08877563476562,
      "KL/mean": -189.05276489257812,
      "KL/rejected_KL_mean": -212.01673889160156,
      "KL/std": 89.52006530761719,
      "epoch": 0.618291761148904,
      "fcm_dpo/beta": 0.0044467998668551445,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.9279670715332,
      "fcm_dpo/q_t": 0.45155104994773865,
      "grad_norm": 10.081195831298828,
      "learning_rate": 1.9234603231438994e-07,
      "logits/chosen": 0.8174892663955688,
      "logits/rejected": 0.8276025652885437,
      "logps/chosen": -228.20700073242188,
      "logps/ref_chosen": -62.11822509765625,
      "logps/ref_rejected": -61.933509826660156,
      "logps/rejected": -273.95025634765625,
      "loss": 1.2391,
      "margin_dpo/margin_mean": 45.92797088623047,
      "margin_dpo/margin_std": 98.16754150390625,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -158.0648956298828,
      "KL/mean": -199.19512939453125,
      "KL/rejected_KL_mean": -240.3253631591797,
      "KL/std": 106.66167449951172,
      "epoch": 0.6198034769463341,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": -0.05007057636976242,
      "fcm_dpo/margin": 82.26046752929688,
      "fcm_dpo/q_t": 0.4151955246925354,
      "grad_norm": 10.520129203796387,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 0.8182001113891602,
      "logits/rejected": 0.8017312288284302,
      "logps/chosen": -219.8675537109375,
      "logps/ref_chosen": -61.80266189575195,
      "logps/ref_rejected": -76.60002136230469,
      "logps/rejected": -316.9253845214844,
      "loss": 1.114,
      "margin_dpo/margin_mean": 82.26046752929688,
      "margin_dpo/margin_std": 108.58341979980469,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -163.16366577148438,
      "KL/mean": -204.993896484375,
      "KL/rejected_KL_mean": -246.82412719726562,
      "KL/std": 111.47209930419922,
      "epoch": 0.6213151927437641,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.66046142578125,
      "fcm_dpo/q_t": 0.414519339799881,
      "grad_norm": 8.745308876037598,
      "learning_rate": 1.8977614860195296e-07,
      "logits/chosen": 0.8693751096725464,
      "logits/rejected": 0.8266406655311584,
      "logps/chosen": -217.6090545654297,
      "logps/ref_chosen": -54.44539260864258,
      "logps/ref_rejected": -74.5650863647461,
      "logps/rejected": -321.38922119140625,
      "loss": 1.1133,
      "margin_dpo/margin_mean": 83.66046142578125,
      "margin_dpo/margin_std": 116.60626220703125,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -167.1968231201172,
      "KL/mean": -203.74581909179688,
      "KL/rejected_KL_mean": -240.29483032226562,
      "KL/std": 93.24005126953125,
      "epoch": 0.6228269085411943,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.09800720214844,
      "fcm_dpo/q_t": 0.42384031414985657,
      "grad_norm": 9.76529598236084,
      "learning_rate": 1.8849371567184662e-07,
      "logits/chosen": 0.8335592746734619,
      "logits/rejected": 0.7884241342544556,
      "logps/chosen": -222.44491577148438,
      "logps/ref_chosen": -55.248085021972656,
      "logps/ref_rejected": -68.96623229980469,
      "logps/rejected": -309.26104736328125,
      "loss": 1.1367,
      "margin_dpo/margin_mean": 73.09800720214844,
      "margin_dpo/margin_std": 99.23159790039062,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -190.23448181152344,
      "KL/mean": -225.3627471923828,
      "KL/rejected_KL_mean": -260.49102783203125,
      "KL/std": 115.87466430664062,
      "epoch": 0.6243386243386243,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.25653076171875,
      "fcm_dpo/q_t": 0.42987310886383057,
      "grad_norm": 12.271631240844727,
      "learning_rate": 1.872130032047302e-07,
      "logits/chosen": 0.6928975582122803,
      "logits/rejected": 0.6748654842376709,
      "logps/chosen": -258.9552307128906,
      "logps/ref_chosen": -68.72074890136719,
      "logps/ref_rejected": -78.76539611816406,
      "logps/rejected": -339.25640869140625,
      "loss": 1.1877,
      "margin_dpo/margin_mean": 70.25653839111328,
      "margin_dpo/margin_std": 137.60073852539062,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -167.34481811523438,
      "KL/mean": -208.9085235595703,
      "KL/rejected_KL_mean": -250.47225952148438,
      "KL/std": 124.45516967773438,
      "epoch": 0.6258503401360545,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.12744140625,
      "fcm_dpo/q_t": 0.4144652783870697,
      "grad_norm": 9.069655418395996,
      "learning_rate": 1.8593404702488436e-07,
      "logits/chosen": 0.8730368614196777,
      "logits/rejected": 0.8275924921035767,
      "logps/chosen": -221.4830322265625,
      "logps/ref_chosen": -54.138214111328125,
      "logps/ref_rejected": -74.65741729736328,
      "logps/rejected": -325.1296691894531,
      "loss": 1.116,
      "margin_dpo/margin_mean": 83.12744140625,
      "margin_dpo/margin_std": 114.78118896484375,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -165.86544799804688,
      "KL/mean": -197.71322631835938,
      "KL/rejected_KL_mean": -229.56100463867188,
      "KL/std": 110.82231903076172,
      "epoch": 0.6273620559334845,
      "fcm_dpo/beta": 0.0044024912640452385,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.69554901123047,
      "fcm_dpo/q_t": 0.4349156320095062,
      "grad_norm": 9.737541198730469,
      "learning_rate": 1.846568829074628e-07,
      "logits/chosen": 0.8533459901809692,
      "logits/rejected": 0.8423031568527222,
      "logps/chosen": -221.7840118408203,
      "logps/ref_chosen": -55.91856002807617,
      "logps/ref_rejected": -61.747703552246094,
      "logps/rejected": -291.3087158203125,
      "loss": 1.1901,
      "margin_dpo/margin_mean": 63.6955451965332,
      "margin_dpo/margin_std": 119.05924987792969,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -175.8607177734375,
      "KL/mean": -207.20225524902344,
      "KL/rejected_KL_mean": -238.54379272460938,
      "KL/std": 120.81781005859375,
      "epoch": 0.6288737717309146,
      "fcm_dpo/beta": 0.004381248727440834,
      "fcm_dpo/delta": -0.04848504811525345,
      "fcm_dpo/margin": 62.68305587768555,
      "fcm_dpo/q_t": 0.4369519352912903,
      "grad_norm": 10.474597930908203,
      "learning_rate": 1.8338154657749128e-07,
      "logits/chosen": 0.8692808151245117,
      "logits/rejected": 0.8307493329048157,
      "logps/chosen": -230.58380126953125,
      "logps/ref_chosen": -54.72308349609375,
      "logps/ref_rejected": -69.17388916015625,
      "logps/rejected": -307.7176818847656,
      "loss": 1.1993,
      "margin_dpo/margin_mean": 62.68305206298828,
      "margin_dpo/margin_std": 111.89498901367188,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -185.3282470703125,
      "KL/mean": -222.47647094726562,
      "KL/rejected_KL_mean": -259.6246643066406,
      "KL/std": 111.57913208007812,
      "epoch": 0.6303854875283447,
      "fcm_dpo/beta": 0.004360006656497717,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.29644012451172,
      "fcm_dpo/q_t": 0.4229382574558258,
      "grad_norm": 10.666007995605469,
      "learning_rate": 1.8210807370886849e-07,
      "logits/chosen": 0.9554958343505859,
      "logits/rejected": 0.9064658880233765,
      "logps/chosen": -242.1195068359375,
      "logps/ref_chosen": -56.791259765625,
      "logps/ref_rejected": -68.7791748046875,
      "logps/rejected": -328.40386962890625,
      "loss": 1.172,
      "margin_dpo/margin_mean": 74.29644012451172,
      "margin_dpo/margin_std": 136.17581176757812,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -198.63465881347656,
      "KL/mean": -230.21011352539062,
      "KL/rejected_KL_mean": -261.78558349609375,
      "KL/std": 129.6304473876953,
      "epoch": 0.6318972033257747,
      "fcm_dpo/beta": 0.004330269526690245,
      "fcm_dpo/delta": -0.03421874716877937,
      "fcm_dpo/margin": 63.15090560913086,
      "fcm_dpo/q_t": 0.437247097492218,
      "grad_norm": 11.617671966552734,
      "learning_rate": 1.8083649992336825e-07,
      "logits/chosen": 0.8838375210762024,
      "logits/rejected": 0.8873622417449951,
      "logps/chosen": -267.7426452636719,
      "logps/ref_chosen": -69.10798645019531,
      "logps/ref_rejected": -75.09132385253906,
      "logps/rejected": -336.87689208984375,
      "loss": 1.2179,
      "margin_dpo/margin_mean": 63.150909423828125,
      "margin_dpo/margin_std": 133.4095458984375,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -160.0953369140625,
      "KL/mean": -207.4534912109375,
      "KL/rejected_KL_mean": -254.81167602539062,
      "KL/std": 113.61080932617188,
      "epoch": 0.6334089191232048,
      "fcm_dpo/beta": 0.004330269526690245,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.71630859375,
      "fcm_dpo/q_t": 0.4075653553009033,
      "grad_norm": 10.965876579284668,
      "learning_rate": 1.7956686078964255e-07,
      "logits/chosen": 0.7953141927719116,
      "logits/rejected": 0.7639617919921875,
      "logps/chosen": -218.26710510253906,
      "logps/ref_chosen": -58.1717643737793,
      "logps/ref_rejected": -71.67066955566406,
      "logps/rejected": -326.4823303222656,
      "loss": 1.0914,
      "margin_dpo/margin_mean": 94.71630859375,
      "margin_dpo/margin_std": 131.05889892578125,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -193.0113525390625,
      "KL/mean": -215.9877166748047,
      "KL/rejected_KL_mean": -238.96408081054688,
      "KL/std": 115.93824768066406,
      "epoch": 0.6349206349206349,
      "fcm_dpo/beta": 0.004330269526690245,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 45.952728271484375,
      "fcm_dpo/q_t": 0.45506352186203003,
      "grad_norm": 11.78740119934082,
      "learning_rate": 1.782991918222275e-07,
      "logits/chosen": 0.8286752700805664,
      "logits/rejected": 0.7962324619293213,
      "logps/chosen": -250.0648651123047,
      "logps/ref_chosen": -57.05351257324219,
      "logps/ref_rejected": -62.670982360839844,
      "logps/rejected": -301.63507080078125,
      "loss": 1.28,
      "margin_dpo/margin_mean": 45.952728271484375,
      "margin_dpo/margin_std": 135.70907592773438,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -182.4674835205078,
      "KL/mean": -216.71278381347656,
      "KL/rejected_KL_mean": -250.95806884765625,
      "KL/std": 116.42398834228516,
      "epoch": 0.636432350718065,
      "fcm_dpo/beta": 0.004330269526690245,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.49060821533203,
      "fcm_dpo/q_t": 0.43248650431632996,
      "grad_norm": 11.323017120361328,
      "learning_rate": 1.7703352848054887e-07,
      "logits/chosen": 0.8234415054321289,
      "logits/rejected": 0.7761733531951904,
      "logps/chosen": -239.7907257080078,
      "logps/ref_chosen": -57.32324981689453,
      "logps/ref_rejected": -75.33782958984375,
      "logps/rejected": -326.2958984375,
      "loss": 1.2224,
      "margin_dpo/margin_mean": 68.49061584472656,
      "margin_dpo/margin_std": 157.9302978515625,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -161.13418579101562,
      "KL/mean": -204.26010131835938,
      "KL/rejected_KL_mean": -247.38600158691406,
      "KL/std": 114.96134948730469,
      "epoch": 0.6379440665154951,
      "fcm_dpo/beta": 0.004321150481700897,
      "fcm_dpo/delta": -0.021103385835886,
      "fcm_dpo/margin": 86.2518310546875,
      "fcm_dpo/q_t": 0.41321080923080444,
      "grad_norm": 11.155305862426758,
      "learning_rate": 1.7576990616793137e-07,
      "logits/chosen": 0.829655647277832,
      "logits/rejected": 0.8312000036239624,
      "logps/chosen": -228.1917724609375,
      "logps/ref_chosen": -67.05757141113281,
      "logps/ref_rejected": -72.12803649902344,
      "logps/rejected": -319.5140380859375,
      "loss": 1.1045,
      "margin_dpo/margin_mean": 86.2518310546875,
      "margin_dpo/margin_std": 112.21836853027344,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -158.37295532226562,
      "KL/mean": -207.0155029296875,
      "KL/rejected_KL_mean": -255.65805053710938,
      "KL/std": 117.40293884277344,
      "epoch": 0.6394557823129252,
      "fcm_dpo/beta": 0.004282133188098669,
      "fcm_dpo/delta": -0.0347890630364418,
      "fcm_dpo/margin": 97.28510284423828,
      "fcm_dpo/q_t": 0.4046144485473633,
      "grad_norm": 10.667545318603516,
      "learning_rate": 1.745083602306071e-07,
      "logits/chosen": 0.8965709209442139,
      "logits/rejected": 0.8465409874916077,
      "logps/chosen": -212.43463134765625,
      "logps/ref_chosen": -54.06167221069336,
      "logps/ref_rejected": -76.64092254638672,
      "logps/rejected": -332.2989807128906,
      "loss": 1.081,
      "margin_dpo/margin_mean": 97.28510284423828,
      "margin_dpo/margin_std": 126.60137939453125,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -172.59568786621094,
      "KL/mean": -219.48318481445312,
      "KL/rejected_KL_mean": -266.37066650390625,
      "KL/std": 107.72590637207031,
      "epoch": 0.6409674981103552,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": -0.05255415290594101,
      "fcm_dpo/margin": 93.7750015258789,
      "fcm_dpo/q_t": 0.40855535864830017,
      "grad_norm": 13.741423606872559,
      "learning_rate": 1.7324892595672804e-07,
      "logits/chosen": 0.7810956239700317,
      "logits/rejected": 0.7487587928771973,
      "logps/chosen": -226.20455932617188,
      "logps/ref_chosen": -53.60887145996094,
      "logps/ref_rejected": -79.2139892578125,
      "logps/rejected": -345.58465576171875,
      "loss": 1.0937,
      "margin_dpo/margin_mean": 93.7750015258789,
      "margin_dpo/margin_std": 124.07484436035156,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -170.74240112304688,
      "KL/mean": -205.09945678710938,
      "KL/rejected_KL_mean": -239.45651245117188,
      "KL/std": 103.23504638671875,
      "epoch": 0.6424792139077853,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.714111328125,
      "fcm_dpo/q_t": 0.43254685401916504,
      "grad_norm": 9.86276912689209,
      "learning_rate": 1.7199163857537824e-07,
      "logits/chosen": 0.8737081289291382,
      "logits/rejected": 0.8555968403816223,
      "logps/chosen": -229.15707397460938,
      "logps/ref_chosen": -58.41468048095703,
      "logps/ref_rejected": -66.59054565429688,
      "logps/rejected": -306.04705810546875,
      "loss": 1.1778,
      "margin_dpo/margin_mean": 68.714111328125,
      "margin_dpo/margin_std": 119.1408462524414,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -205.45645141601562,
      "KL/mean": -226.54766845703125,
      "KL/rejected_KL_mean": -247.63888549804688,
      "KL/std": 116.4287109375,
      "epoch": 0.6439909297052154,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 42.18241882324219,
      "fcm_dpo/q_t": 0.4587920904159546,
      "grad_norm": 13.61494255065918,
      "learning_rate": 1.7073653325558828e-07,
      "logits/chosen": 0.7965115308761597,
      "logits/rejected": 0.7977316379547119,
      "logps/chosen": -277.1646728515625,
      "logps/ref_chosen": -71.70822143554688,
      "logps/ref_rejected": -73.57725524902344,
      "logps/rejected": -321.21612548828125,
      "loss": 1.3067,
      "margin_dpo/margin_mean": 42.18241882324219,
      "margin_dpo/margin_std": 147.598876953125,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -193.80029296875,
      "KL/mean": -232.99258422851562,
      "KL/rejected_KL_mean": -272.1848449707031,
      "KL/std": 128.01194763183594,
      "epoch": 0.6455026455026455,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 78.384521484375,
      "fcm_dpo/q_t": 0.42532289028167725,
      "grad_norm": 11.07363224029541,
      "learning_rate": 1.6948364510535218e-07,
      "logits/chosen": 0.841190755367279,
      "logits/rejected": 0.7872822880744934,
      "logps/chosen": -252.4430694580078,
      "logps/ref_chosen": -58.64276885986328,
      "logps/ref_rejected": -86.25437927246094,
      "logps/rejected": -358.439208984375,
      "loss": 1.1607,
      "margin_dpo/margin_mean": 78.38452911376953,
      "margin_dpo/margin_std": 138.72189331054688,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -189.307861328125,
      "KL/mean": -228.06832885742188,
      "KL/rejected_KL_mean": -266.82879638671875,
      "KL/std": 130.09091186523438,
      "epoch": 0.6470143613000756,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 77.52091217041016,
      "fcm_dpo/q_t": 0.4252082407474518,
      "grad_norm": 9.706740379333496,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 0.7988708019256592,
      "logits/rejected": 0.7730382084846497,
      "logps/chosen": -255.90391540527344,
      "logps/ref_chosen": -66.5960464477539,
      "logps/ref_rejected": -82.3941650390625,
      "logps/rejected": -349.22296142578125,
      "loss": 1.1705,
      "margin_dpo/margin_mean": 77.52090454101562,
      "margin_dpo/margin_std": 139.55557250976562,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -191.9099884033203,
      "KL/mean": -220.21624755859375,
      "KL/rejected_KL_mean": -248.52249145507812,
      "KL/std": 103.39564514160156,
      "epoch": 0.6485260770975056,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.61249542236328,
      "fcm_dpo/q_t": 0.44227659702301025,
      "grad_norm": 10.24012279510498,
      "learning_rate": 1.669846604344412e-07,
      "logits/chosen": 0.8254667520523071,
      "logits/rejected": 0.8455654382705688,
      "logps/chosen": -248.91969299316406,
      "logps/ref_chosen": -57.00970458984375,
      "logps/ref_rejected": -59.86549377441406,
      "logps/rejected": -308.38800048828125,
      "loss": 1.2237,
      "margin_dpo/margin_mean": 56.61249542236328,
      "margin_dpo/margin_std": 118.35189819335938,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -168.93666076660156,
      "KL/mean": -217.52066040039062,
      "KL/rejected_KL_mean": -266.10467529296875,
      "KL/std": 104.36176300048828,
      "epoch": 0.6500377928949358,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 97.16799926757812,
      "fcm_dpo/q_t": 0.402678519487381,
      "grad_norm": 11.644641876220703,
      "learning_rate": 1.6573863381573954e-07,
      "logits/chosen": 0.7442194223403931,
      "logits/rejected": 0.7441260814666748,
      "logps/chosen": -228.49984741210938,
      "logps/ref_chosen": -59.563194274902344,
      "logps/ref_rejected": -70.52289581298828,
      "logps/rejected": -336.6275634765625,
      "loss": 1.0823,
      "margin_dpo/margin_mean": 97.16799926757812,
      "margin_dpo/margin_std": 123.15974426269531,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -166.54751586914062,
      "KL/mean": -204.51138305664062,
      "KL/rejected_KL_mean": -242.4752197265625,
      "KL/std": 112.0499267578125,
      "epoch": 0.6515495086923658,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 75.92768859863281,
      "fcm_dpo/q_t": 0.4258786141872406,
      "grad_norm": 10.839800834655762,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": 0.8408886790275574,
      "logits/rejected": 0.7930970191955566,
      "logps/chosen": -216.7478485107422,
      "logps/ref_chosen": -50.20032501220703,
      "logps/ref_rejected": -77.81680297851562,
      "logps/rejected": -320.2920227050781,
      "loss": 1.1613,
      "margin_dpo/margin_mean": 75.92768096923828,
      "margin_dpo/margin_std": 128.07762145996094,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -177.85556030273438,
      "KL/mean": -216.429931640625,
      "KL/rejected_KL_mean": -255.00430297851562,
      "KL/std": 112.36427307128906,
      "epoch": 0.6530612244897959,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 77.14874267578125,
      "fcm_dpo/q_t": 0.42348039150238037,
      "grad_norm": 11.070759773254395,
      "learning_rate": 1.632536862810844e-07,
      "logits/chosen": 0.8612329959869385,
      "logits/rejected": 0.8160426616668701,
      "logps/chosen": -239.51832580566406,
      "logps/ref_chosen": -61.662757873535156,
      "logps/ref_rejected": -83.94496154785156,
      "logps/rejected": -338.94927978515625,
      "loss": 1.1634,
      "margin_dpo/margin_mean": 77.14874267578125,
      "margin_dpo/margin_std": 134.41839599609375,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -174.31484985351562,
      "KL/mean": -217.500732421875,
      "KL/rejected_KL_mean": -260.6866455078125,
      "KL/std": 102.90592956542969,
      "epoch": 0.654572940287226,
      "fcm_dpo/beta": 0.004237360320985317,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 86.37177276611328,
      "fcm_dpo/q_t": 0.4140329957008362,
      "grad_norm": 11.327740669250488,
      "learning_rate": 1.6201483487445515e-07,
      "logits/chosen": 0.8867149353027344,
      "logits/rejected": 0.8934628963470459,
      "logps/chosen": -238.04403686523438,
      "logps/ref_chosen": -63.72917938232422,
      "logps/ref_rejected": -65.8391342163086,
      "logps/rejected": -326.5257568359375,
      "loss": 1.1126,
      "margin_dpo/margin_mean": 86.37178039550781,
      "margin_dpo/margin_std": 117.17826843261719,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -147.67396545410156,
      "KL/mean": -197.6832275390625,
      "KL/rejected_KL_mean": -247.69247436523438,
      "KL/std": 115.82525634765625,
      "epoch": 0.656084656084656,
      "fcm_dpo/beta": 0.004173349589109421,
      "fcm_dpo/delta": -0.0761074498295784,
      "fcm_dpo/margin": 100.01850891113281,
      "fcm_dpo/q_t": 0.4028673768043518,
      "grad_norm": 10.230304718017578,
      "learning_rate": 1.6077844460203204e-07,
      "logits/chosen": 0.945635199546814,
      "logits/rejected": 0.8929431438446045,
      "logps/chosen": -195.64727783203125,
      "logps/ref_chosen": -47.97331619262695,
      "logps/ref_rejected": -72.51132202148438,
      "logps/rejected": -320.20379638671875,
      "loss": 1.0938,
      "margin_dpo/margin_mean": 100.01850891113281,
      "margin_dpo/margin_std": 138.5220489501953,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -179.42828369140625,
      "KL/mean": -215.53460693359375,
      "KL/rejected_KL_mean": -251.64093017578125,
      "KL/std": 111.70719909667969,
      "epoch": 0.6575963718820862,
      "fcm_dpo/beta": 0.004173349589109421,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 72.21266174316406,
      "fcm_dpo/q_t": 0.42970752716064453,
      "grad_norm": 11.32288646697998,
      "learning_rate": 1.5954455004830878e-07,
      "logits/chosen": 0.9255178570747375,
      "logits/rejected": 0.8949824571609497,
      "logps/chosen": -236.488525390625,
      "logps/ref_chosen": -57.06024932861328,
      "logps/ref_rejected": -71.69146728515625,
      "logps/rejected": -323.3323974609375,
      "loss": 1.1787,
      "margin_dpo/margin_mean": 72.21266174316406,
      "margin_dpo/margin_std": 129.89993286132812,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -183.29244995117188,
      "KL/mean": -212.61312866210938,
      "KL/rejected_KL_mean": -241.93377685546875,
      "KL/std": 108.2564697265625,
      "epoch": 0.6591080876795162,
      "fcm_dpo/beta": 0.004173349589109421,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.641334533691406,
      "fcm_dpo/q_t": 0.44333615899086,
      "grad_norm": 12.016843795776367,
      "learning_rate": 1.5831318572796847e-07,
      "logits/chosen": 0.8693698644638062,
      "logits/rejected": 0.8239619135856628,
      "logps/chosen": -239.45050048828125,
      "logps/ref_chosen": -56.158050537109375,
      "logps/ref_rejected": -67.63787841796875,
      "logps/rejected": -309.5716552734375,
      "loss": 1.2306,
      "margin_dpo/margin_mean": 58.641334533691406,
      "margin_dpo/margin_std": 132.94003295898438,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -185.86080932617188,
      "KL/mean": -225.59451293945312,
      "KL/rejected_KL_mean": -265.3282165527344,
      "KL/std": 121.49739074707031,
      "epoch": 0.6606198034769464,
      "fcm_dpo/beta": 0.004142164718359709,
      "fcm_dpo/delta": -0.03750201314687729,
      "fcm_dpo/margin": 79.46741485595703,
      "fcm_dpo/q_t": 0.42540091276168823,
      "grad_norm": 13.124794960021973,
      "learning_rate": 1.5708438608491815e-07,
      "logits/chosen": 0.8585692048072815,
      "logits/rejected": 0.7549466490745544,
      "logps/chosen": -242.84658813476562,
      "logps/ref_chosen": -56.98578643798828,
      "logps/ref_rejected": -85.61524963378906,
      "logps/rejected": -350.9434814453125,
      "loss": 1.1855,
      "margin_dpo/margin_mean": 79.46741485595703,
      "margin_dpo/margin_std": 155.61752319335938,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -160.57635498046875,
      "KL/mean": -209.36996459960938,
      "KL/rejected_KL_mean": -258.16357421875,
      "KL/std": 124.45043182373047,
      "epoch": 0.6621315192743764,
      "fcm_dpo/beta": 0.004129474051296711,
      "fcm_dpo/delta": -0.030732562765479088,
      "fcm_dpo/margin": 97.58722686767578,
      "fcm_dpo/q_t": 0.41016703844070435,
      "grad_norm": 10.8823881149292,
      "learning_rate": 1.558581854913253e-07,
      "logits/chosen": 0.9030640125274658,
      "logits/rejected": 0.8500392436981201,
      "logps/chosen": -201.8541259765625,
      "logps/ref_chosen": -41.27777862548828,
      "logps/ref_rejected": -65.33840942382812,
      "logps/rejected": -323.5019836425781,
      "loss": 1.0969,
      "margin_dpo/margin_mean": 97.58723449707031,
      "margin_dpo/margin_std": 140.27755737304688,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -190.50978088378906,
      "KL/mean": -234.99398803710938,
      "KL/rejected_KL_mean": -279.47821044921875,
      "KL/std": 127.0751953125,
      "epoch": 0.6636432350718064,
      "fcm_dpo/beta": 0.004095879383385181,
      "fcm_dpo/delta": -0.05103587731719017,
      "fcm_dpo/margin": 88.96844482421875,
      "fcm_dpo/q_t": 0.4150954484939575,
      "grad_norm": 11.545726776123047,
      "learning_rate": 1.5463461824665658e-07,
      "logits/chosen": 0.7603689432144165,
      "logits/rejected": 0.7392828464508057,
      "logps/chosen": -271.92742919921875,
      "logps/ref_chosen": -81.41764831542969,
      "logps/ref_rejected": -94.72309875488281,
      "logps/rejected": -374.2012939453125,
      "loss": 1.1227,
      "margin_dpo/margin_mean": 88.96844482421875,
      "margin_dpo/margin_std": 126.95797729492188,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -157.71478271484375,
      "KL/mean": -202.0891876220703,
      "KL/rejected_KL_mean": -246.4635772705078,
      "KL/std": 111.21897888183594,
      "epoch": 0.6651549508692366,
      "fcm_dpo/beta": 0.004074975848197937,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.74879455566406,
      "fcm_dpo/q_t": 0.41615790128707886,
      "grad_norm": 13.960528373718262,
      "learning_rate": 1.534137185767178e-07,
      "logits/chosen": 0.8458383083343506,
      "logits/rejected": 0.7661876678466797,
      "logps/chosen": -200.2529754638672,
      "logps/ref_chosen": -42.538185119628906,
      "logps/ref_rejected": -69.78813934326172,
      "logps/rejected": -316.251708984375,
      "loss": 1.1194,
      "margin_dpo/margin_mean": 88.74879455566406,
      "margin_dpo/margin_std": 124.50497436523438,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -165.97714233398438,
      "KL/mean": -213.51602172851562,
      "KL/rejected_KL_mean": -261.05487060546875,
      "KL/std": 123.1169204711914,
      "epoch": 0.6666666666666666,
      "fcm_dpo/beta": 0.004050433170050383,
      "fcm_dpo/delta": -0.030205076560378075,
      "fcm_dpo/margin": 95.07774353027344,
      "fcm_dpo/q_t": 0.4091061055660248,
      "grad_norm": 12.10502815246582,
      "learning_rate": 1.521955206326976e-07,
      "logits/chosen": 0.8075554370880127,
      "logits/rejected": 0.7362128496170044,
      "logps/chosen": -223.57037353515625,
      "logps/ref_chosen": -57.593223571777344,
      "logps/ref_rejected": -84.82878875732422,
      "logps/rejected": -345.8836669921875,
      "loss": 1.0876,
      "margin_dpo/margin_mean": 95.07774353027344,
      "margin_dpo/margin_std": 111.09236907958984,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -198.12374877929688,
      "KL/mean": -246.94883728027344,
      "KL/rejected_KL_mean": -295.77392578125,
      "KL/std": 124.66996765136719,
      "epoch": 0.6681783824640968,
      "fcm_dpo/beta": 0.004028483759611845,
      "fcm_dpo/delta": -0.0271687563508749,
      "fcm_dpo/margin": 97.65019226074219,
      "fcm_dpo/q_t": 0.40914517641067505,
      "grad_norm": 12.60295295715332,
      "learning_rate": 1.5098005849021078e-07,
      "logits/chosen": 0.8383224606513977,
      "logits/rejected": 0.8079402446746826,
      "logps/chosen": -265.5849609375,
      "logps/ref_chosen": -67.46121978759766,
      "logps/ref_rejected": -89.0693588256836,
      "logps/rejected": -384.8432922363281,
      "loss": 1.0944,
      "margin_dpo/margin_mean": 97.65019226074219,
      "margin_dpo/margin_std": 127.72940063476562,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -163.28781127929688,
      "KL/mean": -221.1153106689453,
      "KL/rejected_KL_mean": -278.94281005859375,
      "KL/std": 130.93421936035156,
      "epoch": 0.6696900982615268,
      "fcm_dpo/beta": 0.003985672257840633,
      "fcm_dpo/delta": -0.0640106350183487,
      "fcm_dpo/margin": 115.65498352050781,
      "fcm_dpo/q_t": 0.3956124186515808,
      "grad_norm": 11.192153930664062,
      "learning_rate": 1.4976736614834662e-07,
      "logits/chosen": 0.8276779055595398,
      "logits/rejected": 0.7804521322250366,
      "logps/chosen": -218.08392333984375,
      "logps/ref_chosen": -54.79610061645508,
      "logps/ref_rejected": -77.80781555175781,
      "logps/rejected": -356.7506103515625,
      "loss": 1.053,
      "margin_dpo/margin_mean": 115.65498352050781,
      "margin_dpo/margin_std": 147.14096069335938,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -195.40420532226562,
      "KL/mean": -227.3412322998047,
      "KL/rejected_KL_mean": -259.27825927734375,
      "KL/std": 135.2494354248047,
      "epoch": 0.671201814058957,
      "fcm_dpo/beta": 0.00397723913192749,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 63.8740348815918,
      "fcm_dpo/q_t": 0.4439532160758972,
      "grad_norm": 10.997804641723633,
      "learning_rate": 1.4855747752871654e-07,
      "logits/chosen": 0.8664942979812622,
      "logits/rejected": 0.7986875772476196,
      "logps/chosen": -254.15325927734375,
      "logps/ref_chosen": -58.749061584472656,
      "logps/ref_rejected": -86.87396240234375,
      "logps/rejected": -346.1522216796875,
      "loss": 1.2352,
      "margin_dpo/margin_mean": 63.8740348815918,
      "margin_dpo/margin_std": 151.58612060546875,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -186.91659545898438,
      "KL/mean": -236.37582397460938,
      "KL/rejected_KL_mean": -285.83502197265625,
      "KL/std": 114.96759796142578,
      "epoch": 0.672713529856387,
      "fcm_dpo/beta": 0.00397723913192749,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 98.91845703125,
      "fcm_dpo/q_t": 0.40706494450569153,
      "grad_norm": 9.45373821258545,
      "learning_rate": 1.473504264745062e-07,
      "logits/chosen": 0.8414635062217712,
      "logits/rejected": 0.8434320092201233,
      "logps/chosen": -247.8340301513672,
      "logps/ref_chosen": -60.91743850708008,
      "logps/ref_rejected": -71.5637435913086,
      "logps/rejected": -357.3988037109375,
      "loss": 1.0856,
      "margin_dpo/margin_mean": 98.91845703125,
      "margin_dpo/margin_std": 119.76266479492188,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -168.513916015625,
      "KL/mean": -217.90615844726562,
      "KL/rejected_KL_mean": -267.29840087890625,
      "KL/std": 118.74134826660156,
      "epoch": 0.674225245653817,
      "fcm_dpo/beta": 0.00394941633567214,
      "fcm_dpo/delta": -0.07044916599988937,
      "fcm_dpo/margin": 98.78448486328125,
      "fcm_dpo/q_t": 0.4086703062057495,
      "grad_norm": 9.038079261779785,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 0.8681352138519287,
      "logits/rejected": 0.807909369468689,
      "logps/chosen": -217.3131561279297,
      "logps/ref_chosen": -48.79924774169922,
      "logps/ref_rejected": -71.8719482421875,
      "logps/rejected": -339.17034912109375,
      "loss": 1.0904,
      "margin_dpo/margin_mean": 98.78448486328125,
      "margin_dpo/margin_std": 117.52779388427734,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -164.670166015625,
      "KL/mean": -223.03323364257812,
      "KL/rejected_KL_mean": -281.39630126953125,
      "KL/std": 118.19184875488281,
      "epoch": 0.6757369614512472,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": -0.054448507726192474,
      "fcm_dpo/margin": 116.72611999511719,
      "fcm_dpo/q_t": 0.39377325773239136,
      "grad_norm": 9.447699546813965,
      "learning_rate": 1.4494497203727843e-07,
      "logits/chosen": 0.8478422164916992,
      "logits/rejected": 0.7641305923461914,
      "logps/chosen": -218.35289001464844,
      "logps/ref_chosen": -53.682716369628906,
      "logps/ref_rejected": -88.17315673828125,
      "logps/rejected": -369.5694580078125,
      "loss": 1.0486,
      "margin_dpo/margin_mean": 116.72611999511719,
      "margin_dpo/margin_std": 133.0546875,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -182.4579620361328,
      "KL/mean": -227.93060302734375,
      "KL/rejected_KL_mean": -273.4031982421875,
      "KL/std": 124.02155303955078,
      "epoch": 0.6772486772486772,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 90.94525146484375,
      "fcm_dpo/q_t": 0.4181848466396332,
      "grad_norm": 8.76012134552002,
      "learning_rate": 1.4374663593999256e-07,
      "logits/chosen": 0.8726906776428223,
      "logits/rejected": 0.837250828742981,
      "logps/chosen": -236.209228515625,
      "logps/ref_chosen": -53.75125503540039,
      "logps/ref_rejected": -77.17623901367188,
      "logps/rejected": -350.5794677734375,
      "loss": 1.1278,
      "margin_dpo/margin_mean": 90.94525146484375,
      "margin_dpo/margin_std": 134.71463012695312,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -219.57989501953125,
      "KL/mean": -242.92726135253906,
      "KL/rejected_KL_mean": -266.2746276855469,
      "KL/std": 132.32901000976562,
      "epoch": 0.6787603930461074,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 46.694740295410156,
      "fcm_dpo/q_t": 0.45708128809928894,
      "grad_norm": 16.812198638916016,
      "learning_rate": 1.4255127197770707e-07,
      "logits/chosen": 0.7664126753807068,
      "logits/rejected": 0.7682603597640991,
      "logps/chosen": -295.40728759765625,
      "logps/ref_chosen": -75.82737731933594,
      "logps/ref_rejected": -82.20687866210938,
      "logps/rejected": -348.48150634765625,
      "loss": 1.2788,
      "margin_dpo/margin_mean": 46.69473648071289,
      "margin_dpo/margin_std": 132.43252563476562,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -174.35791015625,
      "KL/mean": -214.56973266601562,
      "KL/rejected_KL_mean": -254.78155517578125,
      "KL/std": 129.234130859375,
      "epoch": 0.6802721088435374,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.42366027832031,
      "fcm_dpo/q_t": 0.4304322600364685,
      "grad_norm": 11.050307273864746,
      "learning_rate": 1.4135891358732205e-07,
      "logits/chosen": 0.9499484300613403,
      "logits/rejected": 0.8608253002166748,
      "logps/chosen": -221.4736328125,
      "logps/ref_chosen": -47.11572265625,
      "logps/ref_rejected": -78.7546615600586,
      "logps/rejected": -333.5362243652344,
      "loss": 1.1788,
      "margin_dpo/margin_mean": 80.42366790771484,
      "margin_dpo/margin_std": 149.21299743652344,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -178.43597412109375,
      "KL/mean": -214.04747009277344,
      "KL/rejected_KL_mean": -249.65896606445312,
      "KL/std": 120.39581298828125,
      "epoch": 0.6817838246409675,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.22297668457031,
      "fcm_dpo/q_t": 0.4351150393486023,
      "grad_norm": 10.856236457824707,
      "learning_rate": 1.4016959412166437e-07,
      "logits/chosen": 0.8059324026107788,
      "logits/rejected": 0.7745248079299927,
      "logps/chosen": -241.7864227294922,
      "logps/ref_chosen": -63.350440979003906,
      "logps/ref_rejected": -76.28530883789062,
      "logps/rejected": -325.94427490234375,
      "loss": 1.1836,
      "margin_dpo/margin_mean": 71.22297668457031,
      "margin_dpo/margin_std": 123.11032104492188,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -180.02713012695312,
      "KL/mean": -219.95492553710938,
      "KL/rejected_KL_mean": -259.88275146484375,
      "KL/std": 116.23613739013672,
      "epoch": 0.6832955404383976,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.85560607910156,
      "fcm_dpo/q_t": 0.42794910073280334,
      "grad_norm": 13.129495620727539,
      "learning_rate": 1.3898334684855645e-07,
      "logits/chosen": 0.8197082281112671,
      "logits/rejected": 0.7570655345916748,
      "logps/chosen": -235.6129608154297,
      "logps/ref_chosen": -55.58583450317383,
      "logps/ref_rejected": -77.68738555908203,
      "logps/rejected": -337.57012939453125,
      "loss": 1.1709,
      "margin_dpo/margin_mean": 79.85560607910156,
      "margin_dpo/margin_std": 140.17767333984375,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -182.94837951660156,
      "KL/mean": -218.63784790039062,
      "KL/rejected_KL_mean": -254.3273162841797,
      "KL/std": 119.77302551269531,
      "epoch": 0.6848072562358276,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.37896728515625,
      "fcm_dpo/q_t": 0.43533623218536377,
      "grad_norm": 10.636575698852539,
      "learning_rate": 1.3780020494988445e-07,
      "logits/chosen": 0.8267333507537842,
      "logits/rejected": 0.8144370913505554,
      "logps/chosen": -244.72657775878906,
      "logps/ref_chosen": -61.778202056884766,
      "logps/ref_rejected": -71.51403045654297,
      "logps/rejected": -325.8413391113281,
      "loss": 1.1923,
      "margin_dpo/margin_mean": 71.37896728515625,
      "margin_dpo/margin_std": 132.53733825683594,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -163.52182006835938,
      "KL/mean": -207.54925537109375,
      "KL/rejected_KL_mean": -251.57669067382812,
      "KL/std": 117.58868408203125,
      "epoch": 0.6863189720332578,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.05487060546875,
      "fcm_dpo/q_t": 0.4201856553554535,
      "grad_norm": 9.357555389404297,
      "learning_rate": 1.366202015206706e-07,
      "logits/chosen": 0.8628107309341431,
      "logits/rejected": 0.837904691696167,
      "logps/chosen": -215.11697387695312,
      "logps/ref_chosen": -51.59515380859375,
      "logps/ref_rejected": -63.96732711791992,
      "logps/rejected": -315.54400634765625,
      "loss": 1.1354,
      "margin_dpo/margin_mean": 88.05487060546875,
      "margin_dpo/margin_std": 133.06265258789062,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -196.3070068359375,
      "KL/mean": -236.11773681640625,
      "KL/rejected_KL_mean": -275.928466796875,
      "KL/std": 121.60128784179688,
      "epoch": 0.6878306878306878,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.6214599609375,
      "fcm_dpo/q_t": 0.42936086654663086,
      "grad_norm": 11.233712196350098,
      "learning_rate": 1.354433695681474e-07,
      "logits/chosen": 0.7930549383163452,
      "logits/rejected": 0.7773592472076416,
      "logps/chosen": -266.958740234375,
      "logps/ref_chosen": -70.65170288085938,
      "logps/ref_rejected": -77.44276428222656,
      "logps/rejected": -353.3712463378906,
      "loss": 1.1624,
      "margin_dpo/margin_mean": 79.6214599609375,
      "margin_dpo/margin_std": 131.82275390625,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -184.79290771484375,
      "KL/mean": -225.66094970703125,
      "KL/rejected_KL_mean": -266.52899169921875,
      "KL/std": 121.29298400878906,
      "epoch": 0.6893424036281179,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.73609161376953,
      "fcm_dpo/q_t": 0.42831528186798096,
      "grad_norm": 13.263677597045898,
      "learning_rate": 1.3426974201083439e-07,
      "logits/chosen": 0.7968763709068298,
      "logits/rejected": 0.7505416870117188,
      "logps/chosen": -241.19119262695312,
      "logps/ref_chosen": -56.398284912109375,
      "logps/ref_rejected": -82.61642456054688,
      "logps/rejected": -349.14544677734375,
      "loss": 1.1627,
      "margin_dpo/margin_mean": 81.73609924316406,
      "margin_dpo/margin_std": 141.47987365722656,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -181.234130859375,
      "KL/mean": -221.42344665527344,
      "KL/rejected_KL_mean": -261.61279296875,
      "KL/std": 125.76825714111328,
      "epoch": 0.690854119425548,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.37864685058594,
      "fcm_dpo/q_t": 0.4260377287864685,
      "grad_norm": 12.457381248474121,
      "learning_rate": 1.3309935167761717e-07,
      "logits/chosen": 0.9459880590438843,
      "logits/rejected": 0.8794763684272766,
      "logps/chosen": -225.9547119140625,
      "logps/ref_chosen": -44.72057342529297,
      "logps/ref_rejected": -68.1158676147461,
      "logps/rejected": -329.7286376953125,
      "loss": 1.1543,
      "margin_dpo/margin_mean": 80.37864685058594,
      "margin_dpo/margin_std": 124.81919860839844,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -174.767822265625,
      "KL/mean": -222.24642944335938,
      "KL/rejected_KL_mean": -269.72503662109375,
      "KL/std": 126.41743469238281,
      "epoch": 0.6923658352229781,
      "fcm_dpo/beta": 0.0038791203405708075,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.95720672607422,
      "fcm_dpo/q_t": 0.41461867094039917,
      "grad_norm": 10.82316780090332,
      "learning_rate": 1.3193223130682936e-07,
      "logits/chosen": 0.8550417423248291,
      "logits/rejected": 0.7658358812332153,
      "logps/chosen": -224.77352905273438,
      "logps/ref_chosen": -50.00569152832031,
      "logps/ref_rejected": -87.50015258789062,
      "logps/rejected": -357.2251892089844,
      "loss": 1.1235,
      "margin_dpo/margin_mean": 94.95721435546875,
      "margin_dpo/margin_std": 142.54466247558594,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -177.70147705078125,
      "KL/mean": -230.20147705078125,
      "KL/rejected_KL_mean": -282.70147705078125,
      "KL/std": 154.35580444335938,
      "epoch": 0.6938775510204082,
      "fcm_dpo/beta": 0.0038504954427480698,
      "fcm_dpo/delta": -0.07434207946062088,
      "fcm_dpo/margin": 104.99995422363281,
      "fcm_dpo/q_t": 0.40416857600212097,
      "grad_norm": 9.334559440612793,
      "learning_rate": 1.3076841354533658e-07,
      "logits/chosen": 0.8462595343589783,
      "logits/rejected": 0.8283253908157349,
      "logps/chosen": -243.07943725585938,
      "logps/ref_chosen": -65.37794494628906,
      "logps/ref_rejected": -88.19244384765625,
      "logps/rejected": -370.8939208984375,
      "loss": 1.0988,
      "margin_dpo/margin_mean": 104.99995422363281,
      "margin_dpo/margin_std": 143.6303253173828,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -185.249755859375,
      "KL/mean": -238.11590576171875,
      "KL/rejected_KL_mean": -290.9820556640625,
      "KL/std": 130.9014129638672,
      "epoch": 0.6953892668178382,
      "fcm_dpo/beta": 0.003799508325755596,
      "fcm_dpo/delta": -0.02934153750538826,
      "fcm_dpo/margin": 105.73228454589844,
      "fcm_dpo/q_t": 0.4084155559539795,
      "grad_norm": 9.660005569458008,
      "learning_rate": 1.2960793094762345e-07,
      "logits/chosen": 0.8733299970626831,
      "logits/rejected": 0.7844957113265991,
      "logps/chosen": -249.81143188476562,
      "logps/ref_chosen": -64.5616683959961,
      "logps/ref_rejected": -88.67890167236328,
      "logps/rejected": -379.66094970703125,
      "loss": 1.0947,
      "margin_dpo/margin_mean": 105.73228454589844,
      "margin_dpo/margin_std": 144.70050048828125,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -154.37887573242188,
      "KL/mean": -207.57666015625,
      "KL/rejected_KL_mean": -260.77447509765625,
      "KL/std": 122.10684204101562,
      "epoch": 0.6969009826152683,
      "fcm_dpo/beta": 0.0037420152220875025,
      "fcm_dpo/delta": -0.07623682171106339,
      "fcm_dpo/margin": 106.39558410644531,
      "fcm_dpo/q_t": 0.4072979688644409,
      "grad_norm": 9.591710090637207,
      "learning_rate": 1.2845081597488286e-07,
      "logits/chosen": 0.9570119380950928,
      "logits/rejected": 0.8960863947868347,
      "logps/chosen": -203.85678100585938,
      "logps/ref_chosen": -49.4779167175293,
      "logps/ref_rejected": -72.65262603759766,
      "logps/rejected": -333.4271240234375,
      "loss": 1.0899,
      "margin_dpo/margin_mean": 106.39558410644531,
      "margin_dpo/margin_std": 132.27688598632812,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -168.79248046875,
      "KL/mean": -220.48451232910156,
      "KL/rejected_KL_mean": -272.1765441894531,
      "KL/std": 109.50112915039062,
      "epoch": 0.6984126984126984,
      "fcm_dpo/beta": 0.0037420152220875025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 103.38409423828125,
      "fcm_dpo/q_t": 0.4085114002227783,
      "grad_norm": 10.171971321105957,
      "learning_rate": 1.27297100994108e-07,
      "logits/chosen": 0.8515751361846924,
      "logits/rejected": 0.8222944736480713,
      "logps/chosen": -229.28758239746094,
      "logps/ref_chosen": -60.4951171875,
      "logps/ref_rejected": -74.82136535644531,
      "logps/rejected": -346.9979248046875,
      "loss": 1.0875,
      "margin_dpo/margin_mean": 103.38409423828125,
      "margin_dpo/margin_std": 123.28450012207031,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -209.23985290527344,
      "KL/mean": -237.67771911621094,
      "KL/rejected_KL_mean": -266.1156005859375,
      "KL/std": 116.87516784667969,
      "epoch": 0.6999244142101285,
      "fcm_dpo/beta": 0.0037420152220875025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 56.87575912475586,
      "fcm_dpo/q_t": 0.4490014910697937,
      "grad_norm": 16.511831283569336,
      "learning_rate": 1.2614681827718695e-07,
      "logits/chosen": 0.8656213283538818,
      "logits/rejected": 0.8616636991500854,
      "logps/chosen": -276.9249572753906,
      "logps/ref_chosen": -67.68511962890625,
      "logps/ref_rejected": -71.32196044921875,
      "logps/rejected": -337.43756103515625,
      "loss": 1.2477,
      "margin_dpo/margin_mean": 56.87575912475586,
      "margin_dpo/margin_std": 135.13449096679688,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -185.364013671875,
      "KL/mean": -234.14212036132812,
      "KL/rejected_KL_mean": -282.92022705078125,
      "KL/std": 136.61669921875,
      "epoch": 0.7014361300075586,
      "fcm_dpo/beta": 0.003737367456778884,
      "fcm_dpo/delta": -0.01243579015135765,
      "fcm_dpo/margin": 97.55622863769531,
      "fcm_dpo/q_t": 0.41499170660972595,
      "grad_norm": 9.446571350097656,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 0.8816648721694946,
      "logits/rejected": 0.8628978133201599,
      "logps/chosen": -244.52964782714844,
      "logps/ref_chosen": -59.16564178466797,
      "logps/ref_rejected": -69.56146240234375,
      "logps/rejected": -352.481689453125,
      "loss": 1.1323,
      "margin_dpo/margin_mean": 97.55621337890625,
      "margin_dpo/margin_std": 151.31185913085938,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -191.65701293945312,
      "KL/mean": -236.20742797851562,
      "KL/rejected_KL_mean": -280.7578125,
      "KL/std": 124.93704223632812,
      "epoch": 0.7029478458049887,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.100830078125,
      "fcm_dpo/q_t": 0.42286938428878784,
      "grad_norm": 11.02184009552002,
      "learning_rate": 1.238566782415197e-07,
      "logits/chosen": 0.9379351139068604,
      "logits/rejected": 0.8928974270820618,
      "logps/chosen": -250.17068481445312,
      "logps/ref_chosen": -58.513671875,
      "logps/ref_rejected": -84.31745910644531,
      "logps/rejected": -365.0752868652344,
      "loss": 1.1481,
      "margin_dpo/margin_mean": 89.10082244873047,
      "margin_dpo/margin_std": 139.1053924560547,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -214.32485961914062,
      "KL/mean": -233.25064086914062,
      "KL/rejected_KL_mean": -252.1764373779297,
      "KL/std": 137.39306640625,
      "epoch": 0.7044595616024187,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 37.851585388183594,
      "fcm_dpo/q_t": 0.4645908772945404,
      "grad_norm": 15.143999099731445,
      "learning_rate": 1.2271688498291334e-07,
      "logits/chosen": 0.8913939595222473,
      "logits/rejected": 0.8950200080871582,
      "logps/chosen": -287.5906677246094,
      "logps/ref_chosen": -73.26580810546875,
      "logps/ref_rejected": -74.83621215820312,
      "logps/rejected": -327.01263427734375,
      "loss": 1.3203,
      "margin_dpo/margin_mean": 37.85158920288086,
      "margin_dpo/margin_std": 142.350341796875,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -182.82945251464844,
      "KL/mean": -228.56005859375,
      "KL/rejected_KL_mean": -274.2906799316406,
      "KL/std": 138.61526489257812,
      "epoch": 0.7059712773998488,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.46121978759766,
      "fcm_dpo/q_t": 0.42408573627471924,
      "grad_norm": 10.50765323638916,
      "learning_rate": 1.2158065210664848e-07,
      "logits/chosen": 0.8969777822494507,
      "logits/rejected": 0.7978317737579346,
      "logps/chosen": -230.408935546875,
      "logps/ref_chosen": -47.57947540283203,
      "logps/ref_rejected": -78.68522644042969,
      "logps/rejected": -352.97589111328125,
      "loss": 1.1481,
      "margin_dpo/margin_mean": 91.46122741699219,
      "margin_dpo/margin_std": 154.33111572265625,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -180.0341033935547,
      "KL/mean": -227.7010040283203,
      "KL/rejected_KL_mean": -275.367919921875,
      "KL/std": 131.7947998046875,
      "epoch": 0.7074829931972789,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 95.33381652832031,
      "fcm_dpo/q_t": 0.4157509505748749,
      "grad_norm": 12.930656433105469,
      "learning_rate": 1.204480113956011e-07,
      "logits/chosen": 0.8496984243392944,
      "logits/rejected": 0.8403213024139404,
      "logps/chosen": -243.96188354492188,
      "logps/ref_chosen": -63.92778778076172,
      "logps/ref_rejected": -76.51626586914062,
      "logps/rejected": -351.8841857910156,
      "loss": 1.1424,
      "margin_dpo/margin_mean": 95.33382415771484,
      "margin_dpo/margin_std": 156.22494506835938,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -180.99874877929688,
      "KL/mean": -224.89773559570312,
      "KL/rejected_KL_mean": -268.7967224121094,
      "KL/std": 126.71222686767578,
      "epoch": 0.708994708994709,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 87.7979736328125,
      "fcm_dpo/q_t": 0.42392563819885254,
      "grad_norm": 12.877175331115723,
      "learning_rate": 1.1931899453216697e-07,
      "logits/chosen": 0.9460161924362183,
      "logits/rejected": 0.9314364194869995,
      "logps/chosen": -240.05694580078125,
      "logps/ref_chosen": -59.05818176269531,
      "logps/ref_rejected": -75.67672729492188,
      "logps/rejected": -344.47344970703125,
      "loss": 1.1347,
      "margin_dpo/margin_mean": 87.7979736328125,
      "margin_dpo/margin_std": 122.22289276123047,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -166.74258422851562,
      "KL/mean": -213.37454223632812,
      "KL/rejected_KL_mean": -260.0065002441406,
      "KL/std": 120.32038879394531,
      "epoch": 0.7105064247921391,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 93.263916015625,
      "fcm_dpo/q_t": 0.41877222061157227,
      "grad_norm": 11.062674522399902,
      "learning_rate": 1.1819363309737438e-07,
      "logits/chosen": 0.9049103260040283,
      "logits/rejected": 0.8607597351074219,
      "logps/chosen": -214.61001586914062,
      "logps/ref_chosen": -47.86743927001953,
      "logps/ref_rejected": -65.96859741210938,
      "logps/rejected": -325.97509765625,
      "loss": 1.1311,
      "margin_dpo/margin_mean": 93.263916015625,
      "margin_dpo/margin_std": 137.35110473632812,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -166.78622436523438,
      "KL/mean": -217.06524658203125,
      "KL/rejected_KL_mean": -267.34429931640625,
      "KL/std": 121.04692077636719,
      "epoch": 0.7120181405895691,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 100.55805969238281,
      "fcm_dpo/q_t": 0.41160130500793457,
      "grad_norm": 10.453146934509277,
      "learning_rate": 1.1707195857000215e-07,
      "logits/chosen": 0.8833752870559692,
      "logits/rejected": 0.8440606594085693,
      "logps/chosen": -224.56407165527344,
      "logps/ref_chosen": -57.777854919433594,
      "logps/ref_rejected": -73.81172180175781,
      "logps/rejected": -341.156005859375,
      "loss": 1.1057,
      "margin_dpo/margin_mean": 100.55805969238281,
      "margin_dpo/margin_std": 132.71646118164062,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -167.60638427734375,
      "KL/mean": -207.35028076171875,
      "KL/rejected_KL_mean": -247.0941925048828,
      "KL/std": 126.00967407226562,
      "epoch": 0.7135298563869993,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.48780059814453,
      "fcm_dpo/q_t": 0.43231454491615295,
      "grad_norm": 11.560592651367188,
      "learning_rate": 1.1595400232569768e-07,
      "logits/chosen": 0.8736377954483032,
      "logits/rejected": 0.8380210399627686,
      "logps/chosen": -223.51504516601562,
      "logps/ref_chosen": -55.908668518066406,
      "logps/ref_rejected": -74.70294189453125,
      "logps/rejected": -321.797119140625,
      "loss": 1.1947,
      "margin_dpo/margin_mean": 79.48780059814453,
      "margin_dpo/margin_std": 160.8380126953125,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -170.9720458984375,
      "KL/mean": -227.07763671875,
      "KL/rejected_KL_mean": -283.1832275390625,
      "KL/std": 155.66497802734375,
      "epoch": 0.7150415721844293,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 112.21117401123047,
      "fcm_dpo/q_t": 0.4082377254962921,
      "grad_norm": 12.538541793823242,
      "learning_rate": 1.1483979563610069e-07,
      "logits/chosen": 0.970649003982544,
      "logits/rejected": 0.8827682733535767,
      "logps/chosen": -225.1329345703125,
      "logps/ref_chosen": -54.16088104248047,
      "logps/ref_rejected": -92.76789855957031,
      "logps/rejected": -375.95111083984375,
      "loss": 1.1122,
      "margin_dpo/margin_mean": 112.21117401123047,
      "margin_dpo/margin_std": 179.88638305664062,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -173.3741455078125,
      "KL/mean": -217.32489013671875,
      "KL/rejected_KL_mean": -261.275634765625,
      "KL/std": 130.40184020996094,
      "epoch": 0.7165532879818595,
      "fcm_dpo/beta": 0.0037327196914702654,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 87.90148162841797,
      "fcm_dpo/q_t": 0.4254756569862366,
      "grad_norm": 16.006996154785156,
      "learning_rate": 1.1372936966796709e-07,
      "logits/chosen": 0.9420585632324219,
      "logits/rejected": 0.8917109966278076,
      "logps/chosen": -220.05984497070312,
      "logps/ref_chosen": -46.685707092285156,
      "logps/ref_rejected": -71.44731903076172,
      "logps/rejected": -332.7229309082031,
      "loss": 1.1583,
      "margin_dpo/margin_mean": 87.90147399902344,
      "margin_dpo/margin_std": 150.548828125,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -173.3091583251953,
      "KL/mean": -232.6639404296875,
      "KL/rejected_KL_mean": -292.01873779296875,
      "KL/std": 132.6728057861328,
      "epoch": 0.7180650037792895,
      "fcm_dpo/beta": 0.003699835389852524,
      "fcm_dpo/delta": -0.044244010001420975,
      "fcm_dpo/margin": 118.70957946777344,
      "fcm_dpo/q_t": 0.4001288414001465,
      "grad_norm": 7.808788299560547,
      "learning_rate": 1.126227554822985e-07,
      "logits/chosen": 0.9036856889724731,
      "logits/rejected": 0.8648196458816528,
      "logps/chosen": -231.7964630126953,
      "logps/ref_chosen": -58.4873046875,
      "logps/ref_rejected": -87.00187683105469,
      "logps/rejected": -379.0206298828125,
      "loss": 1.0693,
      "margin_dpo/margin_mean": 118.70957946777344,
      "margin_dpo/margin_std": 152.59393310546875,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -208.72824096679688,
      "KL/mean": -243.93551635742188,
      "KL/rejected_KL_mean": -279.1427917480469,
      "KL/std": 144.14910888671875,
      "epoch": 0.7195767195767195,
      "fcm_dpo/beta": 0.003699835389852524,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.41454315185547,
      "fcm_dpo/q_t": 0.4408169984817505,
      "grad_norm": 12.431097030639648,
      "learning_rate": 1.1151998403347243e-07,
      "logits/chosen": 0.8004306554794312,
      "logits/rejected": 0.8054685592651367,
      "logps/chosen": -284.10986328125,
      "logps/ref_chosen": -75.38162231445312,
      "logps/ref_rejected": -76.99822235107422,
      "logps/rejected": -356.1410217285156,
      "loss": 1.2173,
      "margin_dpo/margin_mean": 70.41454315185547,
      "margin_dpo/margin_std": 151.20233154296875,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -202.79141235351562,
      "KL/mean": -247.09869384765625,
      "KL/rejected_KL_mean": -291.40594482421875,
      "KL/std": 144.90768432617188,
      "epoch": 0.7210884353741497,
      "fcm_dpo/beta": 0.003699835389852524,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.61453247070312,
      "fcm_dpo/q_t": 0.4267289936542511,
      "grad_norm": 11.736209869384766,
      "learning_rate": 1.1042108616837692e-07,
      "logits/chosen": 0.8813755512237549,
      "logits/rejected": 0.8460351228713989,
      "logps/chosen": -263.86480712890625,
      "logps/ref_chosen": -61.073387145996094,
      "logps/ref_rejected": -81.34375,
      "logps/rejected": -372.74969482421875,
      "loss": 1.1971,
      "margin_dpo/margin_mean": 88.61453247070312,
      "margin_dpo/margin_std": 187.2801055908203,
      "step": 477
    },
    {
      "KL/chosen_KL_mean": -178.40023803710938,
      "KL/mean": -209.76089477539062,
      "KL/rejected_KL_mean": -241.12158203125,
      "KL/std": 112.0628890991211,
      "epoch": 0.7226001511715797,
      "fcm_dpo/beta": 0.003699835389852524,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.721343994140625,
      "fcm_dpo/q_t": 0.44511181116104126,
      "grad_norm": 11.34143352508545,
      "learning_rate": 1.0932609262554746e-07,
      "logits/chosen": 0.8605385422706604,
      "logits/rejected": 0.8723839521408081,
      "logps/chosen": -235.56756591796875,
      "logps/ref_chosen": -57.16731643676758,
      "logps/ref_rejected": -53.30917739868164,
      "logps/rejected": -294.4307556152344,
      "loss": 1.2306,
      "margin_dpo/margin_mean": 62.72134017944336,
      "margin_dpo/margin_std": 137.10507202148438,
      "step": 478
    },
    {
      "KL/chosen_KL_mean": -186.5111541748047,
      "KL/mean": -215.21243286132812,
      "KL/rejected_KL_mean": -243.91368103027344,
      "KL/std": 118.79924774169922,
      "epoch": 0.7241118669690099,
      "fcm_dpo/beta": 0.003699835389852524,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.40254211425781,
      "fcm_dpo/q_t": 0.45077085494995117,
      "grad_norm": 11.206938743591309,
      "learning_rate": 1.0823503403430734e-07,
      "logits/chosen": 0.871839165687561,
      "logits/rejected": 0.8502979874610901,
      "logps/chosen": -245.42446899414062,
      "logps/ref_chosen": -58.91331481933594,
      "logps/ref_rejected": -63.7403450012207,
      "logps/rejected": -307.654052734375,
      "loss": 1.2523,
      "margin_dpo/margin_mean": 57.40254211425781,
      "margin_dpo/margin_std": 141.32350158691406,
      "step": 479
    },
    {
      "KL/chosen_KL_mean": -193.8066864013672,
      "KL/mean": -237.73788452148438,
      "KL/rejected_KL_mean": -281.6690673828125,
      "KL/std": 121.91385650634766,
      "epoch": 0.7256235827664399,
      "fcm_dpo/beta": 0.003685213625431061,
      "fcm_dpo/delta": -0.03967710956931114,
      "fcm_dpo/margin": 87.86239624023438,
      "fcm_dpo/q_t": 0.4248046875,
      "grad_norm": 13.300537109375,
      "learning_rate": 1.0714794091391072e-07,
      "logits/chosen": 0.8580498099327087,
      "logits/rejected": 0.8567924499511719,
      "logps/chosen": -256.6072998046875,
      "logps/ref_chosen": -62.80061340332031,
      "logps/ref_rejected": -67.58859252929688,
      "logps/rejected": -349.2576599121094,
      "loss": 1.1564,
      "margin_dpo/margin_mean": 87.86239624023438,
      "margin_dpo/margin_std": 139.7371826171875,
      "step": 480
    },
    {
      "KL/chosen_KL_mean": -193.44094848632812,
      "KL/mean": -233.70297241210938,
      "KL/rejected_KL_mean": -273.96502685546875,
      "KL/std": 130.8161163330078,
      "epoch": 0.72713529856387,
      "fcm_dpo/beta": 0.0036705918610095978,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.52407836914062,
      "fcm_dpo/q_t": 0.43242886662483215,
      "grad_norm": 10.15783977508545,
      "learning_rate": 1.0606484367268906e-07,
      "logits/chosen": 0.8588245511054993,
      "logits/rejected": 0.8666401505470276,
      "logps/chosen": -258.7274169921875,
      "logps/ref_chosen": -65.28649139404297,
      "logps/ref_rejected": -70.78668212890625,
      "logps/rejected": -344.751708984375,
      "loss": 1.1766,
      "margin_dpo/margin_mean": 80.52407836914062,
      "margin_dpo/margin_std": 143.02203369140625,
      "step": 481
    },
    {
      "KL/chosen_KL_mean": -211.6179962158203,
      "KL/mean": -260.3857116699219,
      "KL/rejected_KL_mean": -309.1534423828125,
      "KL/std": 149.3299560546875,
      "epoch": 0.7286470143613001,
      "fcm_dpo/beta": 0.0036705918610095978,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 97.53544616699219,
      "fcm_dpo/q_t": 0.4218747317790985,
      "grad_norm": 11.276646614074707,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 0.8682383298873901,
      "logits/rejected": 0.7406269907951355,
      "logps/chosen": -272.524169921875,
      "logps/ref_chosen": -60.906185150146484,
      "logps/ref_rejected": -103.44656372070312,
      "logps/rejected": -412.6000061035156,
      "loss": 1.1612,
      "margin_dpo/margin_mean": 97.53544616699219,
      "margin_dpo/margin_std": 183.6652069091797,
      "step": 482
    },
    {
      "KL/chosen_KL_mean": -177.52684020996094,
      "KL/mean": -231.13912963867188,
      "KL/rejected_KL_mean": -284.75140380859375,
      "KL/std": 125.70189666748047,
      "epoch": 0.7301587301587301,
      "fcm_dpo/beta": 0.003663320792838931,
      "fcm_dpo/delta": -0.01984819769859314,
      "fcm_dpo/margin": 107.22454833984375,
      "fcm_dpo/q_t": 0.4089610278606415,
      "grad_norm": 11.108445167541504,
      "learning_rate": 1.0391075790138232e-07,
      "logits/chosen": 0.9367642402648926,
      "logits/rejected": 0.8554052114486694,
      "logps/chosen": -230.71885681152344,
      "logps/ref_chosen": -53.192012786865234,
      "logps/ref_rejected": -81.83927154541016,
      "logps/rejected": -366.5906677246094,
      "loss": 1.0983,
      "margin_dpo/margin_mean": 107.22454833984375,
      "margin_dpo/margin_std": 145.13848876953125,
      "step": 483
    },
    {
      "KL/chosen_KL_mean": -186.06248474121094,
      "KL/mean": -219.39501953125,
      "KL/rejected_KL_mean": -252.72755432128906,
      "KL/std": 125.95346069335938,
      "epoch": 0.7316704459561603,
      "fcm_dpo/beta": 0.0036560497246682644,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.66506958007812,
      "fcm_dpo/q_t": 0.4416685998439789,
      "grad_norm": 13.961874961853027,
      "learning_rate": 1.0283982962570681e-07,
      "logits/chosen": 0.938569188117981,
      "logits/rejected": 0.9175790548324585,
      "logps/chosen": -243.83193969726562,
      "logps/ref_chosen": -57.76945877075195,
      "logps/ref_rejected": -71.6829833984375,
      "logps/rejected": -324.4105224609375,
      "loss": 1.1964,
      "margin_dpo/margin_mean": 66.66506958007812,
      "margin_dpo/margin_std": 109.29454803466797,
      "step": 484
    },
    {
      "KL/chosen_KL_mean": -191.56036376953125,
      "KL/mean": -228.56719970703125,
      "KL/rejected_KL_mean": -265.57403564453125,
      "KL/std": 122.90736389160156,
      "epoch": 0.7331821617535903,
      "fcm_dpo/beta": 0.0036560497246682644,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.01366424560547,
      "fcm_dpo/q_t": 0.4370569586753845,
      "grad_norm": 11.334329605102539,
      "learning_rate": 1.0177301773633992e-07,
      "logits/chosen": 0.9071985483169556,
      "logits/rejected": 0.896731972694397,
      "logps/chosen": -248.19619750976562,
      "logps/ref_chosen": -56.63584899902344,
      "logps/ref_rejected": -70.85614013671875,
      "logps/rejected": -336.43017578125,
      "loss": 1.1859,
      "margin_dpo/margin_mean": 74.013671875,
      "margin_dpo/margin_std": 123.91354370117188,
      "step": 485
    },
    {
      "KL/chosen_KL_mean": -215.73367309570312,
      "KL/mean": -254.3745574951172,
      "KL/rejected_KL_mean": -293.01544189453125,
      "KL/std": 147.834716796875,
      "epoch": 0.7346938775510204,
      "fcm_dpo/beta": 0.0036560497246682644,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 77.28172302246094,
      "fcm_dpo/q_t": 0.4370615780353546,
      "grad_norm": 9.572896957397461,
      "learning_rate": 1.007103520743035e-07,
      "logits/chosen": 0.9285538196563721,
      "logits/rejected": 0.8437390923500061,
      "logps/chosen": -272.0806884765625,
      "logps/ref_chosen": -56.347023010253906,
      "logps/ref_rejected": -85.97221374511719,
      "logps/rejected": -378.9876403808594,
      "loss": 1.2158,
      "margin_dpo/margin_mean": 77.28172302246094,
      "margin_dpo/margin_std": 172.080078125,
      "step": 486
    },
    {
      "KL/chosen_KL_mean": -195.94334411621094,
      "KL/mean": -240.12217712402344,
      "KL/rejected_KL_mean": -284.301025390625,
      "KL/std": 138.95465087890625,
      "epoch": 0.7362055933484505,
      "fcm_dpo/beta": 0.0036560497246682644,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.35769653320312,
      "fcm_dpo/q_t": 0.42905205488204956,
      "grad_norm": 10.672550201416016,
      "learning_rate": 9.965186236464046e-08,
      "logits/chosen": 0.9684932231903076,
      "logits/rejected": 0.9252954721450806,
      "logps/chosen": -256.560546875,
      "logps/ref_chosen": -60.617218017578125,
      "logps/ref_rejected": -82.50975036621094,
      "logps/rejected": -366.810791015625,
      "loss": 1.1715,
      "margin_dpo/margin_mean": 88.35769653320312,
      "margin_dpo/margin_std": 167.52935791015625,
      "step": 487
    },
    {
      "KL/chosen_KL_mean": -182.96995544433594,
      "KL/mean": -233.22586059570312,
      "KL/rejected_KL_mean": -283.48175048828125,
      "KL/std": 129.29685974121094,
      "epoch": 0.7377173091458806,
      "fcm_dpo/beta": 0.003645846154540777,
      "fcm_dpo/delta": -0.027987273409962654,
      "fcm_dpo/margin": 100.51178741455078,
      "fcm_dpo/q_t": 0.4163171947002411,
      "grad_norm": 13.601997375488281,
      "learning_rate": 9.859757821558337e-08,
      "logits/chosen": 0.8888124227523804,
      "logits/rejected": 0.8562193512916565,
      "logps/chosen": -246.07901000976562,
      "logps/ref_chosen": -63.10905075073242,
      "logps/ref_rejected": -82.49348449707031,
      "logps/rejected": -365.9752197265625,
      "loss": 1.1183,
      "margin_dpo/margin_mean": 100.51178741455078,
      "margin_dpo/margin_std": 144.21810913085938,
      "step": 488
    },
    {
      "KL/chosen_KL_mean": -216.40370178222656,
      "KL/mean": -245.2534637451172,
      "KL/rejected_KL_mean": -274.10321044921875,
      "KL/std": 141.31771850585938,
      "epoch": 0.7392290249433107,
      "fcm_dpo/beta": 0.0036356423515826464,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 57.69950485229492,
      "fcm_dpo/q_t": 0.4508482813835144,
      "grad_norm": 9.973169326782227,
      "learning_rate": 9.754752911772615e-08,
      "logits/chosen": 0.8845810890197754,
      "logits/rejected": 0.8505979776382446,
      "logps/chosen": -281.3926696777344,
      "logps/ref_chosen": -64.98896026611328,
      "logps/ref_rejected": -84.39607238769531,
      "logps/rejected": -358.4992980957031,
      "loss": 1.2604,
      "margin_dpo/margin_mean": 57.69950866699219,
      "margin_dpo/margin_std": 151.286865234375,
      "step": 489
    },
    {
      "KL/chosen_KL_mean": -193.18675231933594,
      "KL/mean": -229.94459533691406,
      "KL/rejected_KL_mean": -266.7024230957031,
      "KL/std": 135.72817993164062,
      "epoch": 0.7407407407407407,
      "fcm_dpo/beta": 0.0036356423515826464,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.51567077636719,
      "fcm_dpo/q_t": 0.43982094526290894,
      "grad_norm": 9.761716842651367,
      "learning_rate": 9.650174444319956e-08,
      "logits/chosen": 0.9422023296356201,
      "logits/rejected": 0.9359936714172363,
      "logps/chosen": -255.09548950195312,
      "logps/ref_chosen": -61.90874481201172,
      "logps/ref_rejected": -70.58566284179688,
      "logps/rejected": -337.2880859375,
      "loss": 1.2254,
      "margin_dpo/margin_mean": 73.51567077636719,
      "margin_dpo/margin_std": 169.10879516601562,
      "step": 490
    },
    {
      "KL/chosen_KL_mean": -186.3331298828125,
      "KL/mean": -222.13389587402344,
      "KL/rejected_KL_mean": -257.9346618652344,
      "KL/std": 120.4219741821289,
      "epoch": 0.7422524565381708,
      "fcm_dpo/beta": 0.0036356423515826464,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.60151672363281,
      "fcm_dpo/q_t": 0.43975383043289185,
      "grad_norm": 8.621192932128906,
      "learning_rate": 9.546025344484868e-08,
      "logits/chosen": 0.8622674942016602,
      "logits/rejected": 0.811538577079773,
      "logps/chosen": -241.808837890625,
      "logps/ref_chosen": -55.47570037841797,
      "logps/ref_rejected": -78.70318603515625,
      "logps/rejected": -336.6378173828125,
      "loss": 1.1981,
      "margin_dpo/margin_mean": 71.60152435302734,
      "margin_dpo/margin_std": 129.99026489257812,
      "step": 491
    },
    {
      "KL/chosen_KL_mean": -220.4925537109375,
      "KL/mean": -255.36636352539062,
      "KL/rejected_KL_mean": -290.2402038574219,
      "KL/std": 143.24710083007812,
      "epoch": 0.7437641723356009,
      "fcm_dpo/beta": 0.0036356423515826464,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 69.74764251708984,
      "fcm_dpo/q_t": 0.44249725341796875,
      "grad_norm": 12.263677597045898,
      "learning_rate": 9.442308525541589e-08,
      "logits/chosen": 0.8829400539398193,
      "logits/rejected": 0.8344748020172119,
      "logps/chosen": -287.7789306640625,
      "logps/ref_chosen": -67.28638458251953,
      "logps/ref_rejected": -82.78628540039062,
      "logps/rejected": -373.0264892578125,
      "loss": 1.2368,
      "margin_dpo/margin_mean": 69.74765014648438,
      "margin_dpo/margin_std": 165.14651489257812,
      "step": 492
    },
    {
      "KL/chosen_KL_mean": -184.6879425048828,
      "KL/mean": -233.34588623046875,
      "KL/rejected_KL_mean": -282.0038146972656,
      "KL/std": 140.0909423828125,
      "epoch": 0.745275888133031,
      "fcm_dpo/beta": 0.0036191383842378855,
      "fcm_dpo/delta": -0.045602187514305115,
      "fcm_dpo/margin": 97.31587219238281,
      "fcm_dpo/q_t": 0.42029207944869995,
      "grad_norm": 10.198504447937012,
      "learning_rate": 9.339026888672468e-08,
      "logits/chosen": 0.8781353831291199,
      "logits/rejected": 0.8281064033508301,
      "logps/chosen": -240.61544799804688,
      "logps/ref_chosen": -55.92750549316406,
      "logps/ref_rejected": -79.12149810791016,
      "logps/rejected": -361.12530517578125,
      "loss": 1.1388,
      "margin_dpo/margin_mean": 97.31587219238281,
      "margin_dpo/margin_std": 152.83680725097656,
      "step": 493
    },
    {
      "KL/chosen_KL_mean": -191.65391540527344,
      "KL/mean": -236.73977661132812,
      "KL/rejected_KL_mean": -281.82568359375,
      "KL/std": 144.91470336914062,
      "epoch": 0.7467876039304611,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 90.17176055908203,
      "fcm_dpo/q_t": 0.42782270908355713,
      "grad_norm": 12.270206451416016,
      "learning_rate": 9.236183322886945e-08,
      "logits/chosen": 0.8384512066841125,
      "logits/rejected": 0.806420087814331,
      "logps/chosen": -259.6080322265625,
      "logps/ref_chosen": -67.95410919189453,
      "logps/ref_rejected": -90.50865173339844,
      "logps/rejected": -372.3343505859375,
      "loss": 1.1845,
      "margin_dpo/margin_mean": 90.17176818847656,
      "margin_dpo/margin_std": 181.9389190673828,
      "step": 494
    },
    {
      "KL/chosen_KL_mean": -183.88514709472656,
      "KL/mean": -219.1983642578125,
      "KL/rejected_KL_mean": -254.51158142089844,
      "KL/std": 138.21241760253906,
      "epoch": 0.7482993197278912,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 70.62643432617188,
      "fcm_dpo/q_t": 0.4458218216896057,
      "grad_norm": 14.293071746826172,
      "learning_rate": 9.133780704940594e-08,
      "logits/chosen": 0.9193370938301086,
      "logits/rejected": 0.8782978653907776,
      "logps/chosen": -236.5106201171875,
      "logps/ref_chosen": -52.62546157836914,
      "logps/ref_rejected": -72.06781005859375,
      "logps/rejected": -326.57940673828125,
      "loss": 1.237,
      "margin_dpo/margin_mean": 70.62643432617188,
      "margin_dpo/margin_std": 175.55023193359375,
      "step": 495
    },
    {
      "KL/chosen_KL_mean": -198.95504760742188,
      "KL/mean": -249.46466064453125,
      "KL/rejected_KL_mean": -299.9742431640625,
      "KL/std": 150.4727325439453,
      "epoch": 0.7498110355253212,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 101.01922607421875,
      "fcm_dpo/q_t": 0.4242492914199829,
      "grad_norm": 9.73660659790039,
      "learning_rate": 9.031821899254797e-08,
      "logits/chosen": 0.9211750626564026,
      "logits/rejected": 0.8287931680679321,
      "logps/chosen": -256.5523681640625,
      "logps/ref_chosen": -57.597320556640625,
      "logps/ref_rejected": -94.36127471923828,
      "logps/rejected": -394.3355407714844,
      "loss": 1.1633,
      "margin_dpo/margin_mean": 101.01922607421875,
      "margin_dpo/margin_std": 196.87779235839844,
      "step": 496
    },
    {
      "KL/chosen_KL_mean": -206.773193359375,
      "KL/mean": -251.5380859375,
      "KL/rejected_KL_mean": -296.302978515625,
      "KL/std": 133.81814575195312,
      "epoch": 0.7513227513227513,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.52980041503906,
      "fcm_dpo/q_t": 0.42446810007095337,
      "grad_norm": 10.825132369995117,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": 0.9078977108001709,
      "logits/rejected": 0.8813755512237549,
      "logps/chosen": -279.5631408691406,
      "logps/ref_chosen": -72.78994750976562,
      "logps/ref_rejected": -89.48483276367188,
      "logps/rejected": -385.787841796875,
      "loss": 1.1571,
      "margin_dpo/margin_mean": 89.52980041503906,
      "margin_dpo/margin_std": 148.03123474121094,
      "step": 497
    },
    {
      "KL/chosen_KL_mean": -182.50619506835938,
      "KL/mean": -229.74530029296875,
      "KL/rejected_KL_mean": -276.98443603515625,
      "KL/std": 126.8770751953125,
      "epoch": 0.7528344671201814,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.47825622558594,
      "fcm_dpo/q_t": 0.4190598726272583,
      "grad_norm": 11.309755325317383,
      "learning_rate": 8.829247120198563e-08,
      "logits/chosen": 0.8609334230422974,
      "logits/rejected": 0.8546430468559265,
      "logps/chosen": -250.8719024658203,
      "logps/ref_chosen": -68.36572265625,
      "logps/ref_rejected": -71.28846740722656,
      "logps/rejected": -348.27288818359375,
      "loss": 1.1377,
      "margin_dpo/margin_mean": 94.47825622558594,
      "margin_dpo/margin_std": 142.64303588867188,
      "step": 498
    },
    {
      "KL/chosen_KL_mean": -186.2841796875,
      "KL/mean": -240.4781494140625,
      "KL/rejected_KL_mean": -294.672119140625,
      "KL/std": 144.49862670898438,
      "epoch": 0.7543461829176115,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 108.387939453125,
      "fcm_dpo/q_t": 0.4136818051338196,
      "grad_norm": 14.066987991333008,
      "learning_rate": 8.728636813280163e-08,
      "logits/chosen": 0.8882684111595154,
      "logits/rejected": 0.8408522605895996,
      "logps/chosen": -248.19300842285156,
      "logps/ref_chosen": -61.90882873535156,
      "logps/ref_rejected": -91.9411392211914,
      "logps/rejected": -386.61328125,
      "loss": 1.1433,
      "margin_dpo/margin_mean": 108.387939453125,
      "margin_dpo/margin_std": 191.2751007080078,
      "step": 499
    },
    {
      "KL/chosen_KL_mean": -196.25576782226562,
      "KL/mean": -235.48272705078125,
      "KL/rejected_KL_mean": -274.7096862792969,
      "KL/std": 126.16201782226562,
      "epoch": 0.7558578987150416,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 78.45390319824219,
      "fcm_dpo/q_t": 0.43167075514793396,
      "grad_norm": 11.740747451782227,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.84987473487854,
      "logits/rejected": 0.866410493850708,
      "logps/chosen": -266.4815979003906,
      "logps/ref_chosen": -70.225830078125,
      "logps/ref_rejected": -71.72203063964844,
      "logps/rejected": -346.43170166015625,
      "loss": 1.202,
      "margin_dpo/margin_mean": 78.45390319824219,
      "margin_dpo/margin_std": 159.15243530273438,
      "step": 500
    },
    {
      "KL/chosen_KL_mean": -196.87637329101562,
      "KL/mean": -232.41775512695312,
      "KL/rejected_KL_mean": -267.95916748046875,
      "KL/std": 131.5418701171875,
      "epoch": 0.7573696145124716,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 71.08279418945312,
      "fcm_dpo/q_t": 0.43857502937316895,
      "grad_norm": 9.775738716125488,
      "learning_rate": 8.528784436016878e-08,
      "logits/chosen": 0.9283879995346069,
      "logits/rejected": 0.9351827502250671,
      "logps/chosen": -261.4751892089844,
      "logps/ref_chosen": -64.59880828857422,
      "logps/ref_rejected": -70.59329223632812,
      "logps/rejected": -338.5524597167969,
      "loss": 1.1816,
      "margin_dpo/margin_mean": 71.08279418945312,
      "margin_dpo/margin_std": 105.03961944580078,
      "step": 501
    },
    {
      "KL/chosen_KL_mean": -197.69915771484375,
      "KL/mean": -237.3424072265625,
      "KL/rejected_KL_mean": -276.98565673828125,
      "KL/std": 147.6724853515625,
      "epoch": 0.7588813303099018,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.2864761352539,
      "fcm_dpo/q_t": 0.4334084391593933,
      "grad_norm": 12.1803617477417,
      "learning_rate": 8.4295479559726e-08,
      "logits/chosen": 0.9151396751403809,
      "logits/rejected": 0.8784686326980591,
      "logps/chosen": -263.165771484375,
      "logps/ref_chosen": -65.46662902832031,
      "logps/ref_rejected": -90.22233581542969,
      "logps/rejected": -367.2079772949219,
      "loss": 1.1803,
      "margin_dpo/margin_mean": 79.28648376464844,
      "margin_dpo/margin_std": 137.8919219970703,
      "step": 502
    },
    {
      "KL/chosen_KL_mean": -170.47628784179688,
      "KL/mean": -209.97726440429688,
      "KL/rejected_KL_mean": -249.47824096679688,
      "KL/std": 115.76939392089844,
      "epoch": 0.7603930461073318,
      "fcm_dpo/beta": 0.0036026344168931246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.001953125,
      "fcm_dpo/q_t": 0.43191272020339966,
      "grad_norm": 9.445923805236816,
      "learning_rate": 8.330774987092712e-08,
      "logits/chosen": 0.85642409324646,
      "logits/rejected": 0.8654800653457642,
      "logps/chosen": -222.31106567382812,
      "logps/ref_chosen": -51.83476257324219,
      "logps/ref_rejected": -57.62522506713867,
      "logps/rejected": -307.10345458984375,
      "loss": 1.1774,
      "margin_dpo/margin_mean": 79.00196075439453,
      "margin_dpo/margin_std": 132.04278564453125,
      "step": 503
    },
    {
      "KL/chosen_KL_mean": -182.04151916503906,
      "KL/mean": -233.4086151123047,
      "KL/rejected_KL_mean": -284.77569580078125,
      "KL/std": 120.95684814453125,
      "epoch": 0.7619047619047619,
      "fcm_dpo/beta": 0.0035869048442691565,
      "fcm_dpo/delta": -0.04385312646627426,
      "fcm_dpo/margin": 102.73417663574219,
      "fcm_dpo/q_t": 0.41299694776535034,
      "grad_norm": 8.542745590209961,
      "learning_rate": 8.232468292269479e-08,
      "logits/chosen": 0.8525612354278564,
      "logits/rejected": 0.8485938906669617,
      "logps/chosen": -250.69271850585938,
      "logps/ref_chosen": -68.65119934082031,
      "logps/ref_rejected": -77.91394805908203,
      "logps/rejected": -362.68963623046875,
      "loss": 1.1027,
      "margin_dpo/margin_mean": 102.73417663574219,
      "margin_dpo/margin_std": 125.10858154296875,
      "step": 504
    },
    {
      "KL/chosen_KL_mean": -193.14846801757812,
      "KL/mean": -230.13601684570312,
      "KL/rejected_KL_mean": -267.12359619140625,
      "KL/std": 149.0281219482422,
      "epoch": 0.763416477702192,
      "fcm_dpo/beta": 0.0035711752716451883,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.97512817382812,
      "fcm_dpo/q_t": 0.4398888349533081,
      "grad_norm": 11.520407676696777,
      "learning_rate": 8.134630621352483e-08,
      "logits/chosen": 0.8992654085159302,
      "logits/rejected": 0.8699280023574829,
      "logps/chosen": -253.14730834960938,
      "logps/ref_chosen": -59.99884796142578,
      "logps/ref_rejected": -76.88048553466797,
      "logps/rejected": -344.00408935546875,
      "loss": 1.2187,
      "margin_dpo/margin_mean": 73.97512817382812,
      "margin_dpo/margin_std": 160.25271606445312,
      "step": 505
    },
    {
      "KL/chosen_KL_mean": -194.79971313476562,
      "KL/mean": -233.82147216796875,
      "KL/rejected_KL_mean": -272.8432312011719,
      "KL/std": 128.04318237304688,
      "epoch": 0.764928193499622,
      "fcm_dpo/beta": 0.0035711752716451883,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 78.04351806640625,
      "fcm_dpo/q_t": 0.43447548151016235,
      "grad_norm": 10.453583717346191,
      "learning_rate": 8.037264711071698e-08,
      "logits/chosen": 0.8826917409896851,
      "logits/rejected": 0.8774870038032532,
      "logps/chosen": -264.8710021972656,
      "logps/ref_chosen": -70.07130432128906,
      "logps/ref_rejected": -82.03775024414062,
      "logps/rejected": -354.8809814453125,
      "loss": 1.206,
      "margin_dpo/margin_mean": 78.04351806640625,
      "margin_dpo/margin_std": 160.6054229736328,
      "step": 506
    },
    {
      "KL/chosen_KL_mean": -204.0850372314453,
      "KL/mean": -251.54730224609375,
      "KL/rejected_KL_mean": -299.00958251953125,
      "KL/std": 149.53985595703125,
      "epoch": 0.7664399092970522,
      "fcm_dpo/beta": 0.0035711752716451883,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.924560546875,
      "fcm_dpo/q_t": 0.42609667778015137,
      "grad_norm": 11.234076499938965,
      "learning_rate": 7.940373284960933e-08,
      "logits/chosen": 0.8698856830596924,
      "logits/rejected": 0.8384686708450317,
      "logps/chosen": -276.0920715332031,
      "logps/ref_chosen": -72.00703430175781,
      "logps/ref_rejected": -93.94987487792969,
      "logps/rejected": -392.95947265625,
      "loss": 1.1703,
      "margin_dpo/margin_mean": 94.92456817626953,
      "margin_dpo/margin_std": 175.51181030273438,
      "step": 507
    },
    {
      "KL/chosen_KL_mean": -188.01617431640625,
      "KL/mean": -245.24417114257812,
      "KL/rejected_KL_mean": -302.47216796875,
      "KL/std": 159.1145782470703,
      "epoch": 0.7679516250944822,
      "fcm_dpo/beta": 0.0035516000352799892,
      "fcm_dpo/delta": -0.055117167532444,
      "fcm_dpo/margin": 114.45600128173828,
      "fcm_dpo/q_t": 0.4093967080116272,
      "grad_norm": 12.474221229553223,
      "learning_rate": 7.843959053281663e-08,
      "logits/chosen": 0.8432912230491638,
      "logits/rejected": 0.7599025964736938,
      "logps/chosen": -248.23609924316406,
      "logps/ref_chosen": -60.21992492675781,
      "logps/ref_rejected": -95.9200668334961,
      "logps/rejected": -398.3922424316406,
      "loss": 1.1163,
      "margin_dpo/margin_mean": 114.45600891113281,
      "margin_dpo/margin_std": 178.3321990966797,
      "step": 508
    },
    {
      "KL/chosen_KL_mean": -199.39761352539062,
      "KL/mean": -237.2610626220703,
      "KL/rejected_KL_mean": -275.1245422363281,
      "KL/std": 134.06805419921875,
      "epoch": 0.7694633408919124,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 75.72692108154297,
      "fcm_dpo/q_t": 0.43684712052345276,
      "grad_norm": 12.999882698059082,
      "learning_rate": 7.748024712947204e-08,
      "logits/chosen": 0.8402736783027649,
      "logits/rejected": 0.8276892900466919,
      "logps/chosen": -265.66778564453125,
      "logps/ref_chosen": -66.27017211914062,
      "logps/ref_rejected": -71.73065185546875,
      "logps/rejected": -346.85516357421875,
      "loss": 1.2142,
      "margin_dpo/margin_mean": 75.72692108154297,
      "margin_dpo/margin_std": 160.88034057617188,
      "step": 509
    },
    {
      "KL/chosen_KL_mean": -196.36257934570312,
      "KL/mean": -249.74359130859375,
      "KL/rejected_KL_mean": -303.1246032714844,
      "KL/std": 156.79202270507812,
      "epoch": 0.7709750566893424,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 106.76203155517578,
      "fcm_dpo/q_t": 0.4201943874359131,
      "grad_norm": 10.616975784301758,
      "learning_rate": 7.652572947447272e-08,
      "logits/chosen": 0.9781365990638733,
      "logits/rejected": 0.9099892377853394,
      "logps/chosen": -249.90745544433594,
      "logps/ref_chosen": -53.54487609863281,
      "logps/ref_rejected": -91.36648559570312,
      "logps/rejected": -394.4910888671875,
      "loss": 1.1591,
      "margin_dpo/margin_mean": 106.76203155517578,
      "margin_dpo/margin_std": 205.65582275390625,
      "step": 510
    },
    {
      "KL/chosen_KL_mean": -184.873046875,
      "KL/mean": -241.27972412109375,
      "KL/rejected_KL_mean": -297.6864013671875,
      "KL/std": 131.96929931640625,
      "epoch": 0.7724867724867724,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 112.81336212158203,
      "fcm_dpo/q_t": 0.4065815210342407,
      "grad_norm": 12.444725036621094,
      "learning_rate": 7.557606426772961e-08,
      "logits/chosen": 0.9038701057434082,
      "logits/rejected": 0.8630021810531616,
      "logps/chosen": -240.71743774414062,
      "logps/ref_chosen": -55.844383239746094,
      "logps/ref_rejected": -86.49819946289062,
      "logps/rejected": -384.1846008300781,
      "loss": 1.0837,
      "margin_dpo/margin_mean": 112.81336212158203,
      "margin_dpo/margin_std": 135.35617065429688,
      "step": 511
    },
    {
      "KL/chosen_KL_mean": -194.4578857421875,
      "KL/mean": -227.50833129882812,
      "KL/rejected_KL_mean": -260.5587463378906,
      "KL/std": 124.75822448730469,
      "epoch": 0.7739984882842026,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.1008529663086,
      "fcm_dpo/q_t": 0.4442126154899597,
      "grad_norm": 18.04234504699707,
      "learning_rate": 7.463127807341966e-08,
      "logits/chosen": 0.8226577043533325,
      "logits/rejected": 0.8273448944091797,
      "logps/chosen": -256.1109313964844,
      "logps/ref_chosen": -61.653038024902344,
      "logps/ref_rejected": -72.83148193359375,
      "logps/rejected": -333.3902587890625,
      "loss": 1.2403,
      "margin_dpo/margin_mean": 66.1008529663086,
      "margin_dpo/margin_std": 154.18467712402344,
      "step": 512
    },
    {
      "KL/chosen_KL_mean": -172.45489501953125,
      "KL/mean": -215.680419921875,
      "KL/rejected_KL_mean": -258.90594482421875,
      "KL/std": 121.78617858886719,
      "epoch": 0.7755102040816326,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 86.45104217529297,
      "fcm_dpo/q_t": 0.42885059118270874,
      "grad_norm": 8.722911834716797,
      "learning_rate": 7.369139731924401e-08,
      "logits/chosen": 0.9772987365722656,
      "logits/rejected": 0.9377813339233398,
      "logps/chosen": -223.30746459960938,
      "logps/ref_chosen": -50.85256576538086,
      "logps/ref_rejected": -69.21754455566406,
      "logps/rejected": -328.1235046386719,
      "loss": 1.1564,
      "margin_dpo/margin_mean": 86.4510498046875,
      "margin_dpo/margin_std": 131.8882598876953,
      "step": 513
    },
    {
      "KL/chosen_KL_mean": -196.28350830078125,
      "KL/mean": -246.86834716796875,
      "KL/rejected_KL_mean": -297.45318603515625,
      "KL/std": 137.37765502929688,
      "epoch": 0.7770219198790628,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 101.169677734375,
      "fcm_dpo/q_t": 0.41767847537994385,
      "grad_norm": 14.140531539916992,
      "learning_rate": 7.275644829568747e-08,
      "logits/chosen": 0.8947337865829468,
      "logits/rejected": 0.8782483339309692,
      "logps/chosen": -265.66845703125,
      "logps/ref_chosen": -69.38493347167969,
      "logps/ref_rejected": -83.32447814941406,
      "logps/rejected": -380.7776794433594,
      "loss": 1.133,
      "margin_dpo/margin_mean": 101.169677734375,
      "margin_dpo/margin_std": 156.6019287109375,
      "step": 514
    },
    {
      "KL/chosen_KL_mean": -201.03335571289062,
      "KL/mean": -242.63894653320312,
      "KL/rejected_KL_mean": -284.2445373535156,
      "KL/std": 127.74497985839844,
      "epoch": 0.7785336356764928,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.21117401123047,
      "fcm_dpo/q_t": 0.4302670359611511,
      "grad_norm": 9.764320373535156,
      "learning_rate": 7.182645715528435e-08,
      "logits/chosen": 0.9377768039703369,
      "logits/rejected": 0.8733525276184082,
      "logps/chosen": -254.7203826904297,
      "logps/ref_chosen": -53.687034606933594,
      "logps/ref_rejected": -83.59614562988281,
      "logps/rejected": -367.8406677246094,
      "loss": 1.1769,
      "margin_dpo/margin_mean": 83.211181640625,
      "margin_dpo/margin_std": 144.85556030273438,
      "step": 515
    },
    {
      "KL/chosen_KL_mean": -175.96588134765625,
      "KL/mean": -214.3978729248047,
      "KL/rejected_KL_mean": -252.82989501953125,
      "KL/std": 123.38099670410156,
      "epoch": 0.780045351473923,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.86399841308594,
      "fcm_dpo/q_t": 0.43540239334106445,
      "grad_norm": 11.299661636352539,
      "learning_rate": 7.090144991188568e-08,
      "logits/chosen": 0.8877590894699097,
      "logits/rejected": 0.8631953001022339,
      "logps/chosen": -232.8675994873047,
      "logps/ref_chosen": -56.9017219543457,
      "logps/ref_rejected": -67.83477783203125,
      "logps/rejected": -320.6646728515625,
      "loss": 1.1956,
      "margin_dpo/margin_mean": 76.86400604248047,
      "margin_dpo/margin_std": 143.44354248046875,
      "step": 516
    },
    {
      "KL/chosen_KL_mean": -208.1798095703125,
      "KL/mean": -237.89071655273438,
      "KL/rejected_KL_mean": -267.60162353515625,
      "KL/std": 135.55838012695312,
      "epoch": 0.781557067271353,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.42182159423828,
      "fcm_dpo/q_t": 0.4514128565788269,
      "grad_norm": 11.521942138671875,
      "learning_rate": 6.998145243993284e-08,
      "logits/chosen": 0.9006946086883545,
      "logits/rejected": 0.9093080759048462,
      "logps/chosen": -269.9549560546875,
      "logps/ref_chosen": -61.775142669677734,
      "logps/ref_rejected": -62.88270950317383,
      "logps/rejected": -330.484375,
      "loss": 1.2447,
      "margin_dpo/margin_mean": 59.42182922363281,
      "margin_dpo/margin_std": 138.92967224121094,
      "step": 517
    },
    {
      "KL/chosen_KL_mean": -180.6609344482422,
      "KL/mean": -224.84420776367188,
      "KL/rejected_KL_mean": -269.0274353027344,
      "KL/std": 133.86831665039062,
      "epoch": 0.783068783068783,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.36648559570312,
      "fcm_dpo/q_t": 0.4286819100379944,
      "grad_norm": 10.708097457885742,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 0.8798240423202515,
      "logits/rejected": 0.8539550304412842,
      "logps/chosen": -242.68617248535156,
      "logps/ref_chosen": -62.02523422241211,
      "logps/ref_rejected": -79.06085205078125,
      "logps/rejected": -348.0882873535156,
      "loss": 1.1652,
      "margin_dpo/margin_mean": 88.36649322509766,
      "margin_dpo/margin_std": 151.16380310058594,
      "step": 518
    },
    {
      "KL/chosen_KL_mean": -209.5022430419922,
      "KL/mean": -235.99777221679688,
      "KL/rejected_KL_mean": -262.4932861328125,
      "KL/std": 132.36302185058594,
      "epoch": 0.7845804988662132,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 52.991058349609375,
      "fcm_dpo/q_t": 0.45548588037490845,
      "grad_norm": 13.631095886230469,
      "learning_rate": 6.815658960673781e-08,
      "logits/chosen": 0.8932880163192749,
      "logits/rejected": 0.8629090785980225,
      "logps/chosen": -271.1086120605469,
      "logps/ref_chosen": -61.60636901855469,
      "logps/ref_rejected": -74.50727844238281,
      "logps/rejected": -337.0005798339844,
      "loss": 1.2941,
      "margin_dpo/margin_mean": 52.991058349609375,
      "margin_dpo/margin_std": 169.62295532226562,
      "step": 519
    },
    {
      "KL/chosen_KL_mean": -195.7787322998047,
      "KL/mean": -235.96630859375,
      "KL/rejected_KL_mean": -276.1539001464844,
      "KL/std": 137.5313262939453,
      "epoch": 0.7860922146636432,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.37517547607422,
      "fcm_dpo/q_t": 0.4349038004875183,
      "grad_norm": 13.164137840270996,
      "learning_rate": 6.725177529083209e-08,
      "logits/chosen": 0.974045991897583,
      "logits/rejected": 0.9345914125442505,
      "logps/chosen": -258.65216064453125,
      "logps/ref_chosen": -62.87343215942383,
      "logps/ref_rejected": -76.505615234375,
      "logps/rejected": -352.65948486328125,
      "loss": 1.18,
      "margin_dpo/margin_mean": 80.37517547607422,
      "margin_dpo/margin_std": 138.4918212890625,
      "step": 520
    },
    {
      "KL/chosen_KL_mean": -191.95623779296875,
      "KL/mean": -248.52978515625,
      "KL/rejected_KL_mean": -305.1033630371094,
      "KL/std": 137.93133544921875,
      "epoch": 0.7876039304610734,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 113.1470947265625,
      "fcm_dpo/q_t": 0.40972086787223816,
      "grad_norm": 8.968234062194824,
      "learning_rate": 6.63520728356167e-08,
      "logits/chosen": 0.8188831806182861,
      "logits/rejected": 0.7698487043380737,
      "logps/chosen": -256.16290283203125,
      "logps/ref_chosen": -64.20668029785156,
      "logps/ref_rejected": -92.28083038330078,
      "logps/rejected": -397.3841857910156,
      "loss": 1.0978,
      "margin_dpo/margin_mean": 113.1470947265625,
      "margin_dpo/margin_std": 159.48582458496094,
      "step": 521
    },
    {
      "KL/chosen_KL_mean": -198.2237548828125,
      "KL/mean": -229.06884765625,
      "KL/rejected_KL_mean": -259.9139099121094,
      "KL/std": 131.58770751953125,
      "epoch": 0.7891156462585034,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.690155029296875,
      "fcm_dpo/q_t": 0.44906526803970337,
      "grad_norm": 11.111807823181152,
      "learning_rate": 6.545750740770336e-08,
      "logits/chosen": 0.9058156609535217,
      "logits/rejected": 0.9006133079528809,
      "logps/chosen": -256.5934753417969,
      "logps/ref_chosen": -58.369720458984375,
      "logps/ref_rejected": -68.79248046875,
      "logps/rejected": -328.7063903808594,
      "loss": 1.2654,
      "margin_dpo/margin_mean": 61.690147399902344,
      "margin_dpo/margin_std": 168.9842529296875,
      "step": 522
    },
    {
      "KL/chosen_KL_mean": -205.13241577148438,
      "KL/mean": -247.29925537109375,
      "KL/rejected_KL_mean": -289.466064453125,
      "KL/std": 140.7245635986328,
      "epoch": 0.7906273620559335,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 84.33366394042969,
      "fcm_dpo/q_t": 0.42924371361732483,
      "grad_norm": 14.811322212219238,
      "learning_rate": 6.456810403001012e-08,
      "logits/chosen": 0.9063102006912231,
      "logits/rejected": 0.8146883249282837,
      "logps/chosen": -270.84564208984375,
      "logps/ref_chosen": -65.71324157714844,
      "logps/ref_rejected": -91.98896789550781,
      "logps/rejected": -381.455078125,
      "loss": 1.1802,
      "margin_dpo/margin_mean": 84.33366394042969,
      "margin_dpo/margin_std": 153.331787109375,
      "step": 523
    },
    {
      "KL/chosen_KL_mean": -182.45616149902344,
      "KL/mean": -222.56631469726562,
      "KL/rejected_KL_mean": -262.6764831542969,
      "KL/std": 133.9646453857422,
      "epoch": 0.7921390778533636,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.22032928466797,
      "fcm_dpo/q_t": 0.43342095613479614,
      "grad_norm": 11.4309720993042,
      "learning_rate": 6.368388758106134e-08,
      "logits/chosen": 0.8162028789520264,
      "logits/rejected": 0.8029775619506836,
      "logps/chosen": -258.8074035644531,
      "logps/ref_chosen": -76.35124969482422,
      "logps/ref_rejected": -89.96072387695312,
      "logps/rejected": -352.63720703125,
      "loss": 1.1759,
      "margin_dpo/margin_mean": 80.2203369140625,
      "margin_dpo/margin_std": 133.57424926757812,
      "step": 524
    },
    {
      "KL/chosen_KL_mean": -204.7865447998047,
      "KL/mean": -237.8707275390625,
      "KL/rejected_KL_mean": -270.9549255371094,
      "KL/std": 131.46104431152344,
      "epoch": 0.7936507936507936,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.16841125488281,
      "fcm_dpo/q_t": 0.4471135139465332,
      "grad_norm": 14.039843559265137,
      "learning_rate": 6.280488279429185e-08,
      "logits/chosen": 0.7615352869033813,
      "logits/rejected": 0.7642134428024292,
      "logps/chosen": -280.2823486328125,
      "logps/ref_chosen": -75.49578857421875,
      "logps/ref_rejected": -84.04852294921875,
      "logps/rejected": -355.00347900390625,
      "loss": 1.2397,
      "margin_dpo/margin_mean": 66.16841125488281,
      "margin_dpo/margin_std": 158.4366455078125,
      "step": 525
    },
    {
      "KL/chosen_KL_mean": -206.23468017578125,
      "KL/mean": -239.44537353515625,
      "KL/rejected_KL_mean": -272.65606689453125,
      "KL/std": 137.84564208984375,
      "epoch": 0.7951625094482238,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.42141723632812,
      "fcm_dpo/q_t": 0.44594311714172363,
      "grad_norm": 11.381861686706543,
      "learning_rate": 6.193111425735515e-08,
      "logits/chosen": 0.9240107536315918,
      "logits/rejected": 0.8726488351821899,
      "logps/chosen": -267.527099609375,
      "logps/ref_chosen": -61.29241943359375,
      "logps/ref_rejected": -82.47763061523438,
      "logps/rejected": -355.13372802734375,
      "loss": 1.2292,
      "margin_dpo/margin_mean": 66.42142486572266,
      "margin_dpo/margin_std": 142.61502075195312,
      "step": 526
    },
    {
      "KL/chosen_KL_mean": -214.8679656982422,
      "KL/mean": -244.40792846679688,
      "KL/rejected_KL_mean": -273.9479064941406,
      "KL/std": 135.75579833984375,
      "epoch": 0.7966742252456538,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.0799560546875,
      "fcm_dpo/q_t": 0.452206552028656,
      "grad_norm": 11.494239807128906,
      "learning_rate": 6.106260641143546e-08,
      "logits/chosen": 0.9482017755508423,
      "logits/rejected": 0.8839746713638306,
      "logps/chosen": -276.340576171875,
      "logps/ref_chosen": -61.472625732421875,
      "logps/ref_rejected": -90.52831268310547,
      "logps/rejected": -364.4762268066406,
      "loss": 1.2573,
      "margin_dpo/margin_mean": 59.079952239990234,
      "margin_dpo/margin_std": 150.58551025390625,
      "step": 527
    },
    {
      "KL/chosen_KL_mean": -191.18624877929688,
      "KL/mean": -223.8017120361328,
      "KL/rejected_KL_mean": -256.41717529296875,
      "KL/std": 124.5980224609375,
      "epoch": 0.7981859410430839,
      "fcm_dpo/beta": 0.00353202479891479,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 65.23095703125,
      "fcm_dpo/q_t": 0.4472540318965912,
      "grad_norm": 12.8545560836792,
      "learning_rate": 6.019938355056422e-08,
      "logits/chosen": 0.8528811931610107,
      "logits/rejected": 0.8042967915534973,
      "logps/chosen": -249.97825622558594,
      "logps/ref_chosen": -58.792015075683594,
      "logps/ref_rejected": -71.82516479492188,
      "logps/rejected": -328.24237060546875,
      "loss": 1.253,
      "margin_dpo/margin_mean": 65.23095703125,
      "margin_dpo/margin_std": 163.40753173828125,
      "step": 528
    },
    {
      "KL/chosen_KL_mean": -187.9537353515625,
      "KL/mean": -245.75003051757812,
      "KL/rejected_KL_mean": -303.54632568359375,
      "KL/std": 121.35850524902344,
      "epoch": 0.799697656840514,
      "fcm_dpo/beta": 0.003525674808770418,
      "fcm_dpo/delta": -0.01801120676100254,
      "fcm_dpo/margin": 115.59262084960938,
      "fcm_dpo/q_t": 0.40416377782821655,
      "grad_norm": 11.939973831176758,
      "learning_rate": 5.934146982094049e-08,
      "logits/chosen": 0.8495355248451233,
      "logits/rejected": 0.8155577778816223,
      "logps/chosen": -243.02468872070312,
      "logps/ref_chosen": -55.070960998535156,
      "logps/ref_rejected": -75.44007873535156,
      "logps/rejected": -378.9864196777344,
      "loss": 1.0713,
      "margin_dpo/margin_mean": 115.59260559082031,
      "margin_dpo/margin_std": 131.84812927246094,
      "step": 529
    },
    {
      "KL/chosen_KL_mean": -184.20016479492188,
      "KL/mean": -225.19178771972656,
      "KL/rejected_KL_mean": -266.18341064453125,
      "KL/std": 140.19061279296875,
      "epoch": 0.8012093726379441,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.98326110839844,
      "fcm_dpo/q_t": 0.4346047639846802,
      "grad_norm": 12.730396270751953,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": 0.9051386117935181,
      "logits/rejected": 0.874481201171875,
      "logps/chosen": -240.9439697265625,
      "logps/ref_chosen": -56.743812561035156,
      "logps/ref_rejected": -76.6692123413086,
      "logps/rejected": -342.8526306152344,
      "loss": 1.1849,
      "margin_dpo/margin_mean": 81.98326110839844,
      "margin_dpo/margin_std": 149.03494262695312,
      "step": 530
    },
    {
      "KL/chosen_KL_mean": -184.72265625,
      "KL/mean": -225.67088317871094,
      "KL/rejected_KL_mean": -266.6191101074219,
      "KL/std": 133.89361572265625,
      "epoch": 0.8027210884353742,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.89643859863281,
      "fcm_dpo/q_t": 0.4329376816749573,
      "grad_norm": 10.601408958435059,
      "learning_rate": 5.7641665597021435e-08,
      "logits/chosen": 0.8985570669174194,
      "logits/rejected": 0.8340739011764526,
      "logps/chosen": -235.839111328125,
      "logps/ref_chosen": -51.116455078125,
      "logps/ref_rejected": -79.52884674072266,
      "logps/rejected": -346.14794921875,
      "loss": 1.1821,
      "margin_dpo/margin_mean": 81.89643859863281,
      "margin_dpo/margin_std": 146.56626892089844,
      "step": 531
    },
    {
      "KL/chosen_KL_mean": -214.75320434570312,
      "KL/mean": -257.167236328125,
      "KL/rejected_KL_mean": -299.5812683105469,
      "KL/std": 135.9754638671875,
      "epoch": 0.8042328042328042,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 84.8280258178711,
      "fcm_dpo/q_t": 0.43083542585372925,
      "grad_norm": 11.06413459777832,
      "learning_rate": 5.679982264990424e-08,
      "logits/chosen": 0.8801513910293579,
      "logits/rejected": 0.8441125154495239,
      "logps/chosen": -273.03314208984375,
      "logps/ref_chosen": -58.279945373535156,
      "logps/ref_rejected": -78.05426788330078,
      "logps/rejected": -377.6355285644531,
      "loss": 1.1777,
      "margin_dpo/margin_mean": 84.8280258178711,
      "margin_dpo/margin_std": 151.73892211914062,
      "step": 532
    },
    {
      "KL/chosen_KL_mean": -155.84539794921875,
      "KL/mean": -205.1451416015625,
      "KL/rejected_KL_mean": -254.4448699951172,
      "KL/std": 144.37937927246094,
      "epoch": 0.8057445200302343,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 98.59945678710938,
      "fcm_dpo/q_t": 0.4208802580833435,
      "grad_norm": 12.034300804138184,
      "learning_rate": 5.596338392706076e-08,
      "logits/chosen": 0.9465242624282837,
      "logits/rejected": 0.9069874882698059,
      "logps/chosen": -212.263427734375,
      "logps/ref_chosen": -56.41801071166992,
      "logps/ref_rejected": -73.89324951171875,
      "logps/rejected": -328.338134765625,
      "loss": 1.1378,
      "margin_dpo/margin_mean": 98.5994644165039,
      "margin_dpo/margin_std": 153.1092987060547,
      "step": 533
    },
    {
      "KL/chosen_KL_mean": -194.58856201171875,
      "KL/mean": -231.89251708984375,
      "KL/rejected_KL_mean": -269.1965026855469,
      "KL/std": 126.38986206054688,
      "epoch": 0.8072562358276644,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.60794830322266,
      "fcm_dpo/q_t": 0.4392569661140442,
      "grad_norm": 11.195599555969238,
      "learning_rate": 5.513237282548033e-08,
      "logits/chosen": 0.8730405569076538,
      "logits/rejected": 0.8510438799858093,
      "logps/chosen": -255.33724975585938,
      "logps/ref_chosen": -60.748687744140625,
      "logps/ref_rejected": -73.8623046875,
      "logps/rejected": -343.0588073730469,
      "loss": 1.2105,
      "margin_dpo/margin_mean": 74.60794830322266,
      "margin_dpo/margin_std": 153.75209045410156,
      "step": 534
    },
    {
      "KL/chosen_KL_mean": -204.45562744140625,
      "KL/mean": -240.7137451171875,
      "KL/rejected_KL_mean": -276.97186279296875,
      "KL/std": 134.9508056640625,
      "epoch": 0.8087679516250945,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 72.51627349853516,
      "fcm_dpo/q_t": 0.4411739706993103,
      "grad_norm": 11.336689949035645,
      "learning_rate": 5.430681259032957e-08,
      "logits/chosen": 0.8373583555221558,
      "logits/rejected": 0.7964029908180237,
      "logps/chosen": -266.093017578125,
      "logps/ref_chosen": -61.637413024902344,
      "logps/ref_rejected": -80.93138885498047,
      "logps/rejected": -357.90325927734375,
      "loss": 1.2167,
      "margin_dpo/margin_mean": 72.51627349853516,
      "margin_dpo/margin_std": 150.4508819580078,
      "step": 535
    },
    {
      "KL/chosen_KL_mean": -176.42593383789062,
      "KL/mean": -233.67471313476562,
      "KL/rejected_KL_mean": -290.9234924316406,
      "KL/std": 134.7869873046875,
      "epoch": 0.8102796674225246,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 114.49759674072266,
      "fcm_dpo/q_t": 0.4069276452064514,
      "grad_norm": 8.79976749420166,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 0.9333071708679199,
      "logits/rejected": 0.877731442451477,
      "logps/chosen": -228.31491088867188,
      "logps/ref_chosen": -51.88897705078125,
      "logps/ref_rejected": -73.34864044189453,
      "logps/rejected": -364.27215576171875,
      "loss": 1.0802,
      "margin_dpo/margin_mean": 114.49759674072266,
      "margin_dpo/margin_std": 139.86199951171875,
      "step": 536
    },
    {
      "KL/chosen_KL_mean": -200.56451416015625,
      "KL/mean": -251.96243286132812,
      "KL/rejected_KL_mean": -303.3603515625,
      "KL/std": 149.33303833007812,
      "epoch": 0.8117913832199547,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 102.79583740234375,
      "fcm_dpo/q_t": 0.41782182455062866,
      "grad_norm": 10.906473159790039,
      "learning_rate": 5.267213693697695e-08,
      "logits/chosen": 0.9628849029541016,
      "logits/rejected": 0.8948889970779419,
      "logps/chosen": -254.81312561035156,
      "logps/ref_chosen": -54.248619079589844,
      "logps/ref_rejected": -94.94343566894531,
      "logps/rejected": -398.30377197265625,
      "loss": 1.1366,
      "margin_dpo/margin_mean": 102.79583740234375,
      "margin_dpo/margin_std": 165.4451904296875,
      "step": 537
    },
    {
      "KL/chosen_KL_mean": -206.56629943847656,
      "KL/mean": -254.18389892578125,
      "KL/rejected_KL_mean": -301.801513671875,
      "KL/std": 152.03558349609375,
      "epoch": 0.8133030990173847,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 95.23521423339844,
      "fcm_dpo/q_t": 0.42235174775123596,
      "grad_norm": 10.19690227508545,
      "learning_rate": 5.1863067244167144e-08,
      "logits/chosen": 0.9172987937927246,
      "logits/rejected": 0.9068432450294495,
      "logps/chosen": -276.65985107421875,
      "logps/ref_chosen": -70.09353637695312,
      "logps/ref_rejected": -79.49833679199219,
      "logps/rejected": -381.2998352050781,
      "loss": 1.1466,
      "margin_dpo/margin_mean": 95.23521423339844,
      "margin_dpo/margin_std": 152.82855224609375,
      "step": 538
    },
    {
      "KL/chosen_KL_mean": -206.97450256347656,
      "KL/mean": -244.26101684570312,
      "KL/rejected_KL_mean": -281.5475158691406,
      "KL/std": 135.51553344726562,
      "epoch": 0.8148148148148148,
      "fcm_dpo/beta": 0.0035193245857954025,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.57301330566406,
      "fcm_dpo/q_t": 0.4409199357032776,
      "grad_norm": 12.181897163391113,
      "learning_rate": 5.105953986729195e-08,
      "logits/chosen": 0.869612455368042,
      "logits/rejected": 0.8076390027999878,
      "logps/chosen": -268.90618896484375,
      "logps/ref_chosen": -61.93169403076172,
      "logps/ref_rejected": -84.08946228027344,
      "logps/rejected": -365.636962890625,
      "loss": 1.2022,
      "margin_dpo/margin_mean": 74.57301330566406,
      "margin_dpo/margin_std": 146.1897735595703,
      "step": 539
    },
    {
      "KL/chosen_KL_mean": -191.78707885742188,
      "KL/mean": -251.977783203125,
      "KL/rejected_KL_mean": -312.16851806640625,
      "KL/std": 161.7171630859375,
      "epoch": 0.8163265306122449,
      "fcm_dpo/beta": 0.003508302615955472,
      "fcm_dpo/delta": -0.03141680359840393,
      "fcm_dpo/margin": 120.38145446777344,
      "fcm_dpo/q_t": 0.40351608395576477,
      "grad_norm": 9.613325119018555,
      "learning_rate": 5.026157728273966e-08,
      "logits/chosen": 0.9319074749946594,
      "logits/rejected": 0.8714286088943481,
      "logps/chosen": -254.49131774902344,
      "logps/ref_chosen": -62.704254150390625,
      "logps/ref_rejected": -95.63597106933594,
      "logps/rejected": -407.80450439453125,
      "loss": 1.0779,
      "margin_dpo/margin_mean": 120.3814697265625,
      "margin_dpo/margin_std": 157.05987548828125,
      "step": 540
    },
    {
      "KL/chosen_KL_mean": -187.46072387695312,
      "KL/mean": -230.84857177734375,
      "KL/rejected_KL_mean": -274.2364196777344,
      "KL/std": 133.34332275390625,
      "epoch": 0.817838246409675,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 86.77567291259766,
      "fcm_dpo/q_t": 0.4274454712867737,
      "grad_norm": 10.517534255981445,
      "learning_rate": 4.9469201811239035e-08,
      "logits/chosen": 0.9364334344863892,
      "logits/rejected": 0.9676176309585571,
      "logps/chosen": -249.94155883789062,
      "logps/ref_chosen": -62.48084259033203,
      "logps/ref_rejected": -57.55541229248047,
      "logps/rejected": -331.79180908203125,
      "loss": 1.1528,
      "margin_dpo/margin_mean": 86.77566528320312,
      "margin_dpo/margin_std": 125.50430297851562,
      "step": 541
    },
    {
      "KL/chosen_KL_mean": -163.31414794921875,
      "KL/mean": -212.81866455078125,
      "KL/rejected_KL_mean": -262.32318115234375,
      "KL/std": 123.844482421875,
      "epoch": 0.8193499622071051,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 99.00904846191406,
      "fcm_dpo/q_t": 0.41917315125465393,
      "grad_norm": 10.483098983764648,
      "learning_rate": 4.868243561723534e-08,
      "logits/chosen": 0.9898308515548706,
      "logits/rejected": 0.9567674398422241,
      "logps/chosen": -212.76902770996094,
      "logps/ref_chosen": -49.454891204833984,
      "logps/ref_rejected": -65.33275604248047,
      "logps/rejected": -327.65594482421875,
      "loss": 1.1378,
      "margin_dpo/margin_mean": 99.00904846191406,
      "margin_dpo/margin_std": 152.8940887451172,
      "step": 542
    },
    {
      "KL/chosen_KL_mean": -176.56614685058594,
      "KL/mean": -229.4498291015625,
      "KL/rejected_KL_mean": -282.33349609375,
      "KL/std": 136.6936798095703,
      "epoch": 0.8208616780045351,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 105.7673568725586,
      "fcm_dpo/q_t": 0.41475868225097656,
      "grad_norm": 9.79175090789795,
      "learning_rate": 4.790130070827028e-08,
      "logits/chosen": 0.9310851097106934,
      "logits/rejected": 0.8702279329299927,
      "logps/chosen": -227.66700744628906,
      "logps/ref_chosen": -51.100860595703125,
      "logps/ref_rejected": -76.06130981445312,
      "logps/rejected": -358.39483642578125,
      "loss": 1.1182,
      "margin_dpo/margin_mean": 105.76736450195312,
      "margin_dpo/margin_std": 151.32635498046875,
      "step": 543
    },
    {
      "KL/chosen_KL_mean": -192.3682861328125,
      "KL/mean": -251.83297729492188,
      "KL/rejected_KL_mean": -311.29766845703125,
      "KL/std": 152.34619140625,
      "epoch": 0.8223733938019653,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 118.92938232421875,
      "fcm_dpo/q_t": 0.40710359811782837,
      "grad_norm": 12.77357292175293,
      "learning_rate": 4.7125818934366454e-08,
      "logits/chosen": 0.8962240219116211,
      "logits/rejected": 0.8459259271621704,
      "logps/chosen": -252.64552307128906,
      "logps/ref_chosen": -60.2772331237793,
      "logps/ref_rejected": -88.40553283691406,
      "logps/rejected": -399.70318603515625,
      "loss": 1.1037,
      "margin_dpo/margin_mean": 118.92938232421875,
      "margin_dpo/margin_std": 180.5123291015625,
      "step": 544
    },
    {
      "KL/chosen_KL_mean": -209.5484161376953,
      "KL/mean": -240.4080810546875,
      "KL/rejected_KL_mean": -271.2677307128906,
      "KL/std": 137.64572143554688,
      "epoch": 0.8238851095993953,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 61.71931457519531,
      "fcm_dpo/q_t": 0.4491823613643646,
      "grad_norm": 12.780492782592773,
      "learning_rate": 4.635601198741607e-08,
      "logits/chosen": 0.9082489013671875,
      "logits/rejected": 0.8583580851554871,
      "logps/chosen": -271.1636657714844,
      "logps/ref_chosen": -61.61524963378906,
      "logps/ref_rejected": -78.71266174316406,
      "logps/rejected": -349.9803771972656,
      "loss": 1.2488,
      "margin_dpo/margin_mean": 61.71931457519531,
      "margin_dpo/margin_std": 150.1622314453125,
      "step": 545
    },
    {
      "KL/chosen_KL_mean": -198.5826416015625,
      "KL/mean": -232.60609436035156,
      "KL/rejected_KL_mean": -266.6295471191406,
      "KL/std": 130.08131408691406,
      "epoch": 0.8253968253968254,
      "fcm_dpo/beta": 0.0034972806461155415,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.0468978881836,
      "fcm_dpo/q_t": 0.443244993686676,
      "grad_norm": 16.30405044555664,
      "learning_rate": 4.559190140057428e-08,
      "logits/chosen": 0.9910711646080017,
      "logits/rejected": 0.9918454885482788,
      "logps/chosen": -257.8959045410156,
      "logps/ref_chosen": -59.313262939453125,
      "logps/ref_rejected": -64.73631286621094,
      "logps/rejected": -331.3658447265625,
      "loss": 1.2364,
      "margin_dpo/margin_mean": 68.04690551757812,
      "margin_dpo/margin_std": 158.61746215820312,
      "step": 546
    },
    {
      "KL/chosen_KL_mean": -171.865966796875,
      "KL/mean": -226.3336181640625,
      "KL/rejected_KL_mean": -280.80126953125,
      "KL/std": 134.1888885498047,
      "epoch": 0.8269085411942555,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": -0.0254131481051445,
      "fcm_dpo/margin": 108.93531036376953,
      "fcm_dpo/q_t": 0.41161462664604187,
      "grad_norm": 10.086834907531738,
      "learning_rate": 4.483350854765672e-08,
      "logits/chosen": 0.8900845050811768,
      "logits/rejected": 0.8488314747810364,
      "logps/chosen": -226.84271240234375,
      "logps/ref_chosen": -54.97674560546875,
      "logps/ref_rejected": -75.35922241210938,
      "logps/rejected": -356.16046142578125,
      "loss": 1.1111,
      "margin_dpo/margin_mean": 108.93531036376953,
      "margin_dpo/margin_std": 154.0179901123047,
      "step": 547
    },
    {
      "KL/chosen_KL_mean": -204.906982421875,
      "KL/mean": -238.65890502929688,
      "KL/rejected_KL_mean": -272.4108581542969,
      "KL/std": 137.80738830566406,
      "epoch": 0.8284202569916855,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 67.50390625,
      "fcm_dpo/q_t": 0.4453282952308655,
      "grad_norm": 11.766864776611328,
      "learning_rate": 4.4080854642541826e-08,
      "logits/chosen": 0.8757404088973999,
      "logits/rejected": 0.8392364978790283,
      "logps/chosen": -268.1176452636719,
      "logps/ref_chosen": -63.21067428588867,
      "logps/ref_rejected": -81.23347473144531,
      "logps/rejected": -353.64434814453125,
      "loss": 1.2296,
      "margin_dpo/margin_mean": 67.50390625,
      "margin_dpo/margin_std": 149.1422882080078,
      "step": 548
    },
    {
      "KL/chosen_KL_mean": -206.6922149658203,
      "KL/mean": -245.13987731933594,
      "KL/rejected_KL_mean": -283.5875244140625,
      "KL/std": 154.62350463867188,
      "epoch": 0.8299319727891157,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.89530181884766,
      "fcm_dpo/q_t": 0.4395865201950073,
      "grad_norm": 14.56954288482666,
      "learning_rate": 4.333396073857723e-08,
      "logits/chosen": 0.9742203950881958,
      "logits/rejected": 0.9250259399414062,
      "logps/chosen": -270.9657287597656,
      "logps/ref_chosen": -64.27351379394531,
      "logps/ref_rejected": -92.31663513183594,
      "logps/rejected": -375.9041748046875,
      "loss": 1.232,
      "margin_dpo/margin_mean": 76.89530181884766,
      "margin_dpo/margin_std": 181.66064453125,
      "step": 549
    },
    {
      "KL/chosen_KL_mean": -200.8487548828125,
      "KL/mean": -230.66412353515625,
      "KL/rejected_KL_mean": -260.4795227050781,
      "KL/std": 129.66531372070312,
      "epoch": 0.8314436885865457,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.630733489990234,
      "fcm_dpo/q_t": 0.4519397020339966,
      "grad_norm": 11.026690483093262,
      "learning_rate": 4.259284772799099e-08,
      "logits/chosen": 0.9252485036849976,
      "logits/rejected": 0.91241455078125,
      "logps/chosen": -257.0791931152344,
      "logps/ref_chosen": -56.230438232421875,
      "logps/ref_rejected": -62.59788513183594,
      "logps/rejected": -323.077392578125,
      "loss": 1.25,
      "margin_dpo/margin_mean": 59.6307373046875,
      "margin_dpo/margin_std": 142.07293701171875,
      "step": 550
    },
    {
      "KL/chosen_KL_mean": -212.65078735351562,
      "KL/mean": -249.81268310546875,
      "KL/rejected_KL_mean": -286.974609375,
      "KL/std": 144.7467498779297,
      "epoch": 0.8329554043839759,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.32379150390625,
      "fcm_dpo/q_t": 0.44245636463165283,
      "grad_norm": 9.790303230285645,
      "learning_rate": 4.1857536341307176e-08,
      "logits/chosen": 0.9503294229507446,
      "logits/rejected": 0.929543137550354,
      "logps/chosen": -280.39801025390625,
      "logps/ref_chosen": -67.74720764160156,
      "logps/ref_rejected": -87.04285430908203,
      "logps/rejected": -374.0174560546875,
      "loss": 1.2062,
      "margin_dpo/margin_mean": 74.32378387451172,
      "margin_dpo/margin_std": 147.83872985839844,
      "step": 551
    },
    {
      "KL/chosen_KL_mean": -196.72901916503906,
      "KL/mean": -239.97210693359375,
      "KL/rejected_KL_mean": -283.2151794433594,
      "KL/std": 154.41824340820312,
      "epoch": 0.8344671201814059,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 86.48617553710938,
      "fcm_dpo/q_t": 0.429646760225296,
      "grad_norm": 13.021488189697266,
      "learning_rate": 4.112804714676593e-08,
      "logits/chosen": 0.9085184335708618,
      "logits/rejected": 0.8709832429885864,
      "logps/chosen": -259.6552734375,
      "logps/ref_chosen": -62.92625427246094,
      "logps/ref_rejected": -82.98365783691406,
      "logps/rejected": -366.1988525390625,
      "loss": 1.1662,
      "margin_dpo/margin_mean": 86.48617553710938,
      "margin_dpo/margin_std": 141.24722290039062,
      "step": 552
    },
    {
      "KL/chosen_KL_mean": -210.44729614257812,
      "KL/mean": -255.28866577148438,
      "KL/rejected_KL_mean": -300.1300048828125,
      "KL/std": 151.55690002441406,
      "epoch": 0.8359788359788359,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.68273162841797,
      "fcm_dpo/q_t": 0.43007200956344604,
      "grad_norm": 12.203096389770508,
      "learning_rate": 4.0404400549748144e-08,
      "logits/chosen": 0.9325939416885376,
      "logits/rejected": 0.8622708320617676,
      "logps/chosen": -266.48577880859375,
      "logps/ref_chosen": -56.038490295410156,
      "logps/ref_rejected": -84.48454284667969,
      "logps/rejected": -384.61456298828125,
      "loss": 1.1827,
      "margin_dpo/margin_mean": 89.68273162841797,
      "margin_dpo/margin_std": 174.0795440673828,
      "step": 553
    },
    {
      "KL/chosen_KL_mean": -189.37258911132812,
      "KL/mean": -238.8749237060547,
      "KL/rejected_KL_mean": -288.37725830078125,
      "KL/std": 133.0142364501953,
      "epoch": 0.8374905517762661,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 99.00468444824219,
      "fcm_dpo/q_t": 0.4176701605319977,
      "grad_norm": 10.483379364013672,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 0.8873802423477173,
      "logits/rejected": 0.8916881084442139,
      "logps/chosen": -253.90318298339844,
      "logps/ref_chosen": -64.53059387207031,
      "logps/ref_rejected": -71.2155990600586,
      "logps/rejected": -359.5928955078125,
      "loss": 1.1295,
      "margin_dpo/margin_mean": 99.00468444824219,
      "margin_dpo/margin_std": 140.27540588378906,
      "step": 554
    },
    {
      "KL/chosen_KL_mean": -207.7998809814453,
      "KL/mean": -241.81884765625,
      "KL/rejected_KL_mean": -275.83782958984375,
      "KL/std": 131.1754913330078,
      "epoch": 0.8390022675736961,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.0379409790039,
      "fcm_dpo/q_t": 0.44281113147735596,
      "grad_norm": 11.698552131652832,
      "learning_rate": 3.89747159520904e-08,
      "logits/chosen": 0.932090163230896,
      "logits/rejected": 0.9236975908279419,
      "logps/chosen": -274.4517822265625,
      "logps/ref_chosen": -66.65191650390625,
      "logps/ref_rejected": -68.6667251586914,
      "logps/rejected": -344.5045471191406,
      "loss": 1.236,
      "margin_dpo/margin_mean": 68.0379409790039,
      "margin_dpo/margin_std": 154.10720825195312,
      "step": 555
    },
    {
      "KL/chosen_KL_mean": -199.71800231933594,
      "KL/mean": -238.94131469726562,
      "KL/rejected_KL_mean": -278.16461181640625,
      "KL/std": 134.803466796875,
      "epoch": 0.8405139833711263,
      "fcm_dpo/beta": 0.0034795503597706556,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 78.44662475585938,
      "fcm_dpo/q_t": 0.4365660846233368,
      "grad_norm": 11.139961242675781,
      "learning_rate": 3.826871794280192e-08,
      "logits/chosen": 0.9880878925323486,
      "logits/rejected": 0.9601046442985535,
      "logps/chosen": -252.5503692626953,
      "logps/ref_chosen": -52.832366943359375,
      "logps/ref_rejected": -64.49044036865234,
      "logps/rejected": -342.65509033203125,
      "loss": 1.1934,
      "margin_dpo/margin_mean": 78.44662475585938,
      "margin_dpo/margin_std": 143.8062286376953,
      "step": 556
    },
    {
      "KL/chosen_KL_mean": -199.31085205078125,
      "KL/mean": -255.07276916503906,
      "KL/rejected_KL_mean": -310.834716796875,
      "KL/std": 142.0129852294922,
      "epoch": 0.8420256991685563,
      "fcm_dpo/beta": 0.0034691179171204567,
      "fcm_dpo/delta": -0.01501347217708826,
      "fcm_dpo/margin": 111.52387237548828,
      "fcm_dpo/q_t": 0.41104036569595337,
      "grad_norm": 10.035857200622559,
      "learning_rate": 3.756864251262143e-08,
      "logits/chosen": 0.9729760885238647,
      "logits/rejected": 0.9261834621429443,
      "logps/chosen": -254.34683227539062,
      "logps/ref_chosen": -55.03598403930664,
      "logps/ref_rejected": -75.80644989013672,
      "logps/rejected": -386.64117431640625,
      "loss": 1.0961,
      "margin_dpo/margin_mean": 111.52386474609375,
      "margin_dpo/margin_std": 141.70437622070312,
      "step": 557
    },
    {
      "KL/chosen_KL_mean": -191.17112731933594,
      "KL/mean": -249.64356994628906,
      "KL/rejected_KL_mean": -308.11602783203125,
      "KL/std": 148.79632568359375,
      "epoch": 0.8435374149659864,
      "fcm_dpo/beta": 0.0034465533681213856,
      "fcm_dpo/delta": -0.032628294080495834,
      "fcm_dpo/margin": 116.94486999511719,
      "fcm_dpo/q_t": 0.40802621841430664,
      "grad_norm": 10.100082397460938,
      "learning_rate": 3.687450924416341e-08,
      "logits/chosen": 0.9450229406356812,
      "logits/rejected": 0.9094198346138,
      "logps/chosen": -254.39749145507812,
      "logps/ref_chosen": -63.226348876953125,
      "logps/ref_rejected": -91.46881866455078,
      "logps/rejected": -399.5848388671875,
      "loss": 1.0967,
      "margin_dpo/margin_mean": 116.94486999511719,
      "margin_dpo/margin_std": 162.83993530273438,
      "step": 558
    },
    {
      "KL/chosen_KL_mean": -192.72799682617188,
      "KL/mean": -246.63272094726562,
      "KL/rejected_KL_mean": -300.5374450683594,
      "KL/std": 150.26181030273438,
      "epoch": 0.8450491307634165,
      "fcm_dpo/beta": 0.0034465533681213856,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 107.80944061279297,
      "fcm_dpo/q_t": 0.4185447692871094,
      "grad_norm": 11.648037910461426,
      "learning_rate": 3.6186337553827743e-08,
      "logits/chosen": 0.9157607555389404,
      "logits/rejected": 0.8777155876159668,
      "logps/chosen": -254.24964904785156,
      "logps/ref_chosen": -61.521644592285156,
      "logps/ref_rejected": -82.83859252929688,
      "logps/rejected": -383.37603759765625,
      "loss": 1.1398,
      "margin_dpo/margin_mean": 107.8094482421875,
      "margin_dpo/margin_std": 182.46499633789062,
      "step": 559
    },
    {
      "KL/chosen_KL_mean": -209.66896057128906,
      "KL/mean": -258.95849609375,
      "KL/rejected_KL_mean": -308.2480163574219,
      "KL/std": 139.55429077148438,
      "epoch": 0.8465608465608465,
      "fcm_dpo/beta": 0.0034465533681213856,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 98.57904815673828,
      "fcm_dpo/q_t": 0.4205133318901062,
      "grad_norm": 12.194161415100098,
      "learning_rate": 3.550414669125573e-08,
      "logits/chosen": 0.9571900963783264,
      "logits/rejected": 0.9336278438568115,
      "logps/chosen": -270.3101806640625,
      "logps/ref_chosen": -60.64122009277344,
      "logps/ref_rejected": -78.75474548339844,
      "logps/rejected": -387.00274658203125,
      "loss": 1.1241,
      "margin_dpo/margin_mean": 98.57905578613281,
      "margin_dpo/margin_std": 129.6819305419922,
      "step": 560
    },
    {
      "KL/chosen_KL_mean": -197.01702880859375,
      "KL/mean": -237.6276092529297,
      "KL/rejected_KL_mean": -278.2381896972656,
      "KL/std": 132.2579345703125,
      "epoch": 0.8480725623582767,
      "fcm_dpo/beta": 0.0034465533681213856,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.22117614746094,
      "fcm_dpo/q_t": 0.43530896306037903,
      "grad_norm": 10.928399085998535,
      "learning_rate": 3.482795573879241e-08,
      "logits/chosen": 0.8836110830307007,
      "logits/rejected": 0.8617398738861084,
      "logps/chosen": -259.515625,
      "logps/ref_chosen": -62.49859619140625,
      "logps/ref_rejected": -78.72064208984375,
      "logps/rejected": -356.9588317871094,
      "loss": 1.1879,
      "margin_dpo/margin_mean": 81.2211685180664,
      "margin_dpo/margin_std": 147.069580078125,
      "step": 561
    },
    {
      "KL/chosen_KL_mean": -213.68255615234375,
      "KL/mean": -271.47039794921875,
      "KL/rejected_KL_mean": -329.25823974609375,
      "KL/std": 176.3778076171875,
      "epoch": 0.8495842781557067,
      "fcm_dpo/beta": 0.003401133930310607,
      "fcm_dpo/delta": -0.06632918119430542,
      "fcm_dpo/margin": 115.57572174072266,
      "fcm_dpo/q_t": 0.41180339455604553,
      "grad_norm": 13.891858100891113,
      "learning_rate": 3.415778361095226e-08,
      "logits/chosen": 0.9108099937438965,
      "logits/rejected": 0.8991225957870483,
      "logps/chosen": -288.46429443359375,
      "logps/ref_chosen": -74.78173828125,
      "logps/ref_rejected": -92.63499450683594,
      "logps/rejected": -421.89324951171875,
      "loss": 1.1111,
      "margin_dpo/margin_mean": 115.57572937011719,
      "margin_dpo/margin_std": 170.50723266601562,
      "step": 562
    },
    {
      "KL/chosen_KL_mean": -172.10968017578125,
      "KL/mean": -219.07452392578125,
      "KL/rejected_KL_mean": -266.03936767578125,
      "KL/std": 117.71343994140625,
      "epoch": 0.8510959939531368,
      "fcm_dpo/beta": 0.003401133930310607,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 93.92967987060547,
      "fcm_dpo/q_t": 0.424142062664032,
      "grad_norm": 17.967735290527344,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": 1.005516529083252,
      "logits/rejected": 0.9655479192733765,
      "logps/chosen": -222.30819702148438,
      "logps/ref_chosen": -50.19850158691406,
      "logps/ref_rejected": -66.76687622070312,
      "logps/rejected": -332.8062438964844,
      "loss": 1.154,
      "margin_dpo/margin_mean": 93.9296875,
      "margin_dpo/margin_std": 148.81942749023438,
      "step": 563
    },
    {
      "KL/chosen_KL_mean": -172.90924072265625,
      "KL/mean": -230.859619140625,
      "KL/rejected_KL_mean": -288.80999755859375,
      "KL/std": 130.04574584960938,
      "epoch": 0.8526077097505669,
      "fcm_dpo/beta": 0.003401133930310607,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 115.9007339477539,
      "fcm_dpo/q_t": 0.40835243463516235,
      "grad_norm": 11.180398941040039,
      "learning_rate": 3.283557064487785e-08,
      "logits/chosen": 0.90259850025177,
      "logits/rejected": 0.8867555856704712,
      "logps/chosen": -228.65008544921875,
      "logps/ref_chosen": -55.7408447265625,
      "logps/ref_rejected": -74.82323455810547,
      "logps/rejected": -363.63323974609375,
      "loss": 1.1012,
      "margin_dpo/margin_mean": 115.90072631835938,
      "margin_dpo/margin_std": 161.30313110351562,
      "step": 564
    },
    {
      "KL/chosen_KL_mean": -211.1297607421875,
      "KL/mean": -259.3542785644531,
      "KL/rejected_KL_mean": -307.57879638671875,
      "KL/std": 142.05772399902344,
      "epoch": 0.854119425547997,
      "fcm_dpo/beta": 0.003401133930310607,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 96.44905090332031,
      "fcm_dpo/q_t": 0.4226211905479431,
      "grad_norm": 12.189798355102539,
      "learning_rate": 3.218356679178252e-08,
      "logits/chosen": 0.9537685513496399,
      "logits/rejected": 0.9214394092559814,
      "logps/chosen": -269.4671630859375,
      "logps/ref_chosen": -58.33738327026367,
      "logps/ref_rejected": -78.31776428222656,
      "logps/rejected": -385.89654541015625,
      "loss": 1.1404,
      "margin_dpo/margin_mean": 96.44903564453125,
      "margin_dpo/margin_std": 142.17813110351562,
      "step": 565
    },
    {
      "KL/chosen_KL_mean": -202.7068634033203,
      "KL/mean": -244.68670654296875,
      "KL/rejected_KL_mean": -286.6665344238281,
      "KL/std": 151.67178344726562,
      "epoch": 0.8556311413454271,
      "fcm_dpo/beta": 0.003401133930310607,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.95967864990234,
      "fcm_dpo/q_t": 0.4360388517379761,
      "grad_norm": 9.914851188659668,
      "learning_rate": 3.1537655732553764e-08,
      "logits/chosen": 0.944286048412323,
      "logits/rejected": 0.9478252530097961,
      "logps/chosen": -273.93060302734375,
      "logps/ref_chosen": -71.22373962402344,
      "logps/ref_rejected": -71.11601257324219,
      "logps/rejected": -357.78253173828125,
      "loss": 1.2189,
      "margin_dpo/margin_mean": 83.95967102050781,
      "margin_dpo/margin_std": 190.03399658203125,
      "step": 566
    },
    {
      "KL/chosen_KL_mean": -187.10617065429688,
      "KL/mean": -242.869873046875,
      "KL/rejected_KL_mean": -298.63360595703125,
      "KL/std": 141.89431762695312,
      "epoch": 0.8571428571428571,
      "fcm_dpo/beta": 0.003368604462593794,
      "fcm_dpo/delta": -0.04805165156722069,
      "fcm_dpo/margin": 111.52742767333984,
      "fcm_dpo/q_t": 0.41300350427627563,
      "grad_norm": 9.936294555664062,
      "learning_rate": 3.089785553471233e-08,
      "logits/chosen": 0.9251055121421814,
      "logits/rejected": 0.8682990670204163,
      "logps/chosen": -239.77545166015625,
      "logps/ref_chosen": -52.669273376464844,
      "logps/ref_rejected": -74.34785461425781,
      "logps/rejected": -372.9814453125,
      "loss": 1.1155,
      "margin_dpo/margin_mean": 111.52742767333984,
      "margin_dpo/margin_std": 159.84197998046875,
      "step": 567
    },
    {
      "KL/chosen_KL_mean": -175.10830688476562,
      "KL/mean": -240.4864501953125,
      "KL/rejected_KL_mean": -305.8646240234375,
      "KL/std": 149.96920776367188,
      "epoch": 0.8586545729402872,
      "fcm_dpo/beta": 0.0033373315818607807,
      "fcm_dpo/delta": -0.09370847791433334,
      "fcm_dpo/margin": 130.75628662109375,
      "fcm_dpo/q_t": 0.40074622631073,
      "grad_norm": 13.38181209564209,
      "learning_rate": 3.026418409484513e-08,
      "logits/chosen": 0.9603248834609985,
      "logits/rejected": 0.9052588939666748,
      "logps/chosen": -227.28631591796875,
      "logps/ref_chosen": -52.178001403808594,
      "logps/ref_rejected": -85.8277587890625,
      "logps/rejected": -391.6923828125,
      "loss": 1.0673,
      "margin_dpo/margin_mean": 130.75628662109375,
      "margin_dpo/margin_std": 157.99542236328125,
      "step": 568
    },
    {
      "KL/chosen_KL_mean": -209.8172149658203,
      "KL/mean": -239.02120971679688,
      "KL/rejected_KL_mean": -268.2251892089844,
      "KL/std": 151.32420349121094,
      "epoch": 0.8601662887377173,
      "fcm_dpo/beta": 0.003306058933958411,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 58.4079704284668,
      "fcm_dpo/q_t": 0.4554460644721985,
      "grad_norm": 12.692896842956543,
      "learning_rate": 2.963665913810451e-08,
      "logits/chosen": 0.8867205381393433,
      "logits/rejected": 0.8692376017570496,
      "logps/chosen": -272.46649169921875,
      "logps/ref_chosen": -62.649261474609375,
      "logps/ref_rejected": -75.4298324584961,
      "logps/rejected": -343.655029296875,
      "loss": 1.2721,
      "margin_dpo/margin_mean": 58.4079704284668,
      "margin_dpo/margin_std": 159.5178985595703,
      "step": 569
    },
    {
      "KL/chosen_KL_mean": -184.9730987548828,
      "KL/mean": -242.1673126220703,
      "KL/rejected_KL_mean": -299.36151123046875,
      "KL/std": 139.96983337402344,
      "epoch": 0.8616780045351474,
      "fcm_dpo/beta": 0.003306058933958411,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 114.38844299316406,
      "fcm_dpo/q_t": 0.41404989361763,
      "grad_norm": 12.19923210144043,
      "learning_rate": 2.9015298217712453e-08,
      "logits/chosen": 0.928882896900177,
      "logits/rejected": 0.8641595244407654,
      "logps/chosen": -235.014892578125,
      "logps/ref_chosen": -50.04179382324219,
      "logps/ref_rejected": -78.27146911621094,
      "logps/rejected": -377.63299560546875,
      "loss": 1.1134,
      "margin_dpo/margin_mean": 114.38844299316406,
      "margin_dpo/margin_std": 166.88583374023438,
      "step": 570
    },
    {
      "KL/chosen_KL_mean": -192.01284790039062,
      "KL/mean": -229.24810791015625,
      "KL/rejected_KL_mean": -266.4833679199219,
      "KL/std": 138.1226043701172,
      "epoch": 0.8631897203325775,
      "fcm_dpo/beta": 0.003306058933958411,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.47052001953125,
      "fcm_dpo/q_t": 0.44437307119369507,
      "grad_norm": 8.68694019317627,
      "learning_rate": 2.840011871446962e-08,
      "logits/chosen": 0.9437531232833862,
      "logits/rejected": 0.9379255771636963,
      "logps/chosen": -245.669677734375,
      "logps/ref_chosen": -53.65681457519531,
      "logps/ref_rejected": -66.13298034667969,
      "logps/rejected": -332.6163330078125,
      "loss": 1.2239,
      "margin_dpo/margin_mean": 74.47051239013672,
      "margin_dpo/margin_std": 161.92054748535156,
      "step": 571
    },
    {
      "KL/chosen_KL_mean": -208.37600708007812,
      "KL/mean": -239.53163146972656,
      "KL/rejected_KL_mean": -270.687255859375,
      "KL/std": 139.2075653076172,
      "epoch": 0.8647014361300076,
      "fcm_dpo/beta": 0.003306058933958411,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 62.31123352050781,
      "fcm_dpo/q_t": 0.45096078515052795,
      "grad_norm": 12.564879417419434,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 0.9028031826019287,
      "logits/rejected": 0.9370099306106567,
      "logps/chosen": -283.1939392089844,
      "logps/ref_chosen": -74.81792449951172,
      "logps/ref_rejected": -65.88681030273438,
      "logps/rejected": -336.5740661621094,
      "loss": 1.2508,
      "margin_dpo/margin_mean": 62.31123352050781,
      "margin_dpo/margin_std": 148.28436279296875,
      "step": 572
    },
    {
      "KL/chosen_KL_mean": -219.3136749267578,
      "KL/mean": -264.943603515625,
      "KL/rejected_KL_mean": -310.57354736328125,
      "KL/std": 160.91592407226562,
      "epoch": 0.8662131519274376,
      "fcm_dpo/beta": 0.003306058933958411,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.25989532470703,
      "fcm_dpo/q_t": 0.4338764250278473,
      "grad_norm": 12.651811599731445,
      "learning_rate": 2.718837261761528e-08,
      "logits/chosen": 0.9531590938568115,
      "logits/rejected": 0.9273461103439331,
      "logps/chosen": -288.039306640625,
      "logps/ref_chosen": -68.72564697265625,
      "logps/ref_rejected": -88.16201782226562,
      "logps/rejected": -398.735595703125,
      "loss": 1.2021,
      "margin_dpo/margin_mean": 91.25990295410156,
      "margin_dpo/margin_std": 193.46453857421875,
      "step": 573
    },
    {
      "KL/chosen_KL_mean": -191.57583618164062,
      "KL/mean": -251.47848510742188,
      "KL/rejected_KL_mean": -311.3811340332031,
      "KL/std": 150.62881469726562,
      "epoch": 0.8677248677248677,
      "fcm_dpo/beta": 0.003287060186266899,
      "fcm_dpo/delta": -0.05779891833662987,
      "fcm_dpo/margin": 119.80528259277344,
      "fcm_dpo/q_t": 0.4098057150840759,
      "grad_norm": 9.95071792602539,
      "learning_rate": 2.659183991914696e-08,
      "logits/chosen": 0.9709665775299072,
      "logits/rejected": 0.9339627623558044,
      "logps/chosen": -247.88925170898438,
      "logps/ref_chosen": -56.31340026855469,
      "logps/ref_rejected": -83.91553497314453,
      "logps/rejected": -395.29669189453125,
      "loss": 1.0883,
      "margin_dpo/margin_mean": 119.80528259277344,
      "margin_dpo/margin_std": 150.09042358398438,
      "step": 574
    },
    {
      "KL/chosen_KL_mean": -199.07955932617188,
      "KL/mean": -242.06597900390625,
      "KL/rejected_KL_mean": -285.05242919921875,
      "KL/std": 151.01272583007812,
      "epoch": 0.8692365835222978,
      "fcm_dpo/beta": 0.0032680616714060307,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 85.97285461425781,
      "fcm_dpo/q_t": 0.43860968947410583,
      "grad_norm": 11.692818641662598,
      "learning_rate": 2.600155642716606e-08,
      "logits/chosen": 0.9394083023071289,
      "logits/rejected": 0.8856104016304016,
      "logps/chosen": -263.6636962890625,
      "logps/ref_chosen": -64.5841293334961,
      "logps/ref_rejected": -93.47034454345703,
      "logps/rejected": -378.52276611328125,
      "loss": 1.2128,
      "margin_dpo/margin_mean": 85.97285461425781,
      "margin_dpo/margin_std": 182.2880096435547,
      "step": 575
    },
    {
      "KL/chosen_KL_mean": -173.85931396484375,
      "KL/mean": -238.97288513183594,
      "KL/rejected_KL_mean": -304.08648681640625,
      "KL/std": 149.41693115234375,
      "epoch": 0.8707482993197279,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": -0.06305931508541107,
      "fcm_dpo/margin": 130.22715759277344,
      "fcm_dpo/q_t": 0.40280795097351074,
      "grad_norm": 11.600000381469727,
      "learning_rate": 2.5417538653170754e-08,
      "logits/chosen": 0.9049896001815796,
      "logits/rejected": 0.8447592854499817,
      "logps/chosen": -227.1398468017578,
      "logps/ref_chosen": -53.28052520751953,
      "logps/ref_rejected": -84.2000503540039,
      "logps/rejected": -388.2864990234375,
      "loss": 1.0805,
      "margin_dpo/margin_mean": 130.22715759277344,
      "margin_dpo/margin_std": 166.29541015625,
      "step": 576
    },
    {
      "KL/chosen_KL_mean": -195.810791015625,
      "KL/mean": -234.27926635742188,
      "KL/rejected_KL_mean": -272.74774169921875,
      "KL/std": 146.26927185058594,
      "epoch": 0.872260015117158,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.93694305419922,
      "fcm_dpo/q_t": 0.4415355920791626,
      "grad_norm": 11.450098991394043,
      "learning_rate": 2.4839802933393607e-08,
      "logits/chosen": 0.9213416576385498,
      "logits/rejected": 0.926072895526886,
      "logps/chosen": -258.1354675292969,
      "logps/ref_chosen": -62.32468795776367,
      "logps/ref_rejected": -67.300537109375,
      "logps/rejected": -340.04827880859375,
      "loss": 1.2163,
      "margin_dpo/margin_mean": 76.93695068359375,
      "margin_dpo/margin_std": 156.5452117919922,
      "step": 577
    },
    {
      "KL/chosen_KL_mean": -191.14976501464844,
      "KL/mean": -227.45904541015625,
      "KL/rejected_KL_mean": -263.7683410644531,
      "KL/std": 155.82366943359375,
      "epoch": 0.873771730914588,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 72.61858367919922,
      "fcm_dpo/q_t": 0.44642916321754456,
      "grad_norm": 11.682079315185547,
      "learning_rate": 2.4268365428344733e-08,
      "logits/chosen": 0.9547611474990845,
      "logits/rejected": 0.9533353447914124,
      "logps/chosen": -247.80532836914062,
      "logps/ref_chosen": -56.65557861328125,
      "logps/ref_rejected": -68.21835327148438,
      "logps/rejected": -331.9866943359375,
      "loss": 1.2429,
      "margin_dpo/margin_mean": 72.61858367919922,
      "margin_dpo/margin_std": 178.0867919921875,
      "step": 578
    },
    {
      "KL/chosen_KL_mean": -193.22874450683594,
      "KL/mean": -247.7565460205078,
      "KL/rejected_KL_mean": -302.28436279296875,
      "KL/std": 136.99658203125,
      "epoch": 0.8752834467120182,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 109.05561828613281,
      "fcm_dpo/q_t": 0.41695117950439453,
      "grad_norm": 11.134893417358398,
      "learning_rate": 2.3703242122359357e-08,
      "logits/chosen": 0.8828812837600708,
      "logits/rejected": 0.8775182962417603,
      "logps/chosen": -250.03842163085938,
      "logps/ref_chosen": -56.809661865234375,
      "logps/ref_rejected": -68.09613037109375,
      "logps/rejected": -370.3804931640625,
      "loss": 1.1188,
      "margin_dpo/margin_mean": 109.05561828613281,
      "margin_dpo/margin_std": 147.0736083984375,
      "step": 579
    },
    {
      "KL/chosen_KL_mean": -197.42172241210938,
      "KL/mean": -250.31234741210938,
      "KL/rejected_KL_mean": -303.2030029296875,
      "KL/std": 168.3189697265625,
      "epoch": 0.8767951625094482,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 105.78125762939453,
      "fcm_dpo/q_t": 0.42473307251930237,
      "grad_norm": 12.491499900817871,
      "learning_rate": 2.3144448823151392e-08,
      "logits/chosen": 0.9132966995239258,
      "logits/rejected": 0.8900126218795776,
      "logps/chosen": -255.12184143066406,
      "logps/ref_chosen": -57.70011520385742,
      "logps/ref_rejected": -77.90664672851562,
      "logps/rejected": -381.109619140625,
      "loss": 1.1594,
      "margin_dpo/margin_mean": 105.78126525878906,
      "margin_dpo/margin_std": 187.40509033203125,
      "step": 580
    },
    {
      "KL/chosen_KL_mean": -212.98367309570312,
      "KL/mean": -258.75372314453125,
      "KL/rejected_KL_mean": -304.5237731933594,
      "KL/std": 145.23110961914062,
      "epoch": 0.8783068783068783,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.54010009765625,
      "fcm_dpo/q_t": 0.4324272572994232,
      "grad_norm": 9.804346084594727,
      "learning_rate": 2.259200116137039e-08,
      "logits/chosen": 0.9398747086524963,
      "logits/rejected": 0.901383638381958,
      "logps/chosen": -272.3160400390625,
      "logps/ref_chosen": -59.332359313964844,
      "logps/ref_rejected": -83.64482116699219,
      "logps/rejected": -388.1685791015625,
      "loss": 1.1886,
      "margin_dpo/margin_mean": 91.54010009765625,
      "margin_dpo/margin_std": 175.1453399658203,
      "step": 581
    },
    {
      "KL/chosen_KL_mean": -196.52120971679688,
      "KL/mean": -236.0716094970703,
      "KL/rejected_KL_mean": -275.62200927734375,
      "KL/std": 129.60977172851562,
      "epoch": 0.8798185941043084,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.10078430175781,
      "fcm_dpo/q_t": 0.44115710258483887,
      "grad_norm": 8.99724292755127,
      "learning_rate": 2.204591459016525e-08,
      "logits/chosen": 0.9279496073722839,
      "logits/rejected": 0.9599611759185791,
      "logps/chosen": -260.68408203125,
      "logps/ref_chosen": -64.16285705566406,
      "logps/ref_rejected": -58.632896423339844,
      "logps/rejected": -334.2548828125,
      "loss": 1.2082,
      "margin_dpo/margin_mean": 79.10079193115234,
      "margin_dpo/margin_std": 153.8829345703125,
      "step": 582
    },
    {
      "KL/chosen_KL_mean": -191.3851776123047,
      "KL/mean": -253.97930908203125,
      "KL/rejected_KL_mean": -316.57342529296875,
      "KL/std": 171.81069946289062,
      "epoch": 0.8813303099017384,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 125.1882553100586,
      "fcm_dpo/q_t": 0.4104909300804138,
      "grad_norm": 11.005936622619629,
      "learning_rate": 2.1506204384751064e-08,
      "logits/chosen": 1.0140917301177979,
      "logits/rejected": 0.9544622898101807,
      "logps/chosen": -243.257568359375,
      "logps/ref_chosen": -51.87239456176758,
      "logps/ref_rejected": -83.86331176757812,
      "logps/rejected": -400.4367370605469,
      "loss": 1.1127,
      "margin_dpo/margin_mean": 125.18826293945312,
      "margin_dpo/margin_std": 195.55584716796875,
      "step": 583
    },
    {
      "KL/chosen_KL_mean": -175.76718139648438,
      "KL/mean": -224.52301025390625,
      "KL/rejected_KL_mean": -273.27886962890625,
      "KL/std": 139.64749145507812,
      "epoch": 0.8828420256991686,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 97.51168823242188,
      "fcm_dpo/q_t": 0.4288090467453003,
      "grad_norm": 13.306455612182617,
      "learning_rate": 2.09728856419826e-08,
      "logits/chosen": 1.0285711288452148,
      "logits/rejected": 0.957194983959198,
      "logps/chosen": -222.33856201171875,
      "logps/ref_chosen": -46.571388244628906,
      "logps/ref_rejected": -80.67969512939453,
      "logps/rejected": -353.95855712890625,
      "loss": 1.1691,
      "margin_dpo/margin_mean": 97.51168823242188,
      "margin_dpo/margin_std": 173.30181884765625,
      "step": 584
    },
    {
      "KL/chosen_KL_mean": -211.24838256835938,
      "KL/mean": -243.45950317382812,
      "KL/rejected_KL_mean": -275.6706237792969,
      "KL/std": 152.93052673339844,
      "epoch": 0.8843537414965986,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 64.42221069335938,
      "fcm_dpo/q_t": 0.45181816816329956,
      "grad_norm": 11.961204528808594,
      "learning_rate": 2.044597327993153e-08,
      "logits/chosen": 0.9316689968109131,
      "logits/rejected": 0.8908916711807251,
      "logps/chosen": -269.3729248046875,
      "logps/ref_chosen": -58.124534606933594,
      "logps/ref_rejected": -79.00538635253906,
      "logps/rejected": -354.676025390625,
      "loss": 1.2475,
      "margin_dpo/margin_mean": 64.42221069335938,
      "margin_dpo/margin_std": 151.98226928710938,
      "step": 585
    },
    {
      "KL/chosen_KL_mean": -189.38601684570312,
      "KL/mean": -230.9322052001953,
      "KL/rejected_KL_mean": -272.4783935546875,
      "KL/std": 120.71141052246094,
      "epoch": 0.8858654572940288,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.09239196777344,
      "fcm_dpo/q_t": 0.4359871745109558,
      "grad_norm": 10.963727951049805,
      "learning_rate": 1.9925482037469187e-08,
      "logits/chosen": 0.9770946502685547,
      "logits/rejected": 0.9540187120437622,
      "logps/chosen": -243.48764038085938,
      "logps/ref_chosen": -54.10163879394531,
      "logps/ref_rejected": -63.72113037109375,
      "logps/rejected": -336.19952392578125,
      "loss": 1.1685,
      "margin_dpo/margin_mean": 83.09239959716797,
      "margin_dpo/margin_std": 113.74529266357422,
      "step": 586
    },
    {
      "KL/chosen_KL_mean": -208.8435821533203,
      "KL/mean": -248.68414306640625,
      "KL/rejected_KL_mean": -288.5246887207031,
      "KL/std": 154.78927612304688,
      "epoch": 0.8873771730914588,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.68111419677734,
      "fcm_dpo/q_t": 0.43557557463645935,
      "grad_norm": 14.696372985839844,
      "learning_rate": 1.9411426473854687e-08,
      "logits/chosen": 0.9748609066009521,
      "logits/rejected": 0.9778647422790527,
      "logps/chosen": -272.2607727050781,
      "logps/ref_chosen": -63.41719436645508,
      "logps/ref_rejected": -63.47003936767578,
      "logps/rejected": -351.9947509765625,
      "loss": 1.2461,
      "margin_dpo/margin_mean": 79.68111419677734,
      "margin_dpo/margin_std": 209.5115966796875,
      "step": 587
    },
    {
      "KL/chosen_KL_mean": -211.2041015625,
      "KL/mean": -259.3454284667969,
      "KL/rejected_KL_mean": -307.48675537109375,
      "KL/std": 155.98251342773438,
      "epoch": 0.8888888888888888,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 96.28263854980469,
      "fcm_dpo/q_t": 0.42922869324684143,
      "grad_norm": 11.775358200073242,
      "learning_rate": 1.890382096832699e-08,
      "logits/chosen": 0.9626029133796692,
      "logits/rejected": 0.9362854361534119,
      "logps/chosen": -273.4051513671875,
      "logps/ref_chosen": -62.20103454589844,
      "logps/ref_rejected": -82.10249328613281,
      "logps/rejected": -389.5892333984375,
      "loss": 1.1927,
      "margin_dpo/margin_mean": 96.28263854980469,
      "margin_dpo/margin_std": 196.510986328125,
      "step": 588
    },
    {
      "KL/chosen_KL_mean": -196.3404541015625,
      "KL/mean": -242.31954956054688,
      "KL/rejected_KL_mean": -288.29864501953125,
      "KL/std": 142.04229736328125,
      "epoch": 0.890400604686319,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.95817565917969,
      "fcm_dpo/q_t": 0.4290810525417328,
      "grad_norm": 8.143330574035645,
      "learning_rate": 1.840267971970344e-08,
      "logits/chosen": 0.9108644723892212,
      "logits/rejected": 0.8894894123077393,
      "logps/chosen": -253.0540771484375,
      "logps/ref_chosen": -56.71361541748047,
      "logps/ref_rejected": -76.7366943359375,
      "logps/rejected": -365.03533935546875,
      "loss": 1.1599,
      "margin_dpo/margin_mean": 91.95817565917969,
      "margin_dpo/margin_std": 138.236083984375,
      "step": 589
    },
    {
      "KL/chosen_KL_mean": -212.30551147460938,
      "KL/mean": -253.93572998046875,
      "KL/rejected_KL_mean": -295.56597900390625,
      "KL/std": 127.766845703125,
      "epoch": 0.891912320483749,
      "fcm_dpo/beta": 0.003227104200050235,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 83.26048278808594,
      "fcm_dpo/q_t": 0.43615347146987915,
      "grad_norm": 13.782276153564453,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 0.9059382677078247,
      "logits/rejected": 0.8786763548851013,
      "logps/chosen": -278.8193359375,
      "logps/ref_chosen": -66.5138168334961,
      "logps/ref_rejected": -85.70820617675781,
      "logps/rejected": -381.274169921875,
      "loss": 1.1835,
      "margin_dpo/margin_mean": 83.26048278808594,
      "margin_dpo/margin_std": 136.46484375,
      "step": 590
    },
    {
      "KL/chosen_KL_mean": -178.59005737304688,
      "KL/mean": -238.56552124023438,
      "KL/rejected_KL_mean": -298.5409851074219,
      "KL/std": 159.26284790039062,
      "epoch": 0.8934240362811792,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": -0.05250098556280136,
      "fcm_dpo/margin": 119.950927734375,
      "fcm_dpo/q_t": 0.41484156250953674,
      "grad_norm": 10.59327220916748,
      "learning_rate": 1.7419845883949098e-08,
      "logits/chosen": 0.9709192514419556,
      "logits/rejected": 0.9397531747817993,
      "logps/chosen": -239.2872314453125,
      "logps/ref_chosen": -60.697181701660156,
      "logps/ref_rejected": -86.12278747558594,
      "logps/rejected": -384.66375732421875,
      "loss": 1.1338,
      "margin_dpo/margin_mean": 119.950927734375,
      "margin_dpo/margin_std": 200.14447021484375,
      "step": 591
    },
    {
      "KL/chosen_KL_mean": -193.6923065185547,
      "KL/mean": -239.26446533203125,
      "KL/rejected_KL_mean": -284.836669921875,
      "KL/std": 129.58926391601562,
      "epoch": 0.8949357520786092,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.14432525634766,
      "fcm_dpo/q_t": 0.43314582109451294,
      "grad_norm": 10.38012981414795,
      "learning_rate": 1.6938180788793556e-08,
      "logits/chosen": 0.9803141355514526,
      "logits/rejected": 0.8997335433959961,
      "logps/chosen": -244.92962646484375,
      "logps/ref_chosen": -51.237327575683594,
      "logps/ref_rejected": -81.60242462158203,
      "logps/rejected": -366.4390869140625,
      "loss": 1.1699,
      "margin_dpo/margin_mean": 91.14432525634766,
      "margin_dpo/margin_std": 148.00392150878906,
      "step": 592
    },
    {
      "KL/chosen_KL_mean": -159.02511596679688,
      "KL/mean": -209.68447875976562,
      "KL/rejected_KL_mean": -260.3438720703125,
      "KL/std": 127.21440124511719,
      "epoch": 0.8964474678760394,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 101.31876373291016,
      "fcm_dpo/q_t": 0.42233753204345703,
      "grad_norm": 11.570535659790039,
      "learning_rate": 1.6463034933723336e-08,
      "logits/chosen": 0.9546194076538086,
      "logits/rejected": 0.8912557363510132,
      "logps/chosen": -201.10511779785156,
      "logps/ref_chosen": -42.08000183105469,
      "logps/ref_rejected": -68.47499084472656,
      "logps/rejected": -328.81884765625,
      "loss": 1.1397,
      "margin_dpo/margin_mean": 101.31876373291016,
      "margin_dpo/margin_std": 143.3717041015625,
      "step": 593
    },
    {
      "KL/chosen_KL_mean": -208.95913696289062,
      "KL/mean": -247.4608612060547,
      "KL/rejected_KL_mean": -285.96258544921875,
      "KL/std": 137.06649780273438,
      "epoch": 0.8979591836734694,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 77.00344848632812,
      "fcm_dpo/q_t": 0.4417816996574402,
      "grad_norm": 11.07624340057373,
      "learning_rate": 1.5994421609589385e-08,
      "logits/chosen": 0.90055912733078,
      "logits/rejected": 0.896392822265625,
      "logps/chosen": -272.6177978515625,
      "logps/ref_chosen": -63.658668518066406,
      "logps/ref_rejected": -70.35597229003906,
      "logps/rejected": -356.31854248046875,
      "loss": 1.2068,
      "margin_dpo/margin_mean": 77.00344848632812,
      "margin_dpo/margin_std": 142.98709106445312,
      "step": 594
    },
    {
      "KL/chosen_KL_mean": -189.4154510498047,
      "KL/mean": -244.88433837890625,
      "KL/rejected_KL_mean": -300.35321044921875,
      "KL/std": 143.3093719482422,
      "epoch": 0.8994708994708994,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 110.93778991699219,
      "fcm_dpo/q_t": 0.41879788041114807,
      "grad_norm": 9.869068145751953,
      "learning_rate": 1.553235392451377e-08,
      "logits/chosen": 0.9638773202896118,
      "logits/rejected": 0.9036699533462524,
      "logps/chosen": -245.63421630859375,
      "logps/ref_chosen": -56.21875762939453,
      "logps/ref_rejected": -83.95773315429688,
      "logps/rejected": -384.31097412109375,
      "loss": 1.135,
      "margin_dpo/margin_mean": 110.93778991699219,
      "margin_dpo/margin_std": 173.9689178466797,
      "step": 595
    },
    {
      "KL/chosen_KL_mean": -220.83944702148438,
      "KL/mean": -240.2639923095703,
      "KL/rejected_KL_mean": -259.68853759765625,
      "KL/std": 130.09234619140625,
      "epoch": 0.9009826152683296,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 38.849117279052734,
      "fcm_dpo/q_t": 0.47059816122055054,
      "grad_norm": 12.449370384216309,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": 0.9170597195625305,
      "logits/rejected": 0.9390828609466553,
      "logps/chosen": -289.3203125,
      "logps/ref_chosen": -68.48088073730469,
      "logps/ref_rejected": -61.732967376708984,
      "logps/rejected": -321.4215087890625,
      "loss": 1.3083,
      "margin_dpo/margin_mean": 38.849117279052734,
      "margin_dpo/margin_std": 128.81515502929688,
      "step": 596
    },
    {
      "KL/chosen_KL_mean": -168.60281372070312,
      "KL/mean": -208.48593139648438,
      "KL/rejected_KL_mean": -248.3690185546875,
      "KL/std": 111.42578125,
      "epoch": 0.9024943310657596,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 79.7662353515625,
      "fcm_dpo/q_t": 0.43913787603378296,
      "grad_norm": 10.048505783081055,
      "learning_rate": 1.4627906988186111e-08,
      "logits/chosen": 0.9357984066009521,
      "logits/rejected": 0.9276165962219238,
      "logps/chosen": -217.4603271484375,
      "logps/ref_chosen": -48.85750961303711,
      "logps/ref_rejected": -55.068084716796875,
      "logps/rejected": -303.4371337890625,
      "loss": 1.1861,
      "margin_dpo/margin_mean": 79.76622772216797,
      "margin_dpo/margin_std": 123.33467102050781,
      "step": 597
    },
    {
      "KL/chosen_KL_mean": -221.0791015625,
      "KL/mean": -251.432373046875,
      "KL/rejected_KL_mean": -281.78564453125,
      "KL/std": 139.46469116210938,
      "epoch": 0.9040060468631897,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.70652389526367,
      "fcm_dpo/q_t": 0.4551101326942444,
      "grad_norm": 9.732013702392578,
      "learning_rate": 1.4185553036259095e-08,
      "logits/chosen": 0.9217236042022705,
      "logits/rejected": 0.8709498047828674,
      "logps/chosen": -279.96624755859375,
      "logps/ref_chosen": -58.88715362548828,
      "logps/ref_rejected": -81.43145751953125,
      "logps/rejected": -363.21710205078125,
      "loss": 1.2562,
      "margin_dpo/margin_mean": 60.70652770996094,
      "margin_dpo/margin_std": 148.96145629882812,
      "step": 598
    },
    {
      "KL/chosen_KL_mean": -220.0737762451172,
      "KL/mean": -253.25234985351562,
      "KL/rejected_KL_mean": -286.430908203125,
      "KL/std": 134.68807983398438,
      "epoch": 0.9055177626606198,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 66.35713958740234,
      "fcm_dpo/q_t": 0.45046699047088623,
      "grad_norm": 11.252570152282715,
      "learning_rate": 1.3749795321332885e-08,
      "logits/chosen": 0.982478141784668,
      "logits/rejected": 0.9580433964729309,
      "logps/chosen": -277.68096923828125,
      "logps/ref_chosen": -57.60719299316406,
      "logps/ref_rejected": -71.80469512939453,
      "logps/rejected": -358.235595703125,
      "loss": 1.247,
      "margin_dpo/margin_mean": 66.35713958740234,
      "margin_dpo/margin_std": 157.38815307617188,
      "step": 599
    },
    {
      "KL/chosen_KL_mean": -198.6517333984375,
      "KL/mean": -238.66195678710938,
      "KL/rejected_KL_mean": -278.67218017578125,
      "KL/std": 147.49740600585938,
      "epoch": 0.9070294784580499,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.02043914794922,
      "fcm_dpo/q_t": 0.44202420115470886,
      "grad_norm": 12.139007568359375,
      "learning_rate": 1.3320646032487393e-08,
      "logits/chosen": 0.93305504322052,
      "logits/rejected": 0.8942174911499023,
      "logps/chosen": -257.09405517578125,
      "logps/ref_chosen": -58.44231414794922,
      "logps/ref_rejected": -83.64639282226562,
      "logps/rejected": -362.3185729980469,
      "loss": 1.2126,
      "margin_dpo/margin_mean": 80.02043151855469,
      "margin_dpo/margin_std": 163.208251953125,
      "step": 600
    },
    {
      "KL/chosen_KL_mean": -181.49945068359375,
      "KL/mean": -234.6309051513672,
      "KL/rejected_KL_mean": -287.76239013671875,
      "KL/std": 156.81146240234375,
      "epoch": 0.90854119425548,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 106.2629165649414,
      "fcm_dpo/q_t": 0.4221433997154236,
      "grad_norm": 8.865299224853516,
      "learning_rate": 1.2898117173950868e-08,
      "logits/chosen": 0.9232147336006165,
      "logits/rejected": 0.8683621883392334,
      "logps/chosen": -237.09378051757812,
      "logps/ref_chosen": -55.59432601928711,
      "logps/ref_rejected": -83.68630981445312,
      "logps/rejected": -371.44866943359375,
      "loss": 1.1508,
      "margin_dpo/margin_mean": 106.2629165649414,
      "margin_dpo/margin_std": 176.8553009033203,
      "step": 601
    },
    {
      "KL/chosen_KL_mean": -172.61276245117188,
      "KL/mean": -217.75296020507812,
      "KL/rejected_KL_mean": -262.89312744140625,
      "KL/std": 132.79425048828125,
      "epoch": 0.91005291005291,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 90.28034973144531,
      "fcm_dpo/q_t": 0.43157410621643066,
      "grad_norm": 10.930935859680176,
      "learning_rate": 1.2482220564763667e-08,
      "logits/chosen": 0.9340589046478271,
      "logits/rejected": 0.9145586490631104,
      "logps/chosen": -228.9619598388672,
      "logps/ref_chosen": -56.349185943603516,
      "logps/ref_rejected": -71.9959716796875,
      "logps/rejected": -334.88909912109375,
      "loss": 1.1684,
      "margin_dpo/margin_mean": 90.28034973144531,
      "margin_dpo/margin_std": 141.6522216796875,
      "step": 602
    },
    {
      "KL/chosen_KL_mean": -186.05422973632812,
      "KL/mean": -235.9767303466797,
      "KL/rejected_KL_mean": -285.89923095703125,
      "KL/std": 139.99981689453125,
      "epoch": 0.9115646258503401,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 99.84503173828125,
      "fcm_dpo/q_t": 0.427456796169281,
      "grad_norm": 9.996122360229492,
      "learning_rate": 1.2072967838448051e-08,
      "logits/chosen": 0.9382352828979492,
      "logits/rejected": 0.9089177846908569,
      "logps/chosen": -239.2226104736328,
      "logps/ref_chosen": -53.16838836669922,
      "logps/ref_rejected": -73.8604736328125,
      "logps/rejected": -359.7597351074219,
      "loss": 1.1586,
      "margin_dpo/margin_mean": 99.84503173828125,
      "margin_dpo/margin_std": 166.58010864257812,
      "step": 603
    },
    {
      "KL/chosen_KL_mean": -187.37911987304688,
      "KL/mean": -224.50912475585938,
      "KL/rejected_KL_mean": -261.63909912109375,
      "KL/std": 124.96034240722656,
      "epoch": 0.9130763416477702,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.25997161865234,
      "fcm_dpo/q_t": 0.4446590542793274,
      "grad_norm": 10.817628860473633,
      "learning_rate": 1.1670370442682459e-08,
      "logits/chosen": 0.8739547729492188,
      "logits/rejected": 0.8863567113876343,
      "logps/chosen": -260.028564453125,
      "logps/ref_chosen": -72.64942169189453,
      "logps/ref_rejected": -69.8792724609375,
      "logps/rejected": -331.51837158203125,
      "loss": 1.2305,
      "margin_dpo/margin_mean": 74.25996398925781,
      "margin_dpo/margin_std": 166.822021484375,
      "step": 604
    },
    {
      "KL/chosen_KL_mean": -206.32119750976562,
      "KL/mean": -251.2423553466797,
      "KL/rejected_KL_mean": -296.16351318359375,
      "KL/std": 132.05099487304688,
      "epoch": 0.9145880574452003,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.84228515625,
      "fcm_dpo/q_t": 0.43243077397346497,
      "grad_norm": 10.091094970703125,
      "learning_rate": 1.1274439638981532e-08,
      "logits/chosen": 0.9632163643836975,
      "logits/rejected": 0.9292812347412109,
      "logps/chosen": -267.9340515136719,
      "logps/ref_chosen": -61.61284637451172,
      "logps/ref_rejected": -79.34398651123047,
      "logps/rejected": -375.50750732421875,
      "loss": 1.1792,
      "margin_dpo/margin_mean": 89.84228515625,
      "margin_dpo/margin_std": 154.51718139648438,
      "step": 605
    },
    {
      "KL/chosen_KL_mean": -179.38706970214844,
      "KL/mean": -236.68408203125,
      "KL/rejected_KL_mean": -293.9811096191406,
      "KL/std": 143.79409790039062,
      "epoch": 0.9160997732426304,
      "fcm_dpo/beta": 0.003193396143615246,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 114.59405517578125,
      "fcm_dpo/q_t": 0.41510650515556335,
      "grad_norm": 10.840108871459961,
      "learning_rate": 1.0885186502381016e-08,
      "logits/chosen": 0.9171103239059448,
      "logits/rejected": 0.8769623041152954,
      "logps/chosen": -233.85130310058594,
      "logps/ref_chosen": -54.46424102783203,
      "logps/ref_rejected": -79.62708282470703,
      "logps/rejected": -373.6081848144531,
      "loss": 1.1131,
      "margin_dpo/margin_mean": 114.59405517578125,
      "margin_dpo/margin_std": 156.9613494873047,
      "step": 606
    },
    {
      "KL/chosen_KL_mean": -200.49571228027344,
      "KL/mean": -248.2861328125,
      "KL/rejected_KL_mean": -296.0765380859375,
      "KL/std": 139.8798370361328,
      "epoch": 0.9176114890400605,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": -0.04817618429660797,
      "fcm_dpo/margin": 95.58084106445312,
      "fcm_dpo/q_t": 0.4292658865451813,
      "grad_norm": 10.28044605255127,
      "learning_rate": 1.0502621921127774e-08,
      "logits/chosen": 0.9089508056640625,
      "logits/rejected": 0.9013663530349731,
      "logps/chosen": -263.3565673828125,
      "logps/ref_chosen": -62.86086654663086,
      "logps/ref_rejected": -72.5501937866211,
      "logps/rejected": -368.6267395019531,
      "loss": 1.1738,
      "margin_dpo/margin_mean": 95.58084106445312,
      "margin_dpo/margin_std": 158.3627471923828,
      "step": 607
    },
    {
      "KL/chosen_KL_mean": -206.0166473388672,
      "KL/mean": -258.2984313964844,
      "KL/rejected_KL_mean": -310.5802001953125,
      "KL/std": 155.61599731445312,
      "epoch": 0.9191232048374905,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 104.56355285644531,
      "fcm_dpo/q_t": 0.4240010380744934,
      "grad_norm": 10.039460182189941,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 0.9120040535926819,
      "logits/rejected": 0.8582882881164551,
      "logps/chosen": -269.1973876953125,
      "logps/ref_chosen": -63.18071746826172,
      "logps/ref_rejected": -99.15888214111328,
      "logps/rejected": -409.73907470703125,
      "loss": 1.1448,
      "margin_dpo/margin_mean": 104.56355285644531,
      "margin_dpo/margin_std": 163.28131103515625,
      "step": 608
    },
    {
      "KL/chosen_KL_mean": -180.6420440673828,
      "KL/mean": -230.55471801757812,
      "KL/rejected_KL_mean": -280.4674072265625,
      "KL/std": 127.04386901855469,
      "epoch": 0.9206349206349206,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 99.82534790039062,
      "fcm_dpo/q_t": 0.42423099279403687,
      "grad_norm": 10.703655242919922,
      "learning_rate": 9.757601041885694e-09,
      "logits/chosen": 0.9837725162506104,
      "logits/rejected": 0.9593820571899414,
      "logps/chosen": -229.2652587890625,
      "logps/ref_chosen": -48.62322235107422,
      "logps/ref_rejected": -68.28271484375,
      "logps/rejected": -348.7501220703125,
      "loss": 1.1328,
      "margin_dpo/margin_mean": 99.82534790039062,
      "margin_dpo/margin_std": 120.62831115722656,
      "step": 609
    },
    {
      "KL/chosen_KL_mean": -210.98101806640625,
      "KL/mean": -253.97109985351562,
      "KL/rejected_KL_mean": -296.961181640625,
      "KL/std": 153.73622131347656,
      "epoch": 0.9221466364323507,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 85.98018646240234,
      "fcm_dpo/q_t": 0.434524804353714,
      "grad_norm": 13.81159782409668,
      "learning_rate": 9.395165583732379e-09,
      "logits/chosen": 0.9371830224990845,
      "logits/rejected": 0.937787652015686,
      "logps/chosen": -283.6461181640625,
      "logps/ref_chosen": -72.66513061523438,
      "logps/ref_rejected": -87.15310668945312,
      "logps/rejected": -384.1142883300781,
      "loss": 1.2179,
      "margin_dpo/margin_mean": 85.98019409179688,
      "margin_dpo/margin_std": 185.64382934570312,
      "step": 610
    },
    {
      "KL/chosen_KL_mean": -180.6585693359375,
      "KL/mean": -221.9508056640625,
      "KL/rejected_KL_mean": -263.2430419921875,
      "KL/std": 132.38990783691406,
      "epoch": 0.9236583522297808,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 82.58445739746094,
      "fcm_dpo/q_t": 0.4378008246421814,
      "grad_norm": 10.246078491210938,
      "learning_rate": 9.03946036001449e-09,
      "logits/chosen": 0.9471895098686218,
      "logits/rejected": 0.917996883392334,
      "logps/chosen": -228.9671630859375,
      "logps/ref_chosen": -48.30857849121094,
      "logps/ref_rejected": -70.6141128540039,
      "logps/rejected": -333.857177734375,
      "loss": 1.1824,
      "margin_dpo/margin_mean": 82.58445739746094,
      "margin_dpo/margin_std": 128.2288055419922,
      "step": 611
    },
    {
      "KL/chosen_KL_mean": -200.29458618164062,
      "KL/mean": -261.31329345703125,
      "KL/rejected_KL_mean": -322.33203125,
      "KL/std": 136.66183471679688,
      "epoch": 0.9251700680272109,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 122.03746032714844,
      "fcm_dpo/q_t": 0.4092690050601959,
      "grad_norm": 10.354504585266113,
      "learning_rate": 8.690495320571839e-09,
      "logits/chosen": 0.8809033632278442,
      "logits/rejected": 0.8318875432014465,
      "logps/chosen": -261.526123046875,
      "logps/ref_chosen": -61.23155975341797,
      "logps/ref_rejected": -94.37979888916016,
      "logps/rejected": -416.71185302734375,
      "loss": 1.0896,
      "margin_dpo/margin_mean": 122.03746795654297,
      "margin_dpo/margin_std": 148.84161376953125,
      "step": 612
    },
    {
      "KL/chosen_KL_mean": -179.49685668945312,
      "KL/mean": -227.58367919921875,
      "KL/rejected_KL_mean": -275.6705322265625,
      "KL/std": 156.89382934570312,
      "epoch": 0.926681783824641,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 96.17366027832031,
      "fcm_dpo/q_t": 0.42703235149383545,
      "grad_norm": 9.231825828552246,
      "learning_rate": 8.348280226706722e-09,
      "logits/chosen": 0.9118015766143799,
      "logits/rejected": 0.9143195152282715,
      "logps/chosen": -233.47996520996094,
      "logps/ref_chosen": -53.98310852050781,
      "logps/ref_rejected": -58.32208251953125,
      "logps/rejected": -333.99261474609375,
      "loss": 1.1612,
      "margin_dpo/margin_mean": 96.17366027832031,
      "margin_dpo/margin_std": 152.2427978515625,
      "step": 613
    },
    {
      "KL/chosen_KL_mean": -198.9600067138672,
      "KL/mean": -245.7151641845703,
      "KL/rejected_KL_mean": -292.47027587890625,
      "KL/std": 128.48739624023438,
      "epoch": 0.9281934996220711,
      "fcm_dpo/beta": 0.003162774723023176,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 93.5102767944336,
      "fcm_dpo/q_t": 0.4299646317958832,
      "grad_norm": 10.526408195495605,
      "learning_rate": 8.012824650910937e-09,
      "logits/chosen": 0.9376695156097412,
      "logits/rejected": 0.9362810850143433,
      "logps/chosen": -259.20306396484375,
      "logps/ref_chosen": -60.24303436279297,
      "logps/ref_rejected": -72.26258850097656,
      "logps/rejected": -364.7328796386719,
      "loss": 1.169,
      "margin_dpo/margin_mean": 93.5102767944336,
      "margin_dpo/margin_std": 148.775634765625,
      "step": 614
    },
    {
      "KL/chosen_KL_mean": -211.63214111328125,
      "KL/mean": -268.7047119140625,
      "KL/rejected_KL_mean": -325.77728271484375,
      "KL/std": 174.5651092529297,
      "epoch": 0.9297052154195011,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": -0.05142123997211456,
      "fcm_dpo/margin": 114.14509582519531,
      "fcm_dpo/q_t": 0.41916823387145996,
      "grad_norm": 9.828910827636719,
      "learning_rate": 7.684137976598088e-09,
      "logits/chosen": 0.9190930128097534,
      "logits/rejected": 0.8757984042167664,
      "logps/chosen": -283.726806640625,
      "logps/ref_chosen": -72.09467315673828,
      "logps/ref_rejected": -104.02980041503906,
      "logps/rejected": -429.80706787109375,
      "loss": 1.1426,
      "margin_dpo/margin_mean": 114.14510345458984,
      "margin_dpo/margin_std": 186.59616088867188,
      "step": 615
    },
    {
      "KL/chosen_KL_mean": -197.91896057128906,
      "KL/mean": -242.6910400390625,
      "KL/rejected_KL_mean": -287.4631042480469,
      "KL/std": 172.01504516601562,
      "epoch": 0.9312169312169312,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.54414367675781,
      "fcm_dpo/q_t": 0.4340965747833252,
      "grad_norm": 9.829987525939941,
      "learning_rate": 7.36222939784098e-09,
      "logits/chosen": 0.9627779722213745,
      "logits/rejected": 0.9220662713050842,
      "logps/chosen": -256.4496765136719,
      "logps/ref_chosen": -58.530723571777344,
      "logps/ref_rejected": -75.48025512695312,
      "logps/rejected": -362.943359375,
      "loss": 1.1764,
      "margin_dpo/margin_mean": 89.54414367675781,
      "margin_dpo/margin_std": 147.76666259765625,
      "step": 616
    },
    {
      "KL/chosen_KL_mean": -207.77853393554688,
      "KL/mean": -246.21920776367188,
      "KL/rejected_KL_mean": -284.65985107421875,
      "KL/std": 143.22564697265625,
      "epoch": 0.9327286470143613,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.88133239746094,
      "fcm_dpo/q_t": 0.4445750117301941,
      "grad_norm": 13.23380184173584,
      "learning_rate": 7.047107919114586e-09,
      "logits/chosen": 0.9405636787414551,
      "logits/rejected": 0.907119631767273,
      "logps/chosen": -265.38720703125,
      "logps/ref_chosen": -57.608673095703125,
      "logps/ref_rejected": -81.22109985351562,
      "logps/rejected": -365.8809814453125,
      "loss": 1.2199,
      "margin_dpo/margin_mean": 76.88133239746094,
      "margin_dpo/margin_std": 153.86878967285156,
      "step": 617
    },
    {
      "KL/chosen_KL_mean": -194.5389862060547,
      "KL/mean": -245.77511596679688,
      "KL/rejected_KL_mean": -297.0112609863281,
      "KL/std": 160.71217346191406,
      "epoch": 0.9342403628117913,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 102.47227478027344,
      "fcm_dpo/q_t": 0.4275450110435486,
      "grad_norm": 13.20183277130127,
      "learning_rate": 6.738782355044048e-09,
      "logits/chosen": 0.9345172643661499,
      "logits/rejected": 0.8785445690155029,
      "logps/chosen": -251.23492431640625,
      "logps/ref_chosen": -56.69594192504883,
      "logps/ref_rejected": -85.92362976074219,
      "logps/rejected": -382.93487548828125,
      "loss": 1.1553,
      "margin_dpo/margin_mean": 102.47227478027344,
      "margin_dpo/margin_std": 170.03724670410156,
      "step": 618
    },
    {
      "KL/chosen_KL_mean": -184.23806762695312,
      "KL/mean": -236.51092529296875,
      "KL/rejected_KL_mean": -288.78375244140625,
      "KL/std": 148.20510864257812,
      "epoch": 0.9357520786092215,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 104.54570007324219,
      "fcm_dpo/q_t": 0.42483896017074585,
      "grad_norm": 9.666030883789062,
      "learning_rate": 6.437261330158206e-09,
      "logits/chosen": 0.9987913966178894,
      "logits/rejected": 0.9459144473075867,
      "logps/chosen": -238.29649353027344,
      "logps/ref_chosen": -54.05841827392578,
      "logps/ref_rejected": -83.55493927001953,
      "logps/rejected": -372.3387145996094,
      "loss": 1.15,
      "margin_dpo/margin_mean": 104.54570007324219,
      "margin_dpo/margin_std": 168.18020629882812,
      "step": 619
    },
    {
      "KL/chosen_KL_mean": -193.63037109375,
      "KL/mean": -230.4679718017578,
      "KL/rejected_KL_mean": -267.3055725097656,
      "KL/std": 135.5782470703125,
      "epoch": 0.9372637944066515,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 73.67523193359375,
      "fcm_dpo/q_t": 0.44435736536979675,
      "grad_norm": 10.929662704467773,
      "learning_rate": 6.142553278648238e-09,
      "logits/chosen": 0.9473289251327515,
      "logits/rejected": 0.9542825818061829,
      "logps/chosen": -257.0000915527344,
      "logps/ref_chosen": -63.36971664428711,
      "logps/ref_rejected": -65.68269348144531,
      "logps/rejected": -332.98828125,
      "loss": 1.2239,
      "margin_dpo/margin_mean": 73.67523193359375,
      "margin_dpo/margin_std": 142.8062744140625,
      "step": 620
    },
    {
      "KL/chosen_KL_mean": -185.4219970703125,
      "KL/mean": -238.8184356689453,
      "KL/rejected_KL_mean": -292.2148742675781,
      "KL/std": 160.7276611328125,
      "epoch": 0.9387755102040817,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 106.79286193847656,
      "fcm_dpo/q_t": 0.4272248148918152,
      "grad_norm": 9.562493324279785,
      "learning_rate": 5.854666444131934e-09,
      "logits/chosen": 0.9666614532470703,
      "logits/rejected": 0.8935528993606567,
      "logps/chosen": -237.74322509765625,
      "logps/ref_chosen": -52.321224212646484,
      "logps/ref_rejected": -88.09001159667969,
      "logps/rejected": -380.30487060546875,
      "loss": 1.1668,
      "margin_dpo/margin_mean": 106.79286193847656,
      "margin_dpo/margin_std": 194.57687377929688,
      "step": 621
    },
    {
      "KL/chosen_KL_mean": -205.28720092773438,
      "KL/mean": -252.37612915039062,
      "KL/rejected_KL_mean": -299.46502685546875,
      "KL/std": 157.73703002929688,
      "epoch": 0.9402872260015117,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.17784118652344,
      "fcm_dpo/q_t": 0.4326885938644409,
      "grad_norm": 10.958165168762207,
      "learning_rate": 5.573608879422875e-09,
      "logits/chosen": 0.9199116230010986,
      "logits/rejected": 0.8872038722038269,
      "logps/chosen": -265.15264892578125,
      "logps/ref_chosen": -59.86545944213867,
      "logps/ref_rejected": -81.86668395996094,
      "logps/rejected": -381.33172607421875,
      "loss": 1.1775,
      "margin_dpo/margin_mean": 94.17784118652344,
      "margin_dpo/margin_std": 165.80418395996094,
      "step": 622
    },
    {
      "KL/chosen_KL_mean": -222.81295776367188,
      "KL/mean": -274.02532958984375,
      "KL/rejected_KL_mean": -325.23773193359375,
      "KL/std": 174.5893096923828,
      "epoch": 0.9417989417989417,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 102.42472076416016,
      "fcm_dpo/q_t": 0.4269852638244629,
      "grad_norm": 11.868815422058105,
      "learning_rate": 5.299388446305342e-09,
      "logits/chosen": 0.9369495511054993,
      "logits/rejected": 0.9090813994407654,
      "logps/chosen": -290.1814270019531,
      "logps/ref_chosen": -67.36846160888672,
      "logps/ref_rejected": -82.02733612060547,
      "logps/rejected": -407.2650451660156,
      "loss": 1.1739,
      "margin_dpo/margin_mean": 102.42472076416016,
      "margin_dpo/margin_std": 189.164794921875,
      "step": 623
    },
    {
      "KL/chosen_KL_mean": -185.4460906982422,
      "KL/mean": -235.8998260498047,
      "KL/rejected_KL_mean": -286.35357666015625,
      "KL/std": 150.03134155273438,
      "epoch": 0.9433106575963719,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 100.90745544433594,
      "fcm_dpo/q_t": 0.42712944746017456,
      "grad_norm": 11.591996192932129,
      "learning_rate": 5.03201281531429e-09,
      "logits/chosen": 0.9607518911361694,
      "logits/rejected": 0.8958628177642822,
      "logps/chosen": -236.47264099121094,
      "logps/ref_chosen": -51.02655029296875,
      "logps/ref_rejected": -76.49203491210938,
      "logps/rejected": -362.8455810546875,
      "loss": 1.1536,
      "margin_dpo/margin_mean": 100.90745544433594,
      "margin_dpo/margin_std": 158.26055908203125,
      "step": 624
    },
    {
      "KL/chosen_KL_mean": -186.66845703125,
      "KL/mean": -226.89675903320312,
      "KL/rejected_KL_mean": -267.12506103515625,
      "KL/std": 141.98199462890625,
      "epoch": 0.9448223733938019,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.45657348632812,
      "fcm_dpo/q_t": 0.4423113167285919,
      "grad_norm": 10.402351379394531,
      "learning_rate": 4.7714894655209174e-09,
      "logits/chosen": 0.9809439778327942,
      "logits/rejected": 0.9170085191726685,
      "logps/chosen": -240.87608337402344,
      "logps/ref_chosen": -54.20761489868164,
      "logps/ref_rejected": -84.93669128417969,
      "logps/rejected": -352.061767578125,
      "loss": 1.2162,
      "margin_dpo/margin_mean": 80.45657348632812,
      "margin_dpo/margin_std": 167.8040771484375,
      "step": 625
    },
    {
      "KL/chosen_KL_mean": -173.0820770263672,
      "KL/mean": -229.86431884765625,
      "KL/rejected_KL_mean": -286.6465759277344,
      "KL/std": 159.24993896484375,
      "epoch": 0.9463340891912321,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 113.56451416015625,
      "fcm_dpo/q_t": 0.42139124870300293,
      "grad_norm": 10.874939918518066,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 1.0064220428466797,
      "logits/rejected": 0.9120746850967407,
      "logps/chosen": -218.14410400390625,
      "logps/ref_chosen": -45.06201934814453,
      "logps/ref_rejected": -89.66368103027344,
      "logps/rejected": -376.31024169921875,
      "loss": 1.1387,
      "margin_dpo/margin_mean": 113.56451416015625,
      "margin_dpo/margin_std": 190.21609497070312,
      "step": 626
    },
    {
      "KL/chosen_KL_mean": -203.1885986328125,
      "KL/mean": -264.7366943359375,
      "KL/rejected_KL_mean": -326.2847900390625,
      "KL/std": 168.15475463867188,
      "epoch": 0.9478458049886621,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 123.09614562988281,
      "fcm_dpo/q_t": 0.41006141901016235,
      "grad_norm": 11.144246101379395,
      "learning_rate": 4.271028567242818e-09,
      "logits/chosen": 0.8766025304794312,
      "logits/rejected": 0.7959402799606323,
      "logps/chosen": -261.97967529296875,
      "logps/ref_chosen": -58.791053771972656,
      "logps/ref_rejected": -94.90802001953125,
      "logps/rejected": -421.19281005859375,
      "loss": 1.0933,
      "margin_dpo/margin_mean": 123.09615325927734,
      "margin_dpo/margin_std": 155.38308715820312,
      "step": 627
    },
    {
      "KL/chosen_KL_mean": -191.96481323242188,
      "KL/mean": -244.04881286621094,
      "KL/rejected_KL_mean": -296.1328125,
      "KL/std": 150.7305908203125,
      "epoch": 0.9493575207860923,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 104.16798400878906,
      "fcm_dpo/q_t": 0.4224216043949127,
      "grad_norm": 10.786967277526855,
      "learning_rate": 4.0311050177251895e-09,
      "logits/chosen": 0.9671989679336548,
      "logits/rejected": 0.9418696761131287,
      "logps/chosen": -244.7683868408203,
      "logps/ref_chosen": -52.80357360839844,
      "logps/ref_rejected": -76.49468994140625,
      "logps/rejected": -372.62750244140625,
      "loss": 1.1657,
      "margin_dpo/margin_mean": 104.16798400878906,
      "margin_dpo/margin_std": 174.9075164794922,
      "step": 628
    },
    {
      "KL/chosen_KL_mean": -203.937744140625,
      "KL/mean": -241.90890502929688,
      "KL/rejected_KL_mean": -279.88006591796875,
      "KL/std": 139.17115783691406,
      "epoch": 0.9508692365835223,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 75.94229125976562,
      "fcm_dpo/q_t": 0.4455203413963318,
      "grad_norm": 10.165343284606934,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": 0.9274606704711914,
      "logits/rejected": 0.9357987642288208,
      "logps/chosen": -274.6552429199219,
      "logps/ref_chosen": -70.71749877929688,
      "logps/ref_rejected": -78.96273803710938,
      "logps/rejected": -358.8428039550781,
      "loss": 1.2155,
      "margin_dpo/margin_mean": 75.94229125976562,
      "margin_dpo/margin_std": 150.8992919921875,
      "step": 629
    },
    {
      "KL/chosen_KL_mean": -183.7464599609375,
      "KL/mean": -239.07704162597656,
      "KL/rejected_KL_mean": -294.4075927734375,
      "KL/std": 145.11636352539062,
      "epoch": 0.9523809523809523,
      "fcm_dpo/beta": 0.003130414756014943,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 110.66114807128906,
      "fcm_dpo/q_t": 0.41840821504592896,
      "grad_norm": 10.500893592834473,
      "learning_rate": 3.5719052736323806e-09,
      "logits/chosen": 0.9014256000518799,
      "logits/rejected": 0.8830850124359131,
      "logps/chosen": -239.9478759765625,
      "logps/ref_chosen": -56.201412200927734,
      "logps/ref_rejected": -74.69807434082031,
      "logps/rejected": -369.1056823730469,
      "loss": 1.1297,
      "margin_dpo/margin_mean": 110.66114807128906,
      "margin_dpo/margin_std": 160.73321533203125,
      "step": 630
    },
    {
      "KL/chosen_KL_mean": -180.64476013183594,
      "KL/mean": -246.01486206054688,
      "KL/rejected_KL_mean": -311.38494873046875,
      "KL/std": 162.5005340576172,
      "epoch": 0.9538926681783825,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": -0.05842360481619835,
      "fcm_dpo/margin": 130.7401885986328,
      "fcm_dpo/q_t": 0.40720421075820923,
      "grad_norm": 9.99383544921875,
      "learning_rate": 3.352641923861144e-09,
      "logits/chosen": 1.0124080181121826,
      "logits/rejected": 0.9354528784751892,
      "logps/chosen": -239.46536254882812,
      "logps/ref_chosen": -58.82059860229492,
      "logps/ref_rejected": -96.51437377929688,
      "logps/rejected": -407.8993225097656,
      "loss": 1.0968,
      "margin_dpo/margin_mean": 130.7401885986328,
      "margin_dpo/margin_std": 179.59544372558594,
      "step": 631
    },
    {
      "KL/chosen_KL_mean": -180.45394897460938,
      "KL/mean": -236.18548583984375,
      "KL/rejected_KL_mean": -291.9169921875,
      "KL/std": 135.77951049804688,
      "epoch": 0.9554043839758125,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 111.46307373046875,
      "fcm_dpo/q_t": 0.41834402084350586,
      "grad_norm": 10.386039733886719,
      "learning_rate": 3.140277830901428e-09,
      "logits/chosen": 0.9802180528640747,
      "logits/rejected": 0.9753730297088623,
      "logps/chosen": -239.239990234375,
      "logps/ref_chosen": -58.786048889160156,
      "logps/ref_rejected": -67.21923828125,
      "logps/rejected": -359.1362609863281,
      "loss": 1.1238,
      "margin_dpo/margin_mean": 111.46307373046875,
      "margin_dpo/margin_std": 151.68695068359375,
      "step": 632
    },
    {
      "KL/chosen_KL_mean": -175.48123168945312,
      "KL/mean": -222.42837524414062,
      "KL/rejected_KL_mean": -269.3755187988281,
      "KL/std": 135.81710815429688,
      "epoch": 0.9569160997732427,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 93.894287109375,
      "fcm_dpo/q_t": 0.43249619007110596,
      "grad_norm": 12.328335762023926,
      "learning_rate": 2.9348189350335007e-09,
      "logits/chosen": 0.9596331119537354,
      "logits/rejected": 0.9312918782234192,
      "logps/chosen": -227.61141967773438,
      "logps/ref_chosen": -52.13019561767578,
      "logps/ref_rejected": -67.23016357421875,
      "logps/rejected": -336.6056823730469,
      "loss": 1.1702,
      "margin_dpo/margin_mean": 93.894287109375,
      "margin_dpo/margin_std": 150.7270050048828,
      "step": 633
    },
    {
      "KL/chosen_KL_mean": -211.42013549804688,
      "KL/mean": -227.02139282226562,
      "KL/rejected_KL_mean": -242.6226806640625,
      "KL/std": 144.92550659179688,
      "epoch": 0.9584278155706727,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 31.20253562927246,
      "fcm_dpo/q_t": 0.47805124521255493,
      "grad_norm": 11.825292587280273,
      "learning_rate": 2.736270983384276e-09,
      "logits/chosen": 0.9753604531288147,
      "logits/rejected": 0.9930594563484192,
      "logps/chosen": -272.3999328613281,
      "logps/ref_chosen": -60.97979736328125,
      "logps/ref_rejected": -58.50825119018555,
      "logps/rejected": -301.13092041015625,
      "loss": 1.3544,
      "margin_dpo/margin_mean": 31.20253562927246,
      "margin_dpo/margin_std": 162.56167602539062,
      "step": 634
    },
    {
      "KL/chosen_KL_mean": -230.13027954101562,
      "KL/mean": -270.3157958984375,
      "KL/rejected_KL_mean": -310.5013122558594,
      "KL/std": 133.14523315429688,
      "epoch": 0.9599395313681028,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.37104797363281,
      "fcm_dpo/q_t": 0.44142985343933105,
      "grad_norm": 10.514294624328613,
      "learning_rate": 2.5446395297668287e-09,
      "logits/chosen": 0.8804504871368408,
      "logits/rejected": 0.8512506484985352,
      "logps/chosen": -296.103271484375,
      "logps/ref_chosen": -65.9730224609375,
      "logps/ref_rejected": -85.61317443847656,
      "logps/rejected": -396.114501953125,
      "loss": 1.2082,
      "margin_dpo/margin_mean": 80.37104797363281,
      "margin_dpo/margin_std": 152.8987274169922,
      "step": 635
    },
    {
      "KL/chosen_KL_mean": -175.61082458496094,
      "KL/mean": -228.16116333007812,
      "KL/rejected_KL_mean": -280.71148681640625,
      "KL/std": 136.18251037597656,
      "epoch": 0.9614512471655329,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 105.10067749023438,
      "fcm_dpo/q_t": 0.4229128658771515,
      "grad_norm": 8.548711776733398,
      "learning_rate": 2.359929934524829e-09,
      "logits/chosen": 0.9548631310462952,
      "logits/rejected": 0.8846620321273804,
      "logps/chosen": -224.75099182128906,
      "logps/ref_chosen": -49.140167236328125,
      "logps/ref_rejected": -81.26971435546875,
      "logps/rejected": -361.981201171875,
      "loss": 1.134,
      "margin_dpo/margin_mean": 105.10067749023438,
      "margin_dpo/margin_std": 142.31265258789062,
      "step": 636
    },
    {
      "KL/chosen_KL_mean": -241.5238494873047,
      "KL/mean": -282.18572998046875,
      "KL/rejected_KL_mean": -322.84759521484375,
      "KL/std": 149.25390625,
      "epoch": 0.9629629629629629,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.32373809814453,
      "fcm_dpo/q_t": 0.4420722424983978,
      "grad_norm": 9.542378425598145,
      "learning_rate": 2.1821473643827137e-09,
      "logits/chosen": 0.9008264541625977,
      "logits/rejected": 0.8717888593673706,
      "logps/chosen": -315.2204284667969,
      "logps/ref_chosen": -73.69658660888672,
      "logps/ref_rejected": -83.01487731933594,
      "logps/rejected": -405.86248779296875,
      "loss": 1.2203,
      "margin_dpo/margin_mean": 81.32373809814453,
      "margin_dpo/margin_std": 176.375732421875,
      "step": 637
    },
    {
      "KL/chosen_KL_mean": -208.33538818359375,
      "KL/mean": -256.4822082519531,
      "KL/rejected_KL_mean": -304.6290283203125,
      "KL/std": 140.94253540039062,
      "epoch": 0.9644746787603931,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 96.29367065429688,
      "fcm_dpo/q_t": 0.4311360716819763,
      "grad_norm": 11.053569793701172,
      "learning_rate": 2.0112967923011646e-09,
      "logits/chosen": 0.9434064626693726,
      "logits/rejected": 0.9192993640899658,
      "logps/chosen": -271.1169738769531,
      "logps/ref_chosen": -62.78158187866211,
      "logps/ref_rejected": -85.40478515625,
      "logps/rejected": -390.0338134765625,
      "loss": 1.172,
      "margin_dpo/margin_mean": 96.29366302490234,
      "margin_dpo/margin_std": 164.99270629882812,
      "step": 638
    },
    {
      "KL/chosen_KL_mean": -187.1907958984375,
      "KL/mean": -242.9512176513672,
      "KL/rejected_KL_mean": -298.71160888671875,
      "KL/std": 155.54104614257812,
      "epoch": 0.9659863945578231,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 111.52082824707031,
      "fcm_dpo/q_t": 0.42117875814437866,
      "grad_norm": 10.140286445617676,
      "learning_rate": 1.847382997337943e-09,
      "logits/chosen": 0.9174301624298096,
      "logits/rejected": 0.8663163185119629,
      "logps/chosen": -240.95738220214844,
      "logps/ref_chosen": -53.76658630371094,
      "logps/ref_rejected": -72.30009460449219,
      "logps/rejected": -371.01171875,
      "loss": 1.139,
      "margin_dpo/margin_mean": 111.52082824707031,
      "margin_dpo/margin_std": 174.923828125,
      "step": 639
    },
    {
      "KL/chosen_KL_mean": -193.22474670410156,
      "KL/mean": -241.06468200683594,
      "KL/rejected_KL_mean": -288.9046325683594,
      "KL/std": 144.21368408203125,
      "epoch": 0.9674981103552532,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 95.67987823486328,
      "fcm_dpo/q_t": 0.43061840534210205,
      "grad_norm": 10.762125015258789,
      "learning_rate": 1.690410564514244e-09,
      "logits/chosen": 1.0091588497161865,
      "logits/rejected": 0.966665506362915,
      "logps/chosen": -244.64251708984375,
      "logps/ref_chosen": -51.41777801513672,
      "logps/ref_rejected": -77.27879333496094,
      "logps/rejected": -366.18341064453125,
      "loss": 1.1673,
      "margin_dpo/margin_mean": 95.67987823486328,
      "margin_dpo/margin_std": 154.81198120117188,
      "step": 640
    },
    {
      "KL/chosen_KL_mean": -208.03912353515625,
      "KL/mean": -248.72650146484375,
      "KL/rejected_KL_mean": -289.413818359375,
      "KL/std": 132.9041290283203,
      "epoch": 0.9690098261526833,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 81.37470245361328,
      "fcm_dpo/q_t": 0.4405854344367981,
      "grad_norm": 11.396415710449219,
      "learning_rate": 1.5403838846864692e-09,
      "logits/chosen": 0.9047503471374512,
      "logits/rejected": 0.9040436148643494,
      "logps/chosen": -279.09381103515625,
      "logps/ref_chosen": -71.0546646118164,
      "logps/ref_rejected": -82.2440185546875,
      "logps/rejected": -371.6578369140625,
      "loss": 1.1894,
      "margin_dpo/margin_mean": 81.37470245361328,
      "margin_dpo/margin_std": 129.62969970703125,
      "step": 641
    },
    {
      "KL/chosen_KL_mean": -216.7130584716797,
      "KL/mean": -246.51898193359375,
      "KL/rejected_KL_mean": -276.32489013671875,
      "KL/std": 134.2535400390625,
      "epoch": 0.9705215419501134,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 59.61183166503906,
      "fcm_dpo/q_t": 0.4580966532230377,
      "grad_norm": 12.750149726867676,
      "learning_rate": 1.3973071544233218e-09,
      "logits/chosen": 0.8720563650131226,
      "logits/rejected": 0.8946826457977295,
      "logps/chosen": -285.642333984375,
      "logps/ref_chosen": -68.92927551269531,
      "logps/ref_rejected": -70.85682678222656,
      "logps/rejected": -347.18170166015625,
      "loss": 1.2688,
      "margin_dpo/margin_mean": 59.61183166503906,
      "margin_dpo/margin_std": 158.3714141845703,
      "step": 642
    },
    {
      "KL/chosen_KL_mean": -195.25668334960938,
      "KL/mean": -249.47900390625,
      "KL/rejected_KL_mean": -303.7013244628906,
      "KL/std": 150.85220336914062,
      "epoch": 0.9720332577475435,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 108.44462585449219,
      "fcm_dpo/q_t": 0.42308878898620605,
      "grad_norm": 13.6078462600708,
      "learning_rate": 1.261184375888541e-09,
      "logits/chosen": 0.9121971130371094,
      "logits/rejected": 0.8631089329719543,
      "logps/chosen": -260.56573486328125,
      "logps/ref_chosen": -65.30903625488281,
      "logps/ref_rejected": -83.61613464355469,
      "logps/rejected": -387.31744384765625,
      "loss": 1.1468,
      "margin_dpo/margin_mean": 108.44462585449219,
      "margin_dpo/margin_std": 171.8424072265625,
      "step": 643
    },
    {
      "KL/chosen_KL_mean": -183.8955078125,
      "KL/mean": -214.19424438476562,
      "KL/rejected_KL_mean": -244.49301147460938,
      "KL/std": 119.51637268066406,
      "epoch": 0.9735449735449735,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 60.597511291503906,
      "fcm_dpo/q_t": 0.45479634404182434,
      "grad_norm": 8.376975059509277,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 1.0118638277053833,
      "logits/rejected": 0.9890410900115967,
      "logps/chosen": -234.89810180664062,
      "logps/ref_chosen": -51.002601623535156,
      "logps/ref_rejected": -64.46372985839844,
      "logps/rejected": -308.95672607421875,
      "loss": 1.2544,
      "margin_dpo/margin_mean": 60.597511291503906,
      "margin_dpo/margin_std": 138.36883544921875,
      "step": 644
    },
    {
      "KL/chosen_KL_mean": -204.33132934570312,
      "KL/mean": -242.13632202148438,
      "KL/rejected_KL_mean": -279.94134521484375,
      "KL/std": 136.10458374023438,
      "epoch": 0.9750566893424036,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 75.61000061035156,
      "fcm_dpo/q_t": 0.4438573122024536,
      "grad_norm": 15.0194091796875,
      "learning_rate": 1.0098157099674987e-09,
      "logits/chosen": 0.9073389768600464,
      "logits/rejected": 0.8987625241279602,
      "logps/chosen": -265.29473876953125,
      "logps/ref_chosen": -60.963409423828125,
      "logps/ref_rejected": -69.73353576660156,
      "logps/rejected": -349.67486572265625,
      "loss": 1.2134,
      "margin_dpo/margin_mean": 75.61000061035156,
      "margin_dpo/margin_std": 142.1605682373047,
      "step": 645
    },
    {
      "KL/chosen_KL_mean": -222.45260620117188,
      "KL/mean": -262.6767578125,
      "KL/rejected_KL_mean": -302.90093994140625,
      "KL/std": 172.28286743164062,
      "epoch": 0.9765684051398337,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 80.44831848144531,
      "fcm_dpo/q_t": 0.4418299198150635,
      "grad_norm": 9.49789810180664,
      "learning_rate": 8.945768539031783e-10,
      "logits/chosen": 0.9376930594444275,
      "logits/rejected": 0.8963108062744141,
      "logps/chosen": -284.74267578125,
      "logps/ref_chosen": -62.290069580078125,
      "logps/ref_rejected": -85.54812622070312,
      "logps/rejected": -388.44903564453125,
      "loss": 1.2106,
      "margin_dpo/margin_mean": 80.44831848144531,
      "margin_dpo/margin_std": 158.8577423095703,
      "step": 646
    },
    {
      "KL/chosen_KL_mean": -219.18106079101562,
      "KL/mean": -282.8184509277344,
      "KL/rejected_KL_mean": -346.455810546875,
      "KL/std": 167.93240356445312,
      "epoch": 0.9780801209372638,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 127.27474975585938,
      "fcm_dpo/q_t": 0.40855348110198975,
      "grad_norm": 10.709985733032227,
      "learning_rate": 7.863060120144316e-10,
      "logits/chosen": 0.9660693407058716,
      "logits/rejected": 0.8981690406799316,
      "logps/chosen": -286.69696044921875,
      "logps/ref_chosen": -67.515869140625,
      "logps/ref_rejected": -101.50871276855469,
      "logps/rejected": -447.96453857421875,
      "loss": 1.0954,
      "margin_dpo/margin_mean": 127.27474975585938,
      "margin_dpo/margin_std": 169.9162139892578,
      "step": 647
    },
    {
      "KL/chosen_KL_mean": -219.24618530273438,
      "KL/mean": -263.4234924316406,
      "KL/rejected_KL_mean": -307.60076904296875,
      "KL/std": 140.67422485351562,
      "epoch": 0.9795918367346939,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.35462188720703,
      "fcm_dpo/q_t": 0.437153160572052,
      "grad_norm": 9.533730506896973,
      "learning_rate": 6.850062128694045e-10,
      "logits/chosen": 0.9046711921691895,
      "logits/rejected": 0.8672827482223511,
      "logps/chosen": -283.84210205078125,
      "logps/ref_chosen": -64.59593963623047,
      "logps/ref_rejected": -83.384033203125,
      "logps/rejected": -390.98480224609375,
      "loss": 1.2142,
      "margin_dpo/margin_mean": 88.35462188720703,
      "margin_dpo/margin_std": 183.80010986328125,
      "step": 648
    },
    {
      "KL/chosen_KL_mean": -195.35403442382812,
      "KL/mean": -238.9105682373047,
      "KL/rejected_KL_mean": -282.46710205078125,
      "KL/std": 131.21609497070312,
      "epoch": 0.981103552532124,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 87.11306762695312,
      "fcm_dpo/q_t": 0.43540871143341064,
      "grad_norm": 13.80373764038086,
      "learning_rate": 5.906802900412788e-10,
      "logits/chosen": 0.9699212312698364,
      "logits/rejected": 0.9264802932739258,
      "logps/chosen": -244.66368103027344,
      "logps/ref_chosen": -49.30964660644531,
      "logps/ref_rejected": -73.73710632324219,
      "logps/rejected": -356.2042236328125,
      "loss": 1.1903,
      "margin_dpo/margin_mean": 87.11306762695312,
      "margin_dpo/margin_std": 155.24761962890625,
      "step": 649
    },
    {
      "KL/chosen_KL_mean": -189.0690460205078,
      "KL/mean": -237.6030731201172,
      "KL/rejected_KL_mean": -286.1370849609375,
      "KL/std": 138.7324676513672,
      "epoch": 0.982615268329554,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 97.06804656982422,
      "fcm_dpo/q_t": 0.4303385615348816,
      "grad_norm": 9.39942741394043,
      "learning_rate": 5.033308820289184e-10,
      "logits/chosen": 0.9848513603210449,
      "logits/rejected": 0.9468841552734375,
      "logps/chosen": -244.13230895996094,
      "logps/ref_chosen": -55.06325912475586,
      "logps/ref_rejected": -77.39610290527344,
      "logps/rejected": -363.533203125,
      "loss": 1.1745,
      "margin_dpo/margin_mean": 97.06805419921875,
      "margin_dpo/margin_std": 170.67352294921875,
      "step": 650
    },
    {
      "KL/chosen_KL_mean": -187.72344970703125,
      "KL/mean": -232.03823852539062,
      "KL/rejected_KL_mean": -276.35302734375,
      "KL/std": 137.58258056640625,
      "epoch": 0.9841269841269841,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 88.62960815429688,
      "fcm_dpo/q_t": 0.43634384870529175,
      "grad_norm": 11.805054664611816,
      "learning_rate": 4.2296043218295606e-10,
      "logits/chosen": 0.9546950459480286,
      "logits/rejected": 0.9093322157859802,
      "logps/chosen": -241.78860473632812,
      "logps/ref_chosen": -54.065162658691406,
      "logps/ref_rejected": -77.79080200195312,
      "logps/rejected": -354.14385986328125,
      "loss": 1.1785,
      "margin_dpo/margin_mean": 88.62960815429688,
      "margin_dpo/margin_std": 143.01470947265625,
      "step": 651
    },
    {
      "KL/chosen_KL_mean": -223.95025634765625,
      "KL/mean": -261.40289306640625,
      "KL/rejected_KL_mean": -298.8555603027344,
      "KL/std": 159.61614990234375,
      "epoch": 0.9856386999244142,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 74.9052963256836,
      "fcm_dpo/q_t": 0.4478684067726135,
      "grad_norm": 11.329047203063965,
      "learning_rate": 3.4957118863768176e-10,
      "logits/chosen": 0.9421413540840149,
      "logits/rejected": 0.9153538942337036,
      "logps/chosen": -287.590576171875,
      "logps/ref_chosen": -63.64030456542969,
      "logps/ref_rejected": -78.86882019042969,
      "logps/rejected": -377.724365234375,
      "loss": 1.2524,
      "margin_dpo/margin_mean": 74.9052963256836,
      "margin_dpo/margin_std": 195.61993408203125,
      "step": 652
    },
    {
      "KL/chosen_KL_mean": -196.9767608642578,
      "KL/mean": -241.781494140625,
      "KL/rejected_KL_mean": -286.58624267578125,
      "KL/std": 133.9491424560547,
      "epoch": 0.9871504157218443,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 89.60951232910156,
      "fcm_dpo/q_t": 0.4346145987510681,
      "grad_norm": 13.65695858001709,
      "learning_rate": 2.831652042480093e-10,
      "logits/chosen": 0.9071061611175537,
      "logits/rejected": 0.8793363571166992,
      "logps/chosen": -258.6451416015625,
      "logps/ref_chosen": -61.668373107910156,
      "logps/ref_rejected": -73.83012390136719,
      "logps/rejected": -360.4163818359375,
      "loss": 1.1819,
      "margin_dpo/margin_mean": 89.60951232910156,
      "margin_dpo/margin_std": 151.9982147216797,
      "step": 653
    },
    {
      "KL/chosen_KL_mean": -205.64804077148438,
      "KL/mean": -253.06338500976562,
      "KL/rejected_KL_mean": -300.47869873046875,
      "KL/std": 164.1302490234375,
      "epoch": 0.9886621315192744,
      "fcm_dpo/beta": 0.003094049636274576,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 94.83065795898438,
      "fcm_dpo/q_t": 0.4318302869796753,
      "grad_norm": 12.566394805908203,
      "learning_rate": 2.2374433653205016e-10,
      "logits/chosen": 0.914149284362793,
      "logits/rejected": 0.8528527021408081,
      "logps/chosen": -263.21630859375,
      "logps/ref_chosen": -57.568267822265625,
      "logps/ref_rejected": -87.74789428710938,
      "logps/rejected": -388.2265930175781,
      "loss": 1.1715,
      "margin_dpo/margin_mean": 94.83065795898438,
      "margin_dpo/margin_std": 154.06057739257812,
      "step": 654
    },
    {
      "KL/chosen_KL_mean": -161.66128540039062,
      "KL/mean": -227.72610473632812,
      "KL/rejected_KL_mean": -293.7909240722656,
      "KL/std": 143.35321044921875,
      "epoch": 0.9901738473167044,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": -0.08228477090597153,
      "fcm_dpo/margin": 132.12965393066406,
      "fcm_dpo/q_t": 0.4063999652862549,
      "grad_norm": 10.75922966003418,
      "learning_rate": 1.7131024761923852e-10,
      "logits/chosen": 0.9187965393066406,
      "logits/rejected": 0.8673887848854065,
      "logps/chosen": -213.8084259033203,
      "logps/ref_chosen": -52.14714813232422,
      "logps/ref_rejected": -80.85014343261719,
      "logps/rejected": -374.64105224609375,
      "loss": 1.0789,
      "margin_dpo/margin_mean": 132.12965393066406,
      "margin_dpo/margin_std": 152.25323486328125,
      "step": 655
    },
    {
      "KL/chosen_KL_mean": -219.64300537109375,
      "KL/mean": -265.52117919921875,
      "KL/rejected_KL_mean": -311.3993835449219,
      "KL/std": 142.48965454101562,
      "epoch": 0.9916855631141346,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.75637817382812,
      "fcm_dpo/q_t": 0.43464866280555725,
      "grad_norm": 9.470726013183594,
      "learning_rate": 1.2586440420372934e-10,
      "logits/chosen": 0.8957068920135498,
      "logits/rejected": 0.8757596611976624,
      "logps/chosen": -292.89971923828125,
      "logps/ref_chosen": -73.25672912597656,
      "logps/ref_rejected": -85.35127258300781,
      "logps/rejected": -396.75067138671875,
      "loss": 1.1805,
      "margin_dpo/margin_mean": 91.75637817382812,
      "margin_dpo/margin_std": 156.62033081054688,
      "step": 656
    },
    {
      "KL/chosen_KL_mean": -187.28375244140625,
      "KL/mean": -245.9228057861328,
      "KL/rejected_KL_mean": -304.5618591308594,
      "KL/std": 159.7275848388672,
      "epoch": 0.9931972789115646,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 117.27809143066406,
      "fcm_dpo/q_t": 0.41826194524765015,
      "grad_norm": 9.124080657958984,
      "learning_rate": 8.740807750345913e-11,
      "logits/chosen": 1.0158004760742188,
      "logits/rejected": 0.9686501026153564,
      "logps/chosen": -237.0071563720703,
      "logps/ref_chosen": -49.72339630126953,
      "logps/ref_rejected": -75.1568603515625,
      "logps/rejected": -379.7187194824219,
      "loss": 1.13,
      "margin_dpo/margin_mean": 117.27810668945312,
      "margin_dpo/margin_std": 178.3359375,
      "step": 657
    },
    {
      "KL/chosen_KL_mean": -205.32443237304688,
      "KL/mean": -251.24359130859375,
      "KL/rejected_KL_mean": -297.16278076171875,
      "KL/std": 170.65353393554688,
      "epoch": 0.9947089947089947,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 91.83834838867188,
      "fcm_dpo/q_t": 0.4347524642944336,
      "grad_norm": 8.767982482910156,
      "learning_rate": 5.594234322453539e-11,
      "logits/chosen": 0.953624963760376,
      "logits/rejected": 0.9262589812278748,
      "logps/chosen": -268.37078857421875,
      "logps/ref_chosen": -63.04634094238281,
      "logps/ref_rejected": -83.44963073730469,
      "logps/rejected": -380.6123962402344,
      "loss": 1.21,
      "margin_dpo/margin_mean": 91.83834075927734,
      "margin_dpo/margin_std": 194.5883331298828,
      "step": 658
    },
    {
      "KL/chosen_KL_mean": -207.29444885253906,
      "KL/mean": -241.6499481201172,
      "KL/rejected_KL_mean": -276.00543212890625,
      "KL/std": 147.86619567871094,
      "epoch": 0.9962207105064248,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 68.71096801757812,
      "fcm_dpo/q_t": 0.4509459435939789,
      "grad_norm": 11.185192108154297,
      "learning_rate": 3.146808153123293e-11,
      "logits/chosen": 0.9820291996002197,
      "logits/rejected": 0.9410825967788696,
      "logps/chosen": -262.3746337890625,
      "logps/ref_chosen": -55.0802001953125,
      "logps/ref_rejected": -71.91049194335938,
      "logps/rejected": -347.9159240722656,
      "loss": 1.2532,
      "margin_dpo/margin_mean": 68.71096801757812,
      "margin_dpo/margin_std": 169.84005737304688,
      "step": 659
    },
    {
      "KL/chosen_KL_mean": -190.37002563476562,
      "KL/mean": -251.46902465820312,
      "KL/rejected_KL_mean": -312.5680236816406,
      "KL/std": 157.89157104492188,
      "epoch": 0.9977324263038548,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 122.19801330566406,
      "fcm_dpo/q_t": 0.4155905544757843,
      "grad_norm": 11.91143798828125,
      "learning_rate": 1.3985977021235829e-11,
      "logits/chosen": 1.0178961753845215,
      "logits/rejected": 0.983450174331665,
      "logps/chosen": -244.89593505859375,
      "logps/ref_chosen": -54.525917053222656,
      "logps/ref_rejected": -81.23604583740234,
      "logps/rejected": -393.8040771484375,
      "loss": 1.1127,
      "margin_dpo/margin_mean": 122.19801330566406,
      "margin_dpo/margin_std": 177.11691284179688,
      "step": 660
    },
    {
      "KL/chosen_KL_mean": -225.02529907226562,
      "KL/mean": -263.05596923828125,
      "KL/rejected_KL_mean": -301.086669921875,
      "KL/std": 154.0184326171875,
      "epoch": 0.999244142101285,
      "fcm_dpo/beta": 0.0030435477383434772,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 76.06137084960938,
      "fcm_dpo/q_t": 0.4469204545021057,
      "grad_norm": 8.742901802062988,
      "learning_rate": 3.4965187065971735e-12,
      "logits/chosen": 0.9296639561653137,
      "logits/rejected": 0.8836817741394043,
      "logps/chosen": -285.39794921875,
      "logps/ref_chosen": -60.37263870239258,
      "logps/ref_rejected": -77.42874145507812,
      "logps/rejected": -378.51544189453125,
      "loss": 1.2393,
      "margin_dpo/margin_mean": 76.06137084960938,
      "margin_dpo/margin_std": 180.522705078125,
      "step": 661
    },
    {
      "epoch": 0.999244142101285,
      "step": 661,
      "total_flos": 0.0,
      "train_loss": 1.1812428684169693,
      "train_runtime": 1648.6054,
      "train_samples_per_second": 25.68,
      "train_steps_per_second": 0.401
    }
  ],
  "logging_steps": 1,
  "max_steps": 661,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}