{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999244142101285,
  "eval_steps": 200,
  "global_step": 661,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": 0.02867889404296875,
      "KL/mean": 0.029354453086853027,
      "KL/rejected_KL_mean": 0.030029296875,
      "KL/std": 0.2071000635623932,
      "epoch": 0.0015117157974300832,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0013532638549804688,
      "fcm_dpo/q_t": 0.5001497268676758,
      "grad_norm": 141.68185424804688,
      "learning_rate": 0.0,
      "logits/chosen": 0.13337239623069763,
      "logits/rejected": 0.12492949515581131,
      "logps/chosen": -64.5841293334961,
      "logps/ref_chosen": -64.61280822753906,
      "logps/ref_rejected": -64.17195129394531,
      "logps/rejected": -64.14192199707031,
      "loss": 1.3911,
      "margin_dpo/margin_mean": -0.0013527870178222656,
      "margin_dpo/margin_std": 0.2561596930027008,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": -0.00289154052734375,
      "KL/mean": -0.021616414189338684,
      "KL/rejected_KL_mean": -0.04033660888671875,
      "KL/std": 0.19624735414981842,
      "epoch": 0.0030234315948601664,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.037450045347213745,
      "fcm_dpo/q_t": 0.4953404366970062,
      "grad_norm": 138.73599243164062,
      "learning_rate": 7.462686567164179e-09,
      "logits/chosen": 0.09414851665496826,
      "logits/rejected": 0.07363267242908478,
      "logps/chosen": -56.101890563964844,
      "logps/ref_chosen": -56.0989990234375,
      "logps/ref_rejected": -66.59971618652344,
      "logps/rejected": -66.64006042480469,
      "loss": 1.3728,
      "margin_dpo/margin_mean": 0.03744968771934509,
      "margin_dpo/margin_std": 0.27811938524246216,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": 0.0136871337890625,
      "KL/mean": 0.02692541480064392,
      "KL/rejected_KL_mean": 0.040157318115234375,
      "KL/std": 0.2473403811454773,
      "epoch": 0.0045351473922902496,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.026466786861419678,
      "fcm_dpo/q_t": 0.5032904148101807,
      "grad_norm": 160.08132934570312,
      "learning_rate": 1.4925373134328357e-08,
      "logits/chosen": 0.0993448942899704,
      "logits/rejected": 0.06133737042546272,
      "logps/chosen": -65.44357299804688,
      "logps/ref_chosen": -65.45726013183594,
      "logps/ref_rejected": -90.82853698730469,
      "logps/rejected": -90.78837585449219,
      "loss": 1.4055,
      "margin_dpo/margin_mean": -0.026467204093933105,
      "margin_dpo/margin_std": 0.30515891313552856,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.00174713134765625,
      "KL/mean": 0.0021182894706726074,
      "KL/rejected_KL_mean": 0.002483367919921875,
      "KL/std": 0.22779090702533722,
      "epoch": 0.006046863189720333,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0007355809211730957,
      "fcm_dpo/q_t": 0.5001123547554016,
      "grad_norm": 174.59449768066406,
      "learning_rate": 2.2388059701492534e-08,
      "logits/chosen": 0.10049319267272949,
      "logits/rejected": 0.08455335348844528,
      "logps/chosen": -76.85843658447266,
      "logps/ref_chosen": -76.86018371582031,
      "logps/ref_rejected": -79.91523742675781,
      "logps/rejected": -79.91275024414062,
      "loss": 1.3932,
      "margin_dpo/margin_mean": -0.000735849142074585,
      "margin_dpo/margin_std": 0.32438385486602783,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": 0.0012531280517578125,
      "KL/mean": 0.028915926814079285,
      "KL/rejected_KL_mean": 0.056583404541015625,
      "KL/std": 0.22457917034626007,
      "epoch": 0.007558578987150416,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.05532631278038025,
      "fcm_dpo/q_t": 0.506885290145874,
      "grad_norm": 153.40650939941406,
      "learning_rate": 2.9850746268656714e-08,
      "logits/chosen": 0.07975707203149796,
      "logits/rejected": 0.040973931550979614,
      "logps/chosen": -62.97008514404297,
      "logps/ref_chosen": -62.97134017944336,
      "logps/ref_rejected": -79.9192123413086,
      "logps/rejected": -79.86262512207031,
      "loss": 1.4208,
      "margin_dpo/margin_mean": -0.05532556772232056,
      "margin_dpo/margin_std": 0.3242398798465729,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": -0.028623580932617188,
      "KL/mean": -0.01215296983718872,
      "KL/rejected_KL_mean": 0.0043182373046875,
      "KL/std": 0.23431165516376495,
      "epoch": 0.009070294784580499,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03294098377227783,
      "fcm_dpo/q_t": 0.5040556192398071,
      "grad_norm": 154.66268920898438,
      "learning_rate": 3.731343283582089e-08,
      "logits/chosen": 0.1724303513765335,
      "logits/rejected": 0.1311052143573761,
      "logps/chosen": -51.33598709106445,
      "logps/ref_chosen": -51.30736541748047,
      "logps/ref_rejected": -82.77239227294922,
      "logps/rejected": -82.76807403564453,
      "loss": 1.4098,
      "margin_dpo/margin_mean": -0.03294065594673157,
      "margin_dpo/margin_std": 0.32795512676239014,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.051555633544921875,
      "KL/mean": 0.037954360246658325,
      "KL/rejected_KL_mean": 0.024351119995117188,
      "KL/std": 0.20382466912269592,
      "epoch": 0.010582010582010581,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.027201533317565918,
      "fcm_dpo/q_t": 0.4966175854206085,
      "grad_norm": 135.3361053466797,
      "learning_rate": 4.477611940298507e-08,
      "logits/chosen": 0.02253446727991104,
      "logits/rejected": -0.021542033180594444,
      "logps/chosen": -51.40785598754883,
      "logps/ref_chosen": -51.45941162109375,
      "logps/ref_rejected": -66.3828125,
      "logps/rejected": -66.35845947265625,
      "loss": 1.3776,
      "margin_dpo/margin_mean": 0.027201414108276367,
      "margin_dpo/margin_std": 0.27836233377456665,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.0003108978271484375,
      "KL/mean": -0.007853224873542786,
      "KL/rejected_KL_mean": -0.015392303466796875,
      "KL/std": 0.22362451255321503,
      "epoch": 0.012093726379440665,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.015084236860275269,
      "fcm_dpo/q_t": 0.4981544613838196,
      "grad_norm": 141.2877960205078,
      "learning_rate": 5.223880597014925e-08,
      "logits/chosen": 0.09082719683647156,
      "logits/rejected": 0.06828100979328156,
      "logps/chosen": -62.19785690307617,
      "logps/ref_chosen": -62.197547912597656,
      "logps/ref_rejected": -74.66180419921875,
      "logps/rejected": -74.67720031738281,
      "loss": 1.3856,
      "margin_dpo/margin_mean": 0.01508358120918274,
      "margin_dpo/margin_std": 0.32920098304748535,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": -0.012613296508789062,
      "KL/mean": -0.04730965197086334,
      "KL/rejected_KL_mean": -0.08200836181640625,
      "KL/std": 0.2612247169017792,
      "epoch": 0.013605442176870748,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06939497590065002,
      "fcm_dpo/q_t": 0.49150800704956055,
      "grad_norm": 153.1192169189453,
      "learning_rate": 5.970149253731343e-08,
      "logits/chosen": 0.15654343366622925,
      "logits/rejected": 0.09825913608074188,
      "logps/chosen": -55.642333984375,
      "logps/ref_chosen": -55.629722595214844,
      "logps/ref_rejected": -86.21221923828125,
      "logps/rejected": -86.29423522949219,
      "loss": 1.3609,
      "margin_dpo/margin_mean": 0.06939518451690674,
      "margin_dpo/margin_std": 0.37213361263275146,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": 0.015171051025390625,
      "KL/mean": 0.031137198209762573,
      "KL/rejected_KL_mean": 0.047100067138671875,
      "KL/std": 0.27077072858810425,
      "epoch": 0.015117157974300832,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.03192782402038574,
      "fcm_dpo/q_t": 0.504030704498291,
      "grad_norm": 150.78793334960938,
      "learning_rate": 6.71641791044776e-08,
      "logits/chosen": 0.1278713345527649,
      "logits/rejected": 0.09713231027126312,
      "logps/chosen": -62.67543029785156,
      "logps/ref_chosen": -62.69060134887695,
      "logps/ref_rejected": -90.610107421875,
      "logps/rejected": -90.5630111694336,
      "loss": 1.4117,
      "margin_dpo/margin_mean": -0.03192758560180664,
      "margin_dpo/margin_std": 0.3764800429344177,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": 0.00289154052734375,
      "KL/mean": -0.015650570392608643,
      "KL/rejected_KL_mean": -0.03420257568359375,
      "KL/std": 0.21939970552921295,
      "epoch": 0.016628873771730914,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03709930181503296,
      "fcm_dpo/q_t": 0.49537503719329834,
      "grad_norm": 146.3813018798828,
      "learning_rate": 7.462686567164178e-08,
      "logits/chosen": 0.11935083568096161,
      "logits/rejected": 0.11234834790229797,
      "logps/chosen": -65.76422882080078,
      "logps/ref_chosen": -65.76712036132812,
      "logps/ref_rejected": -72.4764633178711,
      "logps/rejected": -72.51066589355469,
      "loss": 1.3732,
      "margin_dpo/margin_mean": 0.03709983825683594,
      "margin_dpo/margin_std": 0.2864682078361511,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": 0.017625808715820312,
      "KL/mean": 0.0031365156173706055,
      "KL/rejected_KL_mean": -0.011358261108398438,
      "KL/std": 0.21327649056911469,
      "epoch": 0.018140589569160998,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02898406982421875,
      "fcm_dpo/q_t": 0.4964328408241272,
      "grad_norm": 137.46507263183594,
      "learning_rate": 8.208955223880596e-08,
      "logits/chosen": 0.02509509213268757,
      "logits/rejected": 0.008943156339228153,
      "logps/chosen": -60.68726348876953,
      "logps/ref_chosen": -60.704891204833984,
      "logps/ref_rejected": -69.41564178466797,
      "logps/rejected": -69.427001953125,
      "loss": 1.3772,
      "margin_dpo/margin_mean": 0.02898406982421875,
      "margin_dpo/margin_std": 0.28719162940979004,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.0032062530517578125,
      "KL/mean": 0.032804936170578,
      "KL/rejected_KL_mean": 0.06882476806640625,
      "KL/std": 0.20512652397155762,
      "epoch": 0.019652305366591082,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07202756404876709,
      "fcm_dpo/q_t": 0.508876621723175,
      "grad_norm": 150.7777557373047,
      "learning_rate": 8.955223880597014e-08,
      "logits/chosen": 0.12878569960594177,
      "logits/rejected": 0.06433200091123581,
      "logps/chosen": -49.91246032714844,
      "logps/ref_chosen": -49.90925598144531,
      "logps/ref_rejected": -92.37818145751953,
      "logps/rejected": -92.30935668945312,
      "loss": 1.4282,
      "margin_dpo/margin_mean": -0.0720277726650238,
      "margin_dpo/margin_std": 0.29628726840019226,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.010652542114257812,
      "KL/mean": -0.012727156281471252,
      "KL/rejected_KL_mean": -0.036102294921875,
      "KL/std": 0.18057866394519806,
      "epoch": 0.021164021164021163,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04675278067588806,
      "fcm_dpo/q_t": 0.4942210912704468,
      "grad_norm": 145.45184326171875,
      "learning_rate": 9.701492537313432e-08,
      "logits/chosen": 0.10228344798088074,
      "logits/rejected": 0.084172323346138,
      "logps/chosen": -60.60813903808594,
      "logps/ref_chosen": -60.61879348754883,
      "logps/ref_rejected": -71.79306030273438,
      "logps/rejected": -71.82916259765625,
      "loss": 1.368,
      "margin_dpo/margin_mean": 0.04675331711769104,
      "margin_dpo/margin_std": 0.2820011377334595,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.004756927490234375,
      "KL/mean": -0.002736493945121765,
      "KL/rejected_KL_mean": -0.000713348388671875,
      "KL/std": 0.2475792020559311,
      "epoch": 0.022675736961451247,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.004045158624649048,
      "fcm_dpo/q_t": 0.5004628300666809,
      "grad_norm": 166.30584716796875,
      "learning_rate": 1.044776119402985e-07,
      "logits/chosen": 0.06926407665014267,
      "logits/rejected": 0.026052623987197876,
      "logps/chosen": -63.47429656982422,
      "logps/ref_chosen": -63.46953582763672,
      "logps/ref_rejected": -88.88951110839844,
      "logps/rejected": -88.89022827148438,
      "loss": 1.3966,
      "margin_dpo/margin_mean": -0.004044860601425171,
      "margin_dpo/margin_std": 0.36409926414489746,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": -0.022321701049804688,
      "KL/mean": -0.021601378917694092,
      "KL/rejected_KL_mean": -0.020885467529296875,
      "KL/std": 0.19117990136146545,
      "epoch": 0.02418745275888133,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0014389753341674805,
      "fcm_dpo/q_t": 0.5002340078353882,
      "grad_norm": 133.1244659423828,
      "learning_rate": 1.1194029850746268e-07,
      "logits/chosen": 0.09801945090293884,
      "logits/rejected": 0.06210765242576599,
      "logps/chosen": -46.55461883544922,
      "logps/ref_chosen": -46.53229904174805,
      "logps/ref_rejected": -74.27533721923828,
      "logps/rejected": -74.29621887207031,
      "loss": 1.3915,
      "margin_dpo/margin_mean": -0.0014390945434570312,
      "margin_dpo/margin_std": 0.2598055899143219,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": -0.011228561401367188,
      "KL/mean": -0.007338464260101318,
      "KL/rejected_KL_mean": -0.003448486328125,
      "KL/std": 0.24270084500312805,
      "epoch": 0.025699168556311415,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0077822208404541016,
      "fcm_dpo/q_t": 0.500993013381958,
      "grad_norm": 163.95631408691406,
      "learning_rate": 1.1940298507462686e-07,
      "logits/chosen": 0.06655038893222809,
      "logits/rejected": 0.04739490523934364,
      "logps/chosen": -64.08906555175781,
      "logps/ref_chosen": -64.07783508300781,
      "logps/ref_rejected": -86.40876770019531,
      "logps/rejected": -86.41221618652344,
      "loss": 1.3998,
      "margin_dpo/margin_mean": -0.007782965898513794,
      "margin_dpo/margin_std": 0.37937384843826294,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.0015811920166015625,
      "KL/mean": -0.005909636616706848,
      "KL/rejected_KL_mean": -0.0102386474609375,
      "KL/std": 0.22778195142745972,
      "epoch": 0.027210884353741496,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.008657962083816528,
      "fcm_dpo/q_t": 0.49891990423202515,
      "grad_norm": 140.09066772460938,
      "learning_rate": 1.2686567164179106e-07,
      "logits/chosen": 0.08881358802318573,
      "logits/rejected": 0.04353434592485428,
      "logps/chosen": -44.87591552734375,
      "logps/ref_chosen": -44.87433624267578,
      "logps/ref_rejected": -70.97604370117188,
      "logps/rejected": -70.98628234863281,
      "loss": 1.3882,
      "margin_dpo/margin_mean": 0.008657693862915039,
      "margin_dpo/margin_std": 0.31492000818252563,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": 0.027660369873046875,
      "KL/mean": 0.008508525788784027,
      "KL/rejected_KL_mean": -0.01064300537109375,
      "KL/std": 0.23382540047168732,
      "epoch": 0.02872260015117158,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03830493986606598,
      "fcm_dpo/q_t": 0.4951217472553253,
      "grad_norm": 155.16275024414062,
      "learning_rate": 1.343283582089552e-07,
      "logits/chosen": 0.051252156496047974,
      "logits/rejected": 0.038061805069446564,
      "logps/chosen": -68.13214874267578,
      "logps/ref_chosen": -68.1598129272461,
      "logps/ref_rejected": -81.17138671875,
      "logps/rejected": -81.18203735351562,
      "loss": 1.3741,
      "margin_dpo/margin_mean": 0.03830514848232269,
      "margin_dpo/margin_std": 0.32980090379714966,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.015218734741210938,
      "KL/mean": -0.012842193245887756,
      "KL/rejected_KL_mean": -0.04090118408203125,
      "KL/std": 0.2211008071899414,
      "epoch": 0.030234315948601664,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.056119710206985474,
      "fcm_dpo/q_t": 0.4929888844490051,
      "grad_norm": 144.45556640625,
      "learning_rate": 1.4179104477611938e-07,
      "logits/chosen": 0.17593975365161896,
      "logits/rejected": 0.15117508172988892,
      "logps/chosen": -53.66334533691406,
      "logps/ref_chosen": -53.67856216430664,
      "logps/ref_rejected": -74.16911315917969,
      "logps/rejected": -74.21002197265625,
      "loss": 1.363,
      "margin_dpo/margin_mean": 0.05611985921859741,
      "margin_dpo/margin_std": 0.26307860016822815,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.026338577270507812,
      "KL/mean": 0.010084077715873718,
      "KL/rejected_KL_mean": -0.00617218017578125,
      "KL/std": 0.2499391734600067,
      "epoch": 0.031746031746031744,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.032517045736312866,
      "fcm_dpo/q_t": 0.4959341883659363,
      "grad_norm": 144.1366424560547,
      "learning_rate": 1.4925373134328355e-07,
      "logits/chosen": 0.1175660490989685,
      "logits/rejected": 0.09148456901311874,
      "logps/chosen": -64.67521667480469,
      "logps/ref_chosen": -64.70155334472656,
      "logps/ref_rejected": -81.02095031738281,
      "logps/rejected": -81.02711486816406,
      "loss": 1.3767,
      "margin_dpo/margin_mean": 0.032516419887542725,
      "margin_dpo/margin_std": 0.31374847888946533,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": -0.013763427734375,
      "KL/mean": -0.009405761957168579,
      "KL/rejected_KL_mean": -0.005046844482421875,
      "KL/std": 0.21892325580120087,
      "epoch": 0.03325774754346183,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.008712172508239746,
      "fcm_dpo/q_t": 0.501030445098877,
      "grad_norm": 146.3167266845703,
      "learning_rate": 1.5671641791044775e-07,
      "logits/chosen": 0.0047190384939312935,
      "logits/rejected": -0.01616102084517479,
      "logps/chosen": -58.04975509643555,
      "logps/ref_chosen": -58.03599166870117,
      "logps/ref_rejected": -80.72721862792969,
      "logps/rejected": -80.73226928710938,
      "loss": 1.3962,
      "margin_dpo/margin_mean": -0.008712053298950195,
      "margin_dpo/margin_std": 0.29911357164382935,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.010440826416015625,
      "KL/mean": -0.00901477038860321,
      "KL/rejected_KL_mean": -0.028472900390625,
      "KL/std": 0.22521373629570007,
      "epoch": 0.03476946334089191,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03891590237617493,
      "fcm_dpo/q_t": 0.495150625705719,
      "grad_norm": 163.6617431640625,
      "learning_rate": 1.6417910447761193e-07,
      "logits/chosen": 0.1290198564529419,
      "logits/rejected": 0.10404293239116669,
      "logps/chosen": -66.34564208984375,
      "logps/ref_chosen": -66.35608673095703,
      "logps/ref_rejected": -93.02769470214844,
      "logps/rejected": -93.05616760253906,
      "loss": 1.3721,
      "margin_dpo/margin_mean": 0.03891530632972717,
      "margin_dpo/margin_std": 0.2813330888748169,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": -0.015592575073242188,
      "KL/mean": -0.013847090303897858,
      "KL/rejected_KL_mean": -0.012102127075195312,
      "KL/std": 0.20355567336082458,
      "epoch": 0.036281179138321996,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.003486260771751404,
      "fcm_dpo/q_t": 0.5004266500473022,
      "grad_norm": 132.54791259765625,
      "learning_rate": 1.716417910447761e-07,
      "logits/chosen": 0.15359747409820557,
      "logits/rejected": 0.12006732821464539,
      "logps/chosen": -54.476829528808594,
      "logps/ref_chosen": -54.461238861083984,
      "logps/ref_rejected": -68.33817291259766,
      "logps/rejected": -68.35028076171875,
      "loss": 1.3921,
      "margin_dpo/margin_mean": -0.00348663330078125,
      "margin_dpo/margin_std": 0.2564446032047272,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": -0.0045871734619140625,
      "KL/mean": -0.020289063453674316,
      "KL/rejected_KL_mean": -0.035991668701171875,
      "KL/std": 0.20003153383731842,
      "epoch": 0.03779289493575208,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03139996528625488,
      "fcm_dpo/q_t": 0.4961238503456116,
      "grad_norm": 146.73809814453125,
      "learning_rate": 1.7910447761194027e-07,
      "logits/chosen": 0.13044767081737518,
      "logits/rejected": 0.07712407410144806,
      "logps/chosen": -60.0087890625,
      "logps/ref_chosen": -60.00420379638672,
      "logps/ref_rejected": -90.47376251220703,
      "logps/rejected": -90.50975036621094,
      "loss": 1.3746,
      "margin_dpo/margin_mean": 0.031399667263031006,
      "margin_dpo/margin_std": 0.2473982870578766,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": -0.019536972045898438,
      "KL/mean": -0.01541091501712799,
      "KL/rejected_KL_mean": -0.01128387451171875,
      "KL/std": 0.2352585345506668,
      "epoch": 0.039304610733182165,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.008253306150436401,
      "fcm_dpo/q_t": 0.5010988116264343,
      "grad_norm": 148.53831481933594,
      "learning_rate": 1.8656716417910447e-07,
      "logits/chosen": 0.10610733926296234,
      "logits/rejected": 0.0877869576215744,
      "logps/chosen": -56.83869171142578,
      "logps/ref_chosen": -56.81915283203125,
      "logps/ref_rejected": -77.84333038330078,
      "logps/rejected": -77.8546142578125,
      "loss": 1.3978,
      "margin_dpo/margin_mean": -0.00825345516204834,
      "margin_dpo/margin_std": 0.34305694699287415,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": -0.032016754150390625,
      "KL/mean": -0.011619418859481812,
      "KL/rejected_KL_mean": 0.008779525756835938,
      "KL/std": 0.2065221071243286,
      "epoch": 0.04081632653061224,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.04079902172088623,
      "fcm_dpo/q_t": 0.5050686597824097,
      "grad_norm": 146.85816955566406,
      "learning_rate": 1.9402985074626865e-07,
      "logits/chosen": 0.1110733151435852,
      "logits/rejected": 0.08588938415050507,
      "logps/chosen": -62.90904235839844,
      "logps/ref_chosen": -62.87702560424805,
      "logps/ref_rejected": -71.34437561035156,
      "logps/rejected": -71.3355941772461,
      "loss": 1.4114,
      "margin_dpo/margin_mean": -0.04079878330230713,
      "margin_dpo/margin_std": 0.2700217366218567,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": -0.011888504028320312,
      "KL/mean": -0.022637784481048584,
      "KL/rejected_KL_mean": -0.03338813781738281,
      "KL/std": 0.22135095298290253,
      "epoch": 0.042328042328042326,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.021501481533050537,
      "fcm_dpo/q_t": 0.49734407663345337,
      "grad_norm": 138.68606567382812,
      "learning_rate": 2.0149253731343282e-07,
      "logits/chosen": 0.05210627242922783,
      "logits/rejected": 0.043426185846328735,
      "logps/chosen": -59.84526443481445,
      "logps/ref_chosen": -59.8333740234375,
      "logps/ref_rejected": -70.39804077148438,
      "logps/rejected": -70.43142700195312,
      "loss": 1.3816,
      "margin_dpo/margin_mean": 0.021502047777175903,
      "margin_dpo/margin_std": 0.3062840700149536,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.008098602294921875,
      "KL/mean": -0.04032225161790848,
      "KL/rejected_KL_mean": -0.08873367309570312,
      "KL/std": 0.21123595535755157,
      "epoch": 0.04383975812547241,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09683476388454437,
      "fcm_dpo/q_t": 0.4879266321659088,
      "grad_norm": 158.5276336669922,
      "learning_rate": 2.08955223880597e-07,
      "logits/chosen": 0.1421521008014679,
      "logits/rejected": 0.12432709336280823,
      "logps/chosen": -74.11210632324219,
      "logps/ref_chosen": -74.12020111083984,
      "logps/ref_rejected": -83.33099365234375,
      "logps/rejected": -83.41972351074219,
      "loss": 1.3441,
      "margin_dpo/margin_mean": 0.09683471918106079,
      "margin_dpo/margin_std": 0.2897757589817047,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.017908096313476562,
      "KL/mean": -0.01277931034564972,
      "KL/rejected_KL_mean": -0.043468475341796875,
      "KL/std": 0.23041898012161255,
      "epoch": 0.045351473922902494,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06137612462043762,
      "fcm_dpo/q_t": 0.49244004487991333,
      "grad_norm": 148.99423217773438,
      "learning_rate": 2.1641791044776117e-07,
      "logits/chosen": 0.12142124027013779,
      "logits/rejected": 0.06727240234613419,
      "logps/chosen": -50.73338317871094,
      "logps/ref_chosen": -50.75128936767578,
      "logps/ref_rejected": -89.29063415527344,
      "logps/rejected": -89.3341064453125,
      "loss": 1.3625,
      "margin_dpo/margin_mean": 0.061375439167022705,
      "margin_dpo/margin_std": 0.32606202363967896,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": -0.0027294158935546875,
      "KL/mean": -0.04074978828430176,
      "KL/rejected_KL_mean": -0.07876968383789062,
      "KL/std": 0.24881835281848907,
      "epoch": 0.04686318972033258,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07603961229324341,
      "fcm_dpo/q_t": 0.49055615067481995,
      "grad_norm": 169.32138061523438,
      "learning_rate": 2.2388059701492537e-07,
      "logits/chosen": 0.1183767020702362,
      "logits/rejected": 0.07146687060594559,
      "logps/chosen": -65.33948516845703,
      "logps/ref_chosen": -65.33675384521484,
      "logps/ref_rejected": -100.76666259765625,
      "logps/rejected": -100.84542846679688,
      "loss": 1.3545,
      "margin_dpo/margin_mean": 0.07603979110717773,
      "margin_dpo/margin_std": 0.29326799511909485,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": -0.006229400634765625,
      "KL/mean": -0.017637237906455994,
      "KL/rejected_KL_mean": -0.02904510498046875,
      "KL/std": 0.22603976726531982,
      "epoch": 0.04837490551776266,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.02282276749610901,
      "fcm_dpo/q_t": 0.497119665145874,
      "grad_norm": 151.53550720214844,
      "learning_rate": 2.3134328358208954e-07,
      "logits/chosen": 0.0814221054315567,
      "logits/rejected": 0.07352820038795471,
      "logps/chosen": -67.18955993652344,
      "logps/ref_chosen": -67.18333435058594,
      "logps/ref_rejected": -82.80763244628906,
      "logps/rejected": -82.83668518066406,
      "loss": 1.3814,
      "margin_dpo/margin_mean": 0.02282300591468811,
      "margin_dpo/margin_std": 0.32116997241973877,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": -0.047595977783203125,
      "KL/mean": -0.04682595282793045,
      "KL/rejected_KL_mean": -0.046054840087890625,
      "KL/std": 0.2515178620815277,
      "epoch": 0.049886621315192746,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.001542612910270691,
      "fcm_dpo/q_t": 0.5000810623168945,
      "grad_norm": 160.85826110839844,
      "learning_rate": 2.388059701492537e-07,
      "logits/chosen": 0.033244818449020386,
      "logits/rejected": 0.007102368399500847,
      "logps/chosen": -64.08707427978516,
      "logps/ref_chosen": -64.03948211669922,
      "logps/ref_rejected": -75.68357849121094,
      "logps/rejected": -75.7296371459961,
      "loss": 1.3958,
      "margin_dpo/margin_mean": -0.0015421658754348755,
      "margin_dpo/margin_std": 0.3630064129829407,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": -0.02014923095703125,
      "KL/mean": -0.06804826855659485,
      "KL/rejected_KL_mean": -0.11594772338867188,
      "KL/std": 0.22508756816387177,
      "epoch": 0.05139833711262283,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09580296277999878,
      "fcm_dpo/q_t": 0.48807454109191895,
      "grad_norm": 139.12904357910156,
      "learning_rate": 2.4626865671641786e-07,
      "logits/chosen": 0.09272102266550064,
      "logits/rejected": 0.06317080557346344,
      "logps/chosen": -53.684444427490234,
      "logps/ref_chosen": -53.6642951965332,
      "logps/ref_rejected": -65.77989959716797,
      "logps/rejected": -65.89584350585938,
      "loss": 1.3451,
      "margin_dpo/margin_mean": 0.09580284357070923,
      "margin_dpo/margin_std": 0.31252580881118774,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": -0.06003761291503906,
      "KL/mean": -0.08559216558933258,
      "KL/rejected_KL_mean": -0.11114883422851562,
      "KL/std": 0.25317007303237915,
      "epoch": 0.05291005291005291,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0511077344417572,
      "fcm_dpo/q_t": 0.4937984347343445,
      "grad_norm": 138.31344604492188,
      "learning_rate": 2.537313432835821e-07,
      "logits/chosen": 0.04526316747069359,
      "logits/rejected": 0.02307654544711113,
      "logps/chosen": -61.076900482177734,
      "logps/ref_chosen": -61.01686096191406,
      "logps/ref_rejected": -72.78598022460938,
      "logps/rejected": -72.89713287353516,
      "loss": 1.371,
      "margin_dpo/margin_mean": 0.05110803246498108,
      "margin_dpo/margin_std": 0.39918971061706543,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": -0.09109878540039062,
      "KL/mean": -0.09426809847354889,
      "KL/rejected_KL_mean": -0.09743881225585938,
      "KL/std": 0.25269731879234314,
      "epoch": 0.05442176870748299,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0063409507274627686,
      "fcm_dpo/q_t": 0.49925148487091064,
      "grad_norm": 144.56039428710938,
      "learning_rate": 2.611940298507462e-07,
      "logits/chosen": 0.12025703489780426,
      "logits/rejected": 0.06593604385852814,
      "logps/chosen": -50.62845993041992,
      "logps/ref_chosen": -50.53736114501953,
      "logps/ref_rejected": -78.11678314208984,
      "logps/rejected": -78.21421813964844,
      "loss": 1.3923,
      "margin_dpo/margin_mean": 0.006341129541397095,
      "margin_dpo/margin_std": 0.38176584243774414,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": -0.03309440612792969,
      "KL/mean": -0.09751610457897186,
      "KL/rejected_KL_mean": -0.16194534301757812,
      "KL/std": 0.25529831647872925,
      "epoch": 0.055933484504913075,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12884435057640076,
      "fcm_dpo/q_t": 0.48412883281707764,
      "grad_norm": 179.44265747070312,
      "learning_rate": 2.686567164179104e-07,
      "logits/chosen": 0.10142149031162262,
      "logits/rejected": 0.021988654509186745,
      "logps/chosen": -59.58704376220703,
      "logps/ref_chosen": -59.55394744873047,
      "logps/ref_rejected": -108.27702331542969,
      "logps/rejected": -108.43897247314453,
      "loss": 1.3303,
      "margin_dpo/margin_mean": 0.1288444697856903,
      "margin_dpo/margin_std": 0.34434449672698975,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": -0.09745025634765625,
      "KL/mean": -0.12052340805530548,
      "KL/rejected_KL_mean": -0.14359664916992188,
      "KL/std": 0.25720837712287903,
      "epoch": 0.05744520030234316,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04614517092704773,
      "fcm_dpo/q_t": 0.4943495988845825,
      "grad_norm": 146.76524353027344,
      "learning_rate": 2.761194029850746e-07,
      "logits/chosen": 0.06609077006578445,
      "logits/rejected": 0.0521436482667923,
      "logps/chosen": -65.88580322265625,
      "logps/ref_chosen": -65.78836059570312,
      "logps/ref_rejected": -76.1619873046875,
      "logps/rejected": -76.30558776855469,
      "loss": 1.3718,
      "margin_dpo/margin_mean": 0.04614526033401489,
      "margin_dpo/margin_std": 0.3619215488433838,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": -0.13004684448242188,
      "KL/mean": -0.135334312915802,
      "KL/rejected_KL_mean": -0.14062118530273438,
      "KL/std": 0.27177947759628296,
      "epoch": 0.05895691609977324,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.010573983192443848,
      "fcm_dpo/q_t": 0.4986897110939026,
      "grad_norm": 146.2440948486328,
      "learning_rate": 2.8358208955223876e-07,
      "logits/chosen": 0.13989418745040894,
      "logits/rejected": 0.11372476071119308,
      "logps/chosen": -57.306854248046875,
      "logps/ref_chosen": -57.17681121826172,
      "logps/ref_rejected": -79.486328125,
      "logps/rejected": -79.626953125,
      "loss": 1.3891,
      "margin_dpo/margin_mean": 0.010573387145996094,
      "margin_dpo/margin_std": 0.35882243514060974,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": -0.1058807373046875,
      "KL/mean": -0.09615175426006317,
      "KL/rejected_KL_mean": -0.08642578125,
      "KL/std": 0.2410488724708557,
      "epoch": 0.06046863189720333,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.01946231722831726,
      "fcm_dpo/q_t": 0.5024391412734985,
      "grad_norm": 161.63197326660156,
      "learning_rate": 2.9104477611940296e-07,
      "logits/chosen": 0.13694174587726593,
      "logits/rejected": 0.08591257035732269,
      "logps/chosen": -61.44004821777344,
      "logps/ref_chosen": -61.33416748046875,
      "logps/ref_rejected": -79.10697174072266,
      "logps/rejected": -79.19339752197266,
      "loss": 1.4037,
      "margin_dpo/margin_mean": -0.019462496042251587,
      "margin_dpo/margin_std": 0.3406964838504791,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": -0.11898040771484375,
      "KL/mean": -0.15551243722438812,
      "KL/rejected_KL_mean": -0.19203948974609375,
      "KL/std": 0.2736630439758301,
      "epoch": 0.06198034769463341,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07305607199668884,
      "fcm_dpo/q_t": 0.4909464120864868,
      "grad_norm": 149.59732055664062,
      "learning_rate": 2.985074626865671e-07,
      "logits/chosen": 0.034129172563552856,
      "logits/rejected": 0.014605993404984474,
      "logps/chosen": -67.66571044921875,
      "logps/ref_chosen": -67.5467300415039,
      "logps/ref_rejected": -83.87788391113281,
      "logps/rejected": -84.06993103027344,
      "loss": 1.3576,
      "margin_dpo/margin_mean": 0.07305684685707092,
      "margin_dpo/margin_std": 0.34322357177734375,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": -0.11904716491699219,
      "KL/mean": -0.13330422341823578,
      "KL/rejected_KL_mean": -0.1475677490234375,
      "KL/std": 0.2623087167739868,
      "epoch": 0.06349206349206349,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.028522074222564697,
      "fcm_dpo/q_t": 0.49644795060157776,
      "grad_norm": 145.64328002929688,
      "learning_rate": 3.059701492537313e-07,
      "logits/chosen": 0.05525980144739151,
      "logits/rejected": 0.03359142690896988,
      "logps/chosen": -61.38390350341797,
      "logps/ref_chosen": -61.26485824584961,
      "logps/ref_rejected": -76.3629150390625,
      "logps/rejected": -76.51048278808594,
      "loss": 1.3807,
      "margin_dpo/margin_mean": 0.028522223234176636,
      "margin_dpo/margin_std": 0.36844220757484436,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": -0.1389312744140625,
      "KL/mean": -0.14179641008377075,
      "KL/rejected_KL_mean": -0.14465904235839844,
      "KL/std": 0.2707711458206177,
      "epoch": 0.06500377928949358,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.00572890043258667,
      "fcm_dpo/q_t": 0.49924448132514954,
      "grad_norm": 172.56381225585938,
      "learning_rate": 3.134328358208955e-07,
      "logits/chosen": 0.08687476813793182,
      "logits/rejected": 0.07593454420566559,
      "logps/chosen": -71.94795989990234,
      "logps/ref_chosen": -71.80902862548828,
      "logps/ref_rejected": -81.12464141845703,
      "logps/rejected": -81.26930236816406,
      "loss": 1.3922,
      "margin_dpo/margin_mean": 0.0057284533977508545,
      "margin_dpo/margin_std": 0.37383711338043213,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": -0.1993885040283203,
      "KL/mean": -0.18441106379032135,
      "KL/rejected_KL_mean": -0.16943359375,
      "KL/std": 0.2821127772331238,
      "epoch": 0.06651549508692366,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.029954224824905396,
      "fcm_dpo/q_t": 0.5037481784820557,
      "grad_norm": 165.5587615966797,
      "learning_rate": 3.2089552238805965e-07,
      "logits/chosen": 0.04478081315755844,
      "logits/rejected": 0.014489535242319107,
      "logps/chosen": -66.74981689453125,
      "logps/ref_chosen": -66.55043029785156,
      "logps/ref_rejected": -85.06198120117188,
      "logps/rejected": -85.23141479492188,
      "loss": 1.4131,
      "margin_dpo/margin_mean": -0.029954195022583008,
      "margin_dpo/margin_std": 0.4348960518836975,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": -0.1451416015625,
      "KL/mean": -0.19692976772785187,
      "KL/rejected_KL_mean": -0.24872207641601562,
      "KL/std": 0.26874667406082153,
      "epoch": 0.06802721088435375,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10358306765556335,
      "fcm_dpo/q_t": 0.4871301054954529,
      "grad_norm": 155.59429931640625,
      "learning_rate": 3.2835820895522385e-07,
      "logits/chosen": 0.11731548607349396,
      "logits/rejected": 0.06474698334932327,
      "logps/chosen": -62.38899612426758,
      "logps/ref_chosen": -62.24385452270508,
      "logps/ref_rejected": -92.96665954589844,
      "logps/rejected": -93.21538543701172,
      "loss": 1.3433,
      "margin_dpo/margin_mean": 0.10358336567878723,
      "margin_dpo/margin_std": 0.35981160402297974,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": -0.07675552368164062,
      "KL/mean": -0.15909118950366974,
      "KL/rejected_KL_mean": -0.24142837524414062,
      "KL/std": 0.34129780530929565,
      "epoch": 0.06953892668178382,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16466912627220154,
      "fcm_dpo/q_t": 0.4797493815422058,
      "grad_norm": 147.95513916015625,
      "learning_rate": 3.3582089552238805e-07,
      "logits/chosen": 0.13846392929553986,
      "logits/rejected": 0.0918339341878891,
      "logps/chosen": -61.575660705566406,
      "logps/ref_chosen": -61.498905181884766,
      "logps/ref_rejected": -78.91172790527344,
      "logps/rejected": -79.15315246582031,
      "loss": 1.3186,
      "margin_dpo/margin_mean": 0.16466832160949707,
      "margin_dpo/margin_std": 0.4573308229446411,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": -0.14517784118652344,
      "KL/mean": -0.21937622129917145,
      "KL/rejected_KL_mean": -0.2935752868652344,
      "KL/std": 0.2848299443721771,
      "epoch": 0.0710506424792139,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1483970582485199,
      "fcm_dpo/q_t": 0.4816315770149231,
      "grad_norm": 138.68087768554688,
      "learning_rate": 3.432835820895522e-07,
      "logits/chosen": 0.027657022699713707,
      "logits/rejected": -0.014819873496890068,
      "logps/chosen": -51.72352600097656,
      "logps/ref_chosen": -51.578346252441406,
      "logps/ref_rejected": -68.2215576171875,
      "logps/rejected": -68.51513671875,
      "loss": 1.3235,
      "margin_dpo/margin_mean": 0.14839708805084229,
      "margin_dpo/margin_std": 0.3989714980125427,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": -0.2627086639404297,
      "KL/mean": -0.2556438446044922,
      "KL/rejected_KL_mean": -0.2485809326171875,
      "KL/std": 0.33050912618637085,
      "epoch": 0.07256235827664399,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.014134973287582397,
      "fcm_dpo/q_t": 0.5017518997192383,
      "grad_norm": 137.5546417236328,
      "learning_rate": 3.507462686567164e-07,
      "logits/chosen": 0.16078418493270874,
      "logits/rejected": 0.130637064576149,
      "logps/chosen": -52.056358337402344,
      "logps/ref_chosen": -51.79365158081055,
      "logps/ref_rejected": -64.22503662109375,
      "logps/rejected": -64.47361755371094,
      "loss": 1.4069,
      "margin_dpo/margin_mean": -0.014135152101516724,
      "margin_dpo/margin_std": 0.4661322236061096,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": -0.2023334503173828,
      "KL/mean": -0.2441607415676117,
      "KL/rejected_KL_mean": -0.28598785400390625,
      "KL/std": 0.3505373001098633,
      "epoch": 0.07407407407407407,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08365237712860107,
      "fcm_dpo/q_t": 0.48992884159088135,
      "grad_norm": 132.201416015625,
      "learning_rate": 3.5820895522388055e-07,
      "logits/chosen": 0.02365894615650177,
      "logits/rejected": 0.0024696458131074905,
      "logps/chosen": -58.336936950683594,
      "logps/ref_chosen": -58.13460159301758,
      "logps/ref_rejected": -64.63206481933594,
      "logps/rejected": -64.91806030273438,
      "loss": 1.3603,
      "margin_dpo/margin_mean": 0.08365324139595032,
      "margin_dpo/margin_std": 0.5017350912094116,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": -0.2725200653076172,
      "KL/mean": -0.3334037661552429,
      "KL/rejected_KL_mean": -0.3942909240722656,
      "KL/std": 0.34573113918304443,
      "epoch": 0.07558578987150416,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12176498770713806,
      "fcm_dpo/q_t": 0.48493263125419617,
      "grad_norm": 135.75376892089844,
      "learning_rate": 3.6567164179104475e-07,
      "logits/chosen": 0.10907851159572601,
      "logits/rejected": 0.0793529525399208,
      "logps/chosen": -53.12895965576172,
      "logps/ref_chosen": -52.85643768310547,
      "logps/ref_rejected": -72.17460632324219,
      "logps/rejected": -72.56889343261719,
      "loss": 1.3368,
      "margin_dpo/margin_mean": 0.12176531553268433,
      "margin_dpo/margin_std": 0.41100114583969116,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": -0.25723838806152344,
      "KL/mean": -0.3518369793891907,
      "KL/rejected_KL_mean": -0.4464378356933594,
      "KL/std": 0.3313744068145752,
      "epoch": 0.07709750566893424,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.18919500708580017,
      "fcm_dpo/q_t": 0.476720929145813,
      "grad_norm": 143.0610809326172,
      "learning_rate": 3.7313432835820895e-07,
      "logits/chosen": 0.08544561266899109,
      "logits/rejected": 0.05770985782146454,
      "logps/chosen": -63.91368103027344,
      "logps/ref_chosen": -63.65644073486328,
      "logps/ref_rejected": -86.13229370117188,
      "logps/rejected": -86.5787353515625,
      "loss": 1.3049,
      "margin_dpo/margin_mean": 0.18919536471366882,
      "margin_dpo/margin_std": 0.4166017174720764,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": -0.3161792755126953,
      "KL/mean": -0.4143037796020508,
      "KL/rejected_KL_mean": -0.5124320983886719,
      "KL/std": 0.395096093416214,
      "epoch": 0.07860922146636433,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1962490975856781,
      "fcm_dpo/q_t": 0.47601208090782166,
      "grad_norm": 155.0868682861328,
      "learning_rate": 3.805970149253731e-07,
      "logits/chosen": 0.09383442997932434,
      "logits/rejected": 0.0436672680079937,
      "logps/chosen": -68.1563949584961,
      "logps/ref_chosen": -67.8402099609375,
      "logps/ref_rejected": -96.97090911865234,
      "logps/rejected": -97.48333740234375,
      "loss": 1.3065,
      "margin_dpo/margin_mean": 0.19624871015548706,
      "margin_dpo/margin_std": 0.5056653618812561,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": -0.31052398681640625,
      "KL/mean": -0.38905656337738037,
      "KL/rejected_KL_mean": -0.4675884246826172,
      "KL/std": 0.34652209281921387,
      "epoch": 0.0801209372637944,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15706393122673035,
      "fcm_dpo/q_t": 0.480529248714447,
      "grad_norm": 132.29354858398438,
      "learning_rate": 3.880597014925373e-07,
      "logits/chosen": 0.07587432116270065,
      "logits/rejected": 0.06526178866624832,
      "logps/chosen": -57.188663482666016,
      "logps/ref_chosen": -56.87813949584961,
      "logps/ref_rejected": -60.75569152832031,
      "logps/rejected": -61.22328186035156,
      "loss": 1.3189,
      "margin_dpo/margin_mean": 0.15706408023834229,
      "margin_dpo/margin_std": 0.37288177013397217,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": -0.34480857849121094,
      "KL/mean": -0.42161333560943604,
      "KL/rejected_KL_mean": -0.4984149932861328,
      "KL/std": 0.36454081535339355,
      "epoch": 0.08163265306122448,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15360459685325623,
      "fcm_dpo/q_t": 0.4812043607234955,
      "grad_norm": 130.47328186035156,
      "learning_rate": 3.9552238805970144e-07,
      "logits/chosen": 0.05367577075958252,
      "logits/rejected": 0.03840417414903641,
      "logps/chosen": -47.61172866821289,
      "logps/ref_chosen": -47.26692199707031,
      "logps/ref_rejected": -62.19426727294922,
      "logps/rejected": -62.69268035888672,
      "loss": 1.3261,
      "margin_dpo/margin_mean": 0.1536046266555786,
      "margin_dpo/margin_std": 0.4828716516494751,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": -0.3637104034423828,
      "KL/mean": -0.48800647258758545,
      "KL/rejected_KL_mean": -0.6123085021972656,
      "KL/std": 0.4421403408050537,
      "epoch": 0.08314436885865457,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.24859541654586792,
      "fcm_dpo/q_t": 0.4701007902622223,
      "grad_norm": 145.4849853515625,
      "learning_rate": 4.0298507462686564e-07,
      "logits/chosen": 0.03831220045685768,
      "logits/rejected": -0.03851715475320816,
      "logps/chosen": -50.689903259277344,
      "logps/ref_chosen": -50.32619094848633,
      "logps/ref_rejected": -92.44389343261719,
      "logps/rejected": -93.05619812011719,
      "loss": 1.2889,
      "margin_dpo/margin_mean": 0.24859526753425598,
      "margin_dpo/margin_std": 0.5815203785896301,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": -0.326263427734375,
      "KL/mean": -0.4176982045173645,
      "KL/rejected_KL_mean": -0.509124755859375,
      "KL/std": 0.46035683155059814,
      "epoch": 0.08465608465608465,
      "fcm_dpo/beta": 0.5,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.18286392092704773,
      "fcm_dpo/q_t": 0.4773871898651123,
      "grad_norm": 134.97463989257812,
      "learning_rate": 4.1044776119402984e-07,
      "logits/chosen": 0.13615721464157104,
      "logits/rejected": 0.11348386853933334,
      "logps/chosen": -57.09323501586914,
      "logps/ref_chosen": -56.766971588134766,
      "logps/ref_rejected": -66.30504608154297,
      "logps/rejected": -66.81417083740234,
      "loss": 1.3192,
      "margin_dpo/margin_mean": 0.18286418914794922,
      "margin_dpo/margin_std": 0.5859323740005493,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": -0.4256000518798828,
      "KL/mean": -0.5890066623687744,
      "KL/rejected_KL_mean": -0.752410888671875,
      "KL/std": 0.5239032506942749,
      "epoch": 0.08616780045351474,
      "fcm_dpo/beta": 0.5084183216094971,
      "fcm_dpo/delta": 0.0834825336933136,
      "fcm_dpo/margin": 0.3268120288848877,
      "fcm_dpo/q_t": 0.46022289991378784,
      "grad_norm": 138.57626342773438,
      "learning_rate": 4.17910447761194e-07,
      "logits/chosen": 0.09445017576217651,
      "logits/rejected": 0.030366262421011925,
      "logps/chosen": -58.19334411621094,
      "logps/ref_chosen": -57.76774597167969,
      "logps/ref_rejected": -82.75698852539062,
      "logps/rejected": -83.5093994140625,
      "loss": 1.25,
      "margin_dpo/margin_mean": 0.3268115520477295,
      "margin_dpo/margin_std": 0.5852609872817993,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": -0.4801292419433594,
      "KL/mean": -0.5928635597229004,
      "KL/rejected_KL_mean": -0.7055931091308594,
      "KL/std": 0.5489867925643921,
      "epoch": 0.08767951625094482,
      "fcm_dpo/beta": 0.5168270468711853,
      "fcm_dpo/delta": 0.08201850950717926,
      "fcm_dpo/margin": 0.22546300292015076,
      "fcm_dpo/q_t": 0.47448039054870605,
      "grad_norm": 150.22698974609375,
      "learning_rate": 4.253731343283582e-07,
      "logits/chosen": 0.04649518430233002,
      "logits/rejected": 0.03131863474845886,
      "logps/chosen": -73.24421691894531,
      "logps/ref_chosen": -72.76408386230469,
      "logps/ref_rejected": -84.49275207519531,
      "logps/rejected": -85.19834899902344,
      "loss": 1.3165,
      "margin_dpo/margin_mean": 0.22546246647834778,
      "margin_dpo/margin_std": 0.7932426333427429,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": -0.46721649169921875,
      "KL/mean": -0.6028290390968323,
      "KL/rejected_KL_mean": -0.7384414672851562,
      "KL/std": 0.5065209269523621,
      "epoch": 0.08919123204837491,
      "fcm_dpo/beta": 0.5200226306915283,
      "fcm_dpo/delta": 0.061451178044080734,
      "fcm_dpo/margin": 0.27122339606285095,
      "fcm_dpo/q_t": 0.46698644757270813,
      "grad_norm": 126.58794403076172,
      "learning_rate": 4.3283582089552234e-07,
      "logits/chosen": 0.1147925928235054,
      "logits/rejected": 0.049807533621788025,
      "logps/chosen": -50.287994384765625,
      "logps/ref_chosen": -49.820777893066406,
      "logps/ref_rejected": -77.14368438720703,
      "logps/rejected": -77.88212585449219,
      "loss": 1.2811,
      "margin_dpo/margin_mean": 0.2712229788303375,
      "margin_dpo/margin_std": 0.6388437151908875,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": -0.5611343383789062,
      "KL/mean": -0.5829050540924072,
      "KL/rejected_KL_mean": -0.6046791076660156,
      "KL/std": 0.5296966433525085,
      "epoch": 0.09070294784580499,
      "fcm_dpo/beta": 0.5232181549072266,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04354429244995117,
      "fcm_dpo/q_t": 0.4938344657421112,
      "grad_norm": 168.19915771484375,
      "learning_rate": 4.4029850746268654e-07,
      "logits/chosen": 0.12396377325057983,
      "logits/rejected": 0.12253884226083755,
      "logps/chosen": -63.78590774536133,
      "logps/ref_chosen": -63.22477340698242,
      "logps/ref_rejected": -61.360477447509766,
      "logps/rejected": -61.96515655517578,
      "loss": 1.3973,
      "margin_dpo/margin_mean": 0.04354393482208252,
      "margin_dpo/margin_std": 0.7023971676826477,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": -0.6069221496582031,
      "KL/mean": -0.6606093645095825,
      "KL/rejected_KL_mean": -0.71429443359375,
      "KL/std": 0.5295801162719727,
      "epoch": 0.09221466364323508,
      "fcm_dpo/beta": 0.5232181549072266,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10737094283103943,
      "fcm_dpo/q_t": 0.4864484965801239,
      "grad_norm": 149.2123565673828,
      "learning_rate": 4.4776119402985074e-07,
      "logits/chosen": 0.10993358492851257,
      "logits/rejected": 0.07838596403598785,
      "logps/chosen": -49.62371826171875,
      "logps/ref_chosen": -49.01679992675781,
      "logps/ref_rejected": -74.90817260742188,
      "logps/rejected": -75.62246704101562,
      "loss": 1.3649,
      "margin_dpo/margin_mean": 0.10737112164497375,
      "margin_dpo/margin_std": 0.7071089744567871,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": -0.6248626708984375,
      "KL/mean": -0.7315359115600586,
      "KL/rejected_KL_mean": -0.8382072448730469,
      "KL/std": 0.5189784169197083,
      "epoch": 0.09372637944066516,
      "fcm_dpo/beta": 0.5232181549072266,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.21334949135780334,
      "fcm_dpo/q_t": 0.47280046343803406,
      "grad_norm": 149.2306365966797,
      "learning_rate": 4.552238805970149e-07,
      "logits/chosen": 0.11066489666700363,
      "logits/rejected": 0.07145200669765472,
      "logps/chosen": -63.376731872558594,
      "logps/ref_chosen": -62.751869201660156,
      "logps/ref_rejected": -78.93360900878906,
      "logps/rejected": -79.77182006835938,
      "loss": 1.3114,
      "margin_dpo/margin_mean": 0.2133486270904541,
      "margin_dpo/margin_std": 0.705337643623352,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": -0.4899425506591797,
      "KL/mean": -0.7270678877830505,
      "KL/rejected_KL_mean": -0.9641990661621094,
      "KL/std": 0.5222895741462708,
      "epoch": 0.09523809523809523,
      "fcm_dpo/beta": 0.5358837842941284,
      "fcm_dpo/delta": 0.14936861395835876,
      "fcm_dpo/margin": 0.4742552936077118,
      "fcm_dpo/q_t": 0.43969613313674927,
      "grad_norm": 142.19345092773438,
      "learning_rate": 4.626865671641791e-07,
      "logits/chosen": 0.17971235513687134,
      "logits/rejected": 0.15489208698272705,
      "logps/chosen": -61.005191802978516,
      "logps/ref_chosen": -60.51525115966797,
      "logps/ref_rejected": -85.11021423339844,
      "logps/rejected": -86.07441711425781,
      "loss": 1.1736,
      "margin_dpo/margin_mean": 0.4742545187473297,
      "margin_dpo/margin_std": 0.606643557548523,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": -0.6924915313720703,
      "KL/mean": -0.7450101375579834,
      "KL/rejected_KL_mean": -0.7975273132324219,
      "KL/std": 0.5659317970275879,
      "epoch": 0.09674981103552532,
      "fcm_dpo/beta": 0.5390844345092773,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10503339767456055,
      "fcm_dpo/q_t": 0.4859582185745239,
      "grad_norm": 144.57730102539062,
      "learning_rate": 4.701492537313433e-07,
      "logits/chosen": 0.08876290917396545,
      "logits/rejected": 0.06363459676504135,
      "logps/chosen": -51.89933776855469,
      "logps/ref_chosen": -51.20684814453125,
      "logps/ref_rejected": -66.93081665039062,
      "logps/rejected": -67.72834777832031,
      "loss": 1.3574,
      "margin_dpo/margin_mean": 0.10503333806991577,
      "margin_dpo/margin_std": 0.6080547571182251,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": -0.6500778198242188,
      "KL/mean": -0.9015808701515198,
      "KL/rejected_KL_mean": -1.1530838012695312,
      "KL/std": 0.6774485111236572,
      "epoch": 0.0982615268329554,
      "fcm_dpo/beta": 0.5490189790725708,
      "fcm_dpo/delta": 0.12744775414466858,
      "fcm_dpo/margin": 0.5030020475387573,
      "fcm_dpo/q_t": 0.4358825385570526,
      "grad_norm": 150.5640106201172,
      "learning_rate": 4.776119402985074e-07,
      "logits/chosen": 0.17361611127853394,
      "logits/rejected": 0.14435096085071564,
      "logps/chosen": -67.93877410888672,
      "logps/ref_chosen": -67.2886962890625,
      "logps/ref_rejected": -74.44281005859375,
      "logps/rejected": -75.59588623046875,
      "loss": 1.1807,
      "margin_dpo/margin_mean": 0.5030020475387573,
      "margin_dpo/margin_std": 0.8455530405044556,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": -0.7109127044677734,
      "KL/mean": -0.8587928414344788,
      "KL/rejected_KL_mean": -1.0066719055175781,
      "KL/std": 0.5832959413528442,
      "epoch": 0.09977324263038549,
      "fcm_dpo/beta": 0.5626637935638428,
      "fcm_dpo/delta": 0.0866028293967247,
      "fcm_dpo/margin": 0.2957577407360077,
      "fcm_dpo/q_t": 0.46131467819213867,
      "grad_norm": 155.40884399414062,
      "learning_rate": 4.850746268656717e-07,
      "logits/chosen": 0.1100161075592041,
      "logits/rejected": 0.08545216917991638,
      "logps/chosen": -71.4543228149414,
      "logps/ref_chosen": -70.743408203125,
      "logps/ref_rejected": -77.26499938964844,
      "logps/rejected": -78.27167510986328,
      "loss": 1.2694,
      "margin_dpo/margin_mean": 0.2957572937011719,
      "margin_dpo/margin_std": 0.7396960854530334,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": -0.6375637054443359,
      "KL/mean": -0.7534744739532471,
      "KL/rejected_KL_mean": -0.869384765625,
      "KL/std": 0.59464430809021,
      "epoch": 0.10128495842781557,
      "fcm_dpo/beta": 0.5626637935638428,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.23182055354118347,
      "fcm_dpo/q_t": 0.4688982665538788,
      "grad_norm": 154.8133544921875,
      "learning_rate": 4.925373134328357e-07,
      "logits/chosen": 0.08120022714138031,
      "logits/rejected": 0.025385765358805656,
      "logps/chosen": -61.24016571044922,
      "logps/ref_chosen": -60.60260009765625,
      "logps/ref_rejected": -75.22235870361328,
      "logps/rejected": -76.09174346923828,
      "loss": 1.3007,
      "margin_dpo/margin_mean": 0.23182040452957153,
      "margin_dpo/margin_std": 0.7166241407394409,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": -0.8958282470703125,
      "KL/mean": -1.0450494289398193,
      "KL/rejected_KL_mean": -1.19427490234375,
      "KL/std": 0.6658141613006592,
      "epoch": 0.10279667422524566,
      "fcm_dpo/beta": 0.5716010332107544,
      "fcm_dpo/delta": 0.0787949189543724,
      "fcm_dpo/margin": 0.29844844341278076,
      "fcm_dpo/q_t": 0.4608234167098999,
      "grad_norm": 169.2407989501953,
      "learning_rate": 5e-07,
      "logits/chosen": 0.06966448575258255,
      "logits/rejected": 0.03915044665336609,
      "logps/chosen": -78.4241943359375,
      "logps/ref_chosen": -77.52836608886719,
      "logps/ref_rejected": -93.17778015136719,
      "logps/rejected": -94.37205505371094,
      "loss": 1.2751,
      "margin_dpo/margin_mean": 0.2984488904476166,
      "margin_dpo/margin_std": 0.8083846569061279,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": -0.8310146331787109,
      "KL/mean": -1.0754246711730957,
      "KL/rejected_KL_mean": -1.3198318481445312,
      "KL/std": 0.694922685623169,
      "epoch": 0.10430839002267574,
      "fcm_dpo/beta": 0.5718780159950256,
      "fcm_dpo/delta": 0.004843501374125481,
      "fcm_dpo/margin": 0.4888237416744232,
      "fcm_dpo/q_t": 0.43570476770401,
      "grad_norm": 149.18028259277344,
      "learning_rate": 4.999965034812934e-07,
      "logits/chosen": 0.0899805799126625,
      "logits/rejected": 0.04690591245889664,
      "logps/chosen": -66.77406311035156,
      "logps/ref_chosen": -65.94305419921875,
      "logps/ref_rejected": -89.7735595703125,
      "logps/rejected": -91.09339141845703,
      "loss": 1.1842,
      "margin_dpo/margin_mean": 0.48882368206977844,
      "margin_dpo/margin_std": 0.8487541079521179,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": -0.8911018371582031,
      "KL/mean": -1.047347068786621,
      "KL/rejected_KL_mean": -1.2035884857177734,
      "KL/std": 0.647836446762085,
      "epoch": 0.10582010582010581,
      "fcm_dpo/beta": 0.5721549987792969,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.31248238682746887,
      "fcm_dpo/q_t": 0.45649653673171997,
      "grad_norm": 156.76638793945312,
      "learning_rate": 4.999860140229787e-07,
      "logits/chosen": 0.11240847408771515,
      "logits/rejected": 0.08975277841091156,
      "logps/chosen": -62.84901428222656,
      "logps/ref_chosen": -61.95791244506836,
      "logps/ref_rejected": -75.80945587158203,
      "logps/rejected": -77.01304626464844,
      "loss": 1.2716,
      "margin_dpo/margin_mean": 0.3124830424785614,
      "margin_dpo/margin_std": 0.8370497226715088,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": -1.1248493194580078,
      "KL/mean": -1.2039525508880615,
      "KL/rejected_KL_mean": -1.2830581665039062,
      "KL/std": 0.6881119012832642,
      "epoch": 0.1073318216175359,
      "fcm_dpo/beta": 0.5721549987792969,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15820838510990143,
      "fcm_dpo/q_t": 0.4769324064254761,
      "grad_norm": 169.25108337402344,
      "learning_rate": 4.999685319184688e-07,
      "logits/chosen": 0.06681202352046967,
      "logits/rejected": 0.05146068334579468,
      "logps/chosen": -64.47242736816406,
      "logps/ref_chosen": -63.34757995605469,
      "logps/ref_rejected": -67.49658203125,
      "logps/rejected": -68.7796401977539,
      "loss": 1.3643,
      "margin_dpo/margin_mean": 0.15820787847042084,
      "margin_dpo/margin_std": 0.9106104373931885,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": -0.882232666015625,
      "KL/mean": -1.1684211492538452,
      "KL/rejected_KL_mean": -1.4546089172363281,
      "KL/std": 0.7250270247459412,
      "epoch": 0.10884353741496598,
      "fcm_dpo/beta": 0.5770248174667358,
      "fcm_dpo/delta": 0.07216573506593704,
      "fcm_dpo/margin": 0.5723739862442017,
      "fcm_dpo/q_t": 0.42376360297203064,
      "grad_norm": 151.21566772460938,
      "learning_rate": 4.999440576567755e-07,
      "logits/chosen": 0.11811242997646332,
      "logits/rejected": 0.05374206230044365,
      "logps/chosen": -56.74153137207031,
      "logps/ref_chosen": -55.85929870605469,
      "logps/ref_rejected": -68.45423889160156,
      "logps/rejected": -69.90885162353516,
      "loss": 1.1417,
      "margin_dpo/margin_mean": 0.5723739862442017,
      "margin_dpo/margin_std": 0.860072672367096,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": -1.2896194458007812,
      "KL/mean": -1.3716013431549072,
      "KL/rejected_KL_mean": -1.4535808563232422,
      "KL/std": 0.8279663920402527,
      "epoch": 0.11035525321239607,
      "fcm_dpo/beta": 0.5804728269577026,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16396650671958923,
      "fcm_dpo/q_t": 0.4804548919200897,
      "grad_norm": 186.76443481445312,
      "learning_rate": 4.999125919224965e-07,
      "logits/chosen": 0.07135484367609024,
      "logits/rejected": 0.05760319530963898,
      "logps/chosen": -70.42842102050781,
      "logps/ref_chosen": -69.13880920410156,
      "logps/ref_rejected": -79.04586791992188,
      "logps/rejected": -80.49945068359375,
      "loss": 1.3723,
      "margin_dpo/margin_mean": 0.16396701335906982,
      "margin_dpo/margin_std": 0.9709917306900024,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": -0.9494285583496094,
      "KL/mean": -1.215151309967041,
      "KL/rejected_KL_mean": -1.4808769226074219,
      "KL/std": 0.6764031648635864,
      "epoch": 0.11186696900982615,
      "fcm_dpo/beta": 0.5794328451156616,
      "fcm_dpo/delta": -0.017948877066373825,
      "fcm_dpo/margin": 0.5314480066299438,
      "fcm_dpo/q_t": 0.4282793402671814,
      "grad_norm": 140.96795654296875,
      "learning_rate": 4.998741355957963e-07,
      "logits/chosen": 0.09334755688905716,
      "logits/rejected": 0.04302297160029411,
      "logps/chosen": -50.873165130615234,
      "logps/ref_chosen": -49.923736572265625,
      "logps/ref_rejected": -81.73213958740234,
      "logps/rejected": -83.2130126953125,
      "loss": 1.1643,
      "margin_dpo/margin_mean": 0.5314477682113647,
      "margin_dpo/margin_std": 0.8572825789451599,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": -1.0329856872558594,
      "KL/mean": -1.330980896949768,
      "KL/rejected_KL_mean": -1.628976821899414,
      "KL/std": 0.7908544540405273,
      "epoch": 0.11337868480725624,
      "fcm_dpo/beta": 0.5815718770027161,
      "fcm_dpo/delta": 0.05530213937163353,
      "fcm_dpo/margin": 0.5959901809692383,
      "fcm_dpo/q_t": 0.42089396715164185,
      "grad_norm": 126.52373504638672,
      "learning_rate": 4.998286897523808e-07,
      "logits/chosen": 0.11120344698429108,
      "logits/rejected": 0.07785911858081818,
      "logps/chosen": -47.10173797607422,
      "logps/ref_chosen": -46.06875228881836,
      "logps/ref_rejected": -66.1181411743164,
      "logps/rejected": -67.74711608886719,
      "loss": 1.1427,
      "margin_dpo/margin_mean": 0.5959901809692383,
      "margin_dpo/margin_std": 0.9563091993331909,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": -1.1540660858154297,
      "KL/mean": -1.2968565225601196,
      "KL/rejected_KL_mean": -1.4396495819091797,
      "KL/std": 0.832923173904419,
      "epoch": 0.11489040060468632,
      "fcm_dpo/beta": 0.5848255753517151,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.28558349609375,
      "fcm_dpo/q_t": 0.46233969926834106,
      "grad_norm": 163.6553497314453,
      "learning_rate": 4.997762556634679e-07,
      "logits/chosen": 0.12046054005622864,
      "logits/rejected": 0.07550361752510071,
      "logps/chosen": -55.21681594848633,
      "logps/ref_chosen": -54.06275177001953,
      "logps/ref_rejected": -74.87464141845703,
      "logps/rejected": -76.31429290771484,
      "loss": 1.2999,
      "margin_dpo/margin_mean": 0.28558364510536194,
      "margin_dpo/margin_std": 0.9496116638183594,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": -1.2467975616455078,
      "KL/mean": -1.4568101167678833,
      "KL/rejected_KL_mean": -1.6668205261230469,
      "KL/std": 0.7679809331893921,
      "epoch": 0.1164021164021164,
      "fcm_dpo/beta": 0.5857464075088501,
      "fcm_dpo/delta": 0.007866356521844864,
      "fcm_dpo/margin": 0.4200241267681122,
      "fcm_dpo/q_t": 0.44254666566848755,
      "grad_norm": 154.96896362304688,
      "learning_rate": 4.99716834795752e-07,
      "logits/chosen": 0.1474646031856537,
      "logits/rejected": 0.10533631592988968,
      "logps/chosen": -54.32289123535156,
      "logps/ref_chosen": -53.07609176635742,
      "logps/ref_rejected": -74.45601654052734,
      "logps/rejected": -76.12283325195312,
      "loss": 1.2175,
      "margin_dpo/margin_mean": 0.4200243055820465,
      "margin_dpo/margin_std": 0.8364007472991943,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": -1.2273178100585938,
      "KL/mean": -1.4201856851577759,
      "KL/rejected_KL_mean": -1.6130561828613281,
      "KL/std": 0.8160399198532104,
      "epoch": 0.11791383219954649,
      "fcm_dpo/beta": 0.5935271382331848,
      "fcm_dpo/delta": 0.06598014384508133,
      "fcm_dpo/margin": 0.38572752475738525,
      "fcm_dpo/q_t": 0.44638317823410034,
      "grad_norm": 169.8009796142578,
      "learning_rate": 4.996504288113623e-07,
      "logits/chosen": 0.0921347588300705,
      "logits/rejected": 0.07184503972530365,
      "logps/chosen": -68.95274353027344,
      "logps/ref_chosen": -67.72541809082031,
      "logps/ref_rejected": -79.03926849365234,
      "logps/rejected": -80.65232849121094,
      "loss": 1.2354,
      "margin_dpo/margin_mean": 0.38572707772254944,
      "margin_dpo/margin_std": 0.8717095851898193,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": -1.2739982604980469,
      "KL/mean": -1.5649783611297607,
      "KL/rejected_KL_mean": -1.85595703125,
      "KL/std": 0.8624995946884155,
      "epoch": 0.11942554799697656,
      "fcm_dpo/beta": 0.5987710952758789,
      "fcm_dpo/delta": 0.05334286019206047,
      "fcm_dpo/margin": 0.5819566249847412,
      "fcm_dpo/q_t": 0.42462414503097534,
      "grad_norm": 146.59410095214844,
      "learning_rate": 4.995770395678171e-07,
      "logits/chosen": 0.14114433526992798,
      "logits/rejected": 0.08149135112762451,
      "logps/chosen": -53.43464279174805,
      "logps/ref_chosen": -52.16064453125,
      "logps/ref_rejected": -83.31062316894531,
      "logps/rejected": -85.16658020019531,
      "loss": 1.1553,
      "margin_dpo/margin_mean": 0.5819563865661621,
      "margin_dpo/margin_std": 1.0548887252807617,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": -1.3399429321289062,
      "KL/mean": -1.5763564109802246,
      "KL/rejected_KL_mean": -1.8127670288085938,
      "KL/std": 0.846062421798706,
      "epoch": 0.12093726379440665,
      "fcm_dpo/beta": 0.6001569032669067,
      "fcm_dpo/delta": 0.004394013434648514,
      "fcm_dpo/margin": 0.47282540798187256,
      "fcm_dpo/q_t": 0.4358983635902405,
      "grad_norm": 165.1598663330078,
      "learning_rate": 4.994966691179711e-07,
      "logits/chosen": 0.13495443761348724,
      "logits/rejected": 0.07404427230358124,
      "logps/chosen": -62.75050354003906,
      "logps/ref_chosen": -61.410560607910156,
      "logps/ref_rejected": -78.66004943847656,
      "logps/rejected": -80.47281646728516,
      "loss": 1.2124,
      "margin_dpo/margin_mean": 0.47282546758651733,
      "margin_dpo/margin_std": 1.0053895711898804,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": -1.4206085205078125,
      "KL/mean": -1.7109118700027466,
      "KL/rejected_KL_mean": -2.001209259033203,
      "KL/std": 0.8699509501457214,
      "epoch": 0.12244897959183673,
      "fcm_dpo/beta": 0.6019116640090942,
      "fcm_dpo/delta": 0.05206025391817093,
      "fcm_dpo/margin": 0.5806019902229309,
      "fcm_dpo/q_t": 0.42219868302345276,
      "grad_norm": 154.65374755859375,
      "learning_rate": 4.994093197099587e-07,
      "logits/chosen": 0.10450653731822968,
      "logits/rejected": 0.07043684273958206,
      "logps/chosen": -65.2249755859375,
      "logps/ref_chosen": -63.80437088012695,
      "logps/ref_rejected": -79.3484115600586,
      "logps/rejected": -81.34962463378906,
      "loss": 1.1344,
      "margin_dpo/margin_mean": 0.5806014537811279,
      "margin_dpo/margin_std": 0.8735475540161133,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": -1.2224960327148438,
      "KL/mean": -1.6014692783355713,
      "KL/rejected_KL_mean": -1.9804420471191406,
      "KL/std": 0.8369277715682983,
      "epoch": 0.12396069538926682,
      "fcm_dpo/beta": 0.5999414920806885,
      "fcm_dpo/delta": -0.05762239545583725,
      "fcm_dpo/margin": 0.7579433917999268,
      "fcm_dpo/q_t": 0.3933570086956024,
      "grad_norm": 130.31703186035156,
      "learning_rate": 4.993149937871306e-07,
      "logits/chosen": 0.09142476320266724,
      "logits/rejected": 0.02778010442852974,
      "logps/chosen": -50.04039001464844,
      "logps/ref_chosen": -48.817893981933594,
      "logps/ref_rejected": -70.31497955322266,
      "logps/rejected": -72.29542541503906,
      "loss": 1.0341,
      "margin_dpo/margin_mean": 0.7579435110092163,
      "margin_dpo/margin_std": 0.7797366380691528,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": -1.3939533233642578,
      "KL/mean": -1.739598274230957,
      "KL/rejected_KL_mean": -2.085247039794922,
      "KL/std": 0.8719719648361206,
      "epoch": 0.1254724111866969,
      "fcm_dpo/beta": 0.5986208319664001,
      "fcm_dpo/delta": -0.014418380334973335,
      "fcm_dpo/margin": 0.6912956237792969,
      "fcm_dpo/q_t": 0.4073178172111511,
      "grad_norm": 141.97286987304688,
      "learning_rate": 4.992136939879856e-07,
      "logits/chosen": 0.15078996121883392,
      "logits/rejected": 0.10129296779632568,
      "logps/chosen": -58.54472732543945,
      "logps/ref_chosen": -57.15077209472656,
      "logps/ref_rejected": -75.1710205078125,
      "logps/rejected": -77.25627136230469,
      "loss": 1.0986,
      "margin_dpo/margin_mean": 0.6912951469421387,
      "margin_dpo/margin_std": 1.0455197095870972,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": -1.5992927551269531,
      "KL/mean": -1.8510921001434326,
      "KL/rejected_KL_mean": -2.1028976440429688,
      "KL/std": 0.9712103009223938,
      "epoch": 0.12698412698412698,
      "fcm_dpo/beta": 0.605029821395874,
      "fcm_dpo/delta": 0.09840921312570572,
      "fcm_dpo/margin": 0.5036056041717529,
      "fcm_dpo/q_t": 0.42928096652030945,
      "grad_norm": 179.0936737060547,
      "learning_rate": 4.991054231460969e-07,
      "logits/chosen": 0.12862388789653778,
      "logits/rejected": 0.08714120090007782,
      "logps/chosen": -66.37659454345703,
      "logps/ref_chosen": -64.77729797363281,
      "logps/ref_rejected": -84.71949768066406,
      "logps/rejected": -86.82239532470703,
      "loss": 1.1782,
      "margin_dpo/margin_mean": 0.503605842590332,
      "margin_dpo/margin_std": 0.9077202081680298,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": -1.3807716369628906,
      "KL/mean": -1.7777116298675537,
      "KL/rejected_KL_mean": -2.1746482849121094,
      "KL/std": 0.9808340072631836,
      "epoch": 0.12849584278155707,
      "fcm_dpo/beta": 0.6040663719177246,
      "fcm_dpo/delta": -0.08348983526229858,
      "fcm_dpo/margin": 0.793880820274353,
      "fcm_dpo/q_t": 0.39104384183883667,
      "grad_norm": 145.84339904785156,
      "learning_rate": 4.989901842900325e-07,
      "logits/chosen": 0.13091807067394257,
      "logits/rejected": 0.08705229312181473,
      "logps/chosen": -51.632469177246094,
      "logps/ref_chosen": -50.25169372558594,
      "logps/ref_rejected": -66.55439758300781,
      "logps/rejected": -68.72904205322266,
      "loss": 1.0561,
      "margin_dpo/margin_mean": 0.793880820274353,
      "margin_dpo/margin_std": 1.0297434329986572,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": -1.6255569458007812,
      "KL/mean": -1.9461750984191895,
      "KL/rejected_KL_mean": -2.266796112060547,
      "KL/std": 0.9864938259124756,
      "epoch": 0.13000755857898716,
      "fcm_dpo/beta": 0.600227952003479,
      "fcm_dpo/delta": 0.015688400715589523,
      "fcm_dpo/margin": 0.6412345767021179,
      "fcm_dpo/q_t": 0.4129602313041687,
      "grad_norm": 132.6361846923828,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": 0.1796223670244217,
      "logits/rejected": 0.16019511222839355,
      "logps/chosen": -62.354736328125,
      "logps/ref_chosen": -60.72917938232422,
      "logps/ref_rejected": -72.30961608886719,
      "logps/rejected": -74.57640838623047,
      "loss": 1.1228,
      "margin_dpo/margin_mean": 0.6412345170974731,
      "margin_dpo/margin_std": 1.0058636665344238,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": -1.7138938903808594,
      "KL/mean": -2.054884910583496,
      "KL/rejected_KL_mean": -2.3958740234375,
      "KL/std": 1.117079257965088,
      "epoch": 0.13151927437641722,
      "fcm_dpo/beta": 0.5979399085044861,
      "fcm_dpo/delta": -0.008664969354867935,
      "fcm_dpo/margin": 0.6819803714752197,
      "fcm_dpo/q_t": 0.40824219584465027,
      "grad_norm": 174.53688049316406,
      "learning_rate": 4.987388156241114e-07,
      "logits/chosen": 0.12052236497402191,
      "logits/rejected": 0.06240718811750412,
      "logps/chosen": -67.47186279296875,
      "logps/ref_chosen": -65.75796508789062,
      "logps/ref_rejected": -84.81159973144531,
      "logps/rejected": -87.20747375488281,
      "loss": 1.1556,
      "margin_dpo/margin_mean": 0.6819804906845093,
      "margin_dpo/margin_std": 1.2779656648635864,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": -1.6697559356689453,
      "KL/mean": -1.9886265993118286,
      "KL/rejected_KL_mean": -2.3074951171875,
      "KL/std": 1.0730674266815186,
      "epoch": 0.1330309901738473,
      "fcm_dpo/beta": 0.5980923175811768,
      "fcm_dpo/delta": 0.018423061817884445,
      "fcm_dpo/margin": 0.6377410888671875,
      "fcm_dpo/q_t": 0.41762399673461914,
      "grad_norm": 167.81472778320312,
      "learning_rate": 4.986026928455767e-07,
      "logits/chosen": 0.1676180362701416,
      "logits/rejected": 0.14126545190811157,
      "logps/chosen": -64.49378204345703,
      "logps/ref_chosen": -62.82402801513672,
      "logps/ref_rejected": -74.9607162475586,
      "logps/rejected": -77.2682113647461,
      "loss": 1.1798,
      "margin_dpo/margin_mean": 0.6377411484718323,
      "margin_dpo/margin_std": 1.292412519454956,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": -1.4845314025878906,
      "KL/mean": -1.8100913763046265,
      "KL/rejected_KL_mean": -2.1356544494628906,
      "KL/std": 1.0822101831436157,
      "epoch": 0.1345427059712774,
      "fcm_dpo/beta": 0.606133222579956,
      "fcm_dpo/delta": 0.005094341933727264,
      "fcm_dpo/margin": 0.6511192321777344,
      "fcm_dpo/q_t": 0.4151947796344757,
      "grad_norm": 160.05416870117188,
      "learning_rate": 4.984596161153135e-07,
      "logits/chosen": 0.21865971386432648,
      "logits/rejected": 0.13502703607082367,
      "logps/chosen": -42.67596435546875,
      "logps/ref_chosen": -41.191436767578125,
      "logps/ref_rejected": -85.44769287109375,
      "logps/rejected": -87.5833511352539,
      "loss": 1.1613,
      "margin_dpo/margin_mean": 0.6511195302009583,
      "margin_dpo/margin_std": 1.2548928260803223,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": -1.700155258178711,
      "KL/mean": -2.0352396965026855,
      "KL/rejected_KL_mean": -2.3703269958496094,
      "KL/std": 1.1013118028640747,
      "epoch": 0.1360544217687075,
      "fcm_dpo/beta": 0.6037954688072205,
      "fcm_dpo/delta": -0.004868221469223499,
      "fcm_dpo/margin": 0.6701726317405701,
      "fcm_dpo/q_t": 0.40914061665534973,
      "grad_norm": 153.80262756347656,
      "learning_rate": 4.983095894354857e-07,
      "logits/chosen": 0.11302797496318817,
      "logits/rejected": 0.0577833354473114,
      "logps/chosen": -58.28406524658203,
      "logps/ref_chosen": -56.58390808105469,
      "logps/ref_rejected": -86.86978149414062,
      "logps/rejected": -89.2401123046875,
      "loss": 1.1352,
      "margin_dpo/margin_mean": 0.6701725721359253,
      "margin_dpo/margin_std": 1.1711037158966064,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": -1.7016277313232422,
      "KL/mean": -2.0665078163146973,
      "KL/rejected_KL_mean": -2.431385040283203,
      "KL/std": 1.1532518863677979,
      "epoch": 0.13756613756613756,
      "fcm_dpo/beta": 0.6002589464187622,
      "fcm_dpo/delta": -0.03977450355887413,
      "fcm_dpo/margin": 0.7297595739364624,
      "fcm_dpo/q_t": 0.40282371640205383,
      "grad_norm": 137.2855682373047,
      "learning_rate": 4.98152617002662e-07,
      "logits/chosen": 0.1415039300918579,
      "logits/rejected": 0.09611248224973679,
      "logps/chosen": -54.08396911621094,
      "logps/ref_chosen": -52.38234329223633,
      "logps/ref_rejected": -72.17642211914062,
      "logps/rejected": -74.60780334472656,
      "loss": 1.1221,
      "margin_dpo/margin_mean": 0.7297590970993042,
      "margin_dpo/margin_std": 1.2456122636795044,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": -1.8641185760498047,
      "KL/mean": -2.1978578567504883,
      "KL/rejected_KL_mean": -2.5316009521484375,
      "KL/std": 1.1908236742019653,
      "epoch": 0.13907785336356765,
      "fcm_dpo/beta": 0.592298686504364,
      "fcm_dpo/delta": -0.09835251420736313,
      "fcm_dpo/margin": 0.6674777269363403,
      "fcm_dpo/q_t": 0.41342562437057495,
      "grad_norm": 139.87245178222656,
      "learning_rate": 4.979887032076988e-07,
      "logits/chosen": 0.17002242803573608,
      "logits/rejected": 0.12984851002693176,
      "logps/chosen": -54.87281799316406,
      "logps/ref_chosen": -53.00870132446289,
      "logps/ref_rejected": -79.77812957763672,
      "logps/rejected": -82.30973052978516,
      "loss": 1.1747,
      "margin_dpo/margin_mean": 0.6674777865409851,
      "margin_dpo/margin_std": 1.2964469194412231,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": -1.906198501586914,
      "KL/mean": -2.205145835876465,
      "KL/rejected_KL_mean": -2.5040931701660156,
      "KL/std": 1.2032487392425537,
      "epoch": 0.14058956916099774,
      "fcm_dpo/beta": 0.5906627178192139,
      "fcm_dpo/delta": 0.048564787954092026,
      "fcm_dpo/margin": 0.5978977680206299,
      "fcm_dpo/q_t": 0.42513328790664673,
      "grad_norm": 130.52691650390625,
      "learning_rate": 4.978178526356172e-07,
      "logits/chosen": 0.1441243588924408,
      "logits/rejected": 0.11534170806407928,
      "logps/chosen": -46.81325149536133,
      "logps/ref_chosen": -44.90705108642578,
      "logps/ref_rejected": -58.7879524230957,
      "logps/rejected": -61.29204559326172,
      "loss": 1.196,
      "margin_dpo/margin_mean": 0.5978972911834717,
      "margin_dpo/margin_std": 1.2997081279754639,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": -1.5615501403808594,
      "KL/mean": -1.9796390533447266,
      "KL/rejected_KL_mean": -2.3977317810058594,
      "KL/std": 1.2239587306976318,
      "epoch": 0.1421012849584278,
      "fcm_dpo/beta": 0.5802878737449646,
      "fcm_dpo/delta": -0.09112384915351868,
      "fcm_dpo/margin": 0.8361775279045105,
      "fcm_dpo/q_t": 0.39014649391174316,
      "grad_norm": 130.34103393554688,
      "learning_rate": 4.976400700654751e-07,
      "logits/chosen": 0.1767624169588089,
      "logits/rejected": 0.1383756548166275,
      "logps/chosen": -61.49932098388672,
      "logps/ref_chosen": -59.93777084350586,
      "logps/ref_rejected": -79.3138427734375,
      "logps/rejected": -81.7115707397461,
      "loss": 1.1246,
      "margin_dpo/margin_mean": 0.836177408695221,
      "margin_dpo/margin_std": 1.4772560596466064,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": -2.0556583404541016,
      "KL/mean": -2.4872384071350098,
      "KL/rejected_KL_mean": -2.9188156127929688,
      "KL/std": 1.155489206314087,
      "epoch": 0.1436130007558579,
      "fcm_dpo/beta": 0.5702659487724304,
      "fcm_dpo/delta": -0.09798791259527206,
      "fcm_dpo/margin": 0.8631570339202881,
      "fcm_dpo/q_t": 0.39131200313568115,
      "grad_norm": 139.64341735839844,
      "learning_rate": 4.974553604702332e-07,
      "logits/chosen": 0.10588128864765167,
      "logits/rejected": 0.04163233935832977,
      "logps/chosen": -62.22414779663086,
      "logps/ref_chosen": -60.168487548828125,
      "logps/ref_rejected": -90.73665618896484,
      "logps/rejected": -93.65547180175781,
      "loss": 1.0739,
      "margin_dpo/margin_mean": 0.8631570339202881,
      "margin_dpo/margin_std": 1.274303674697876,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": -1.9305763244628906,
      "KL/mean": -2.3768763542175293,
      "KL/rejected_KL_mean": -2.82318115234375,
      "KL/std": 1.2732088565826416,
      "epoch": 0.14512471655328799,
      "fcm_dpo/beta": 0.5618535280227661,
      "fcm_dpo/delta": -0.10678368806838989,
      "fcm_dpo/margin": 0.892608642578125,
      "fcm_dpo/q_t": 0.3913339376449585,
      "grad_norm": 128.86244201660156,
      "learning_rate": 4.972637290166157e-07,
      "logits/chosen": 0.14235463738441467,
      "logits/rejected": 0.09900492429733276,
      "logps/chosen": -62.5993537902832,
      "logps/ref_chosen": -60.66877746582031,
      "logps/ref_rejected": -88.30673217773438,
      "logps/rejected": -91.12991333007812,
      "loss": 1.0864,
      "margin_dpo/margin_mean": 0.892608106136322,
      "margin_dpo/margin_std": 1.4024059772491455,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": -2.237579345703125,
      "KL/mean": -2.5677237510681152,
      "KL/rejected_KL_mean": -2.897869110107422,
      "KL/std": 1.2186585664749146,
      "epoch": 0.14663643235071808,
      "fcm_dpo/beta": 0.5508826375007629,
      "fcm_dpo/delta": -0.06569742411375046,
      "fcm_dpo/margin": 0.660297155380249,
      "fcm_dpo/q_t": 0.42373794317245483,
      "grad_norm": 161.62107849121094,
      "learning_rate": 4.970651810649666e-07,
      "logits/chosen": 0.06538835167884827,
      "logits/rejected": 0.022402917966246605,
      "logps/chosen": -67.28170013427734,
      "logps/ref_chosen": -65.04412078857422,
      "logps/ref_rejected": -78.42092895507812,
      "logps/rejected": -81.31880187988281,
      "loss": 1.197,
      "margin_dpo/margin_mean": 0.660297155380249,
      "margin_dpo/margin_std": 1.4268206357955933,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": -1.9205188751220703,
      "KL/mean": -2.1965370178222656,
      "KL/rejected_KL_mean": -2.4725570678710938,
      "KL/std": 1.1956684589385986,
      "epoch": 0.14814814814814814,
      "fcm_dpo/beta": 0.5502942204475403,
      "fcm_dpo/delta": -0.005343480966985226,
      "fcm_dpo/margin": 0.5520362854003906,
      "fcm_dpo/q_t": 0.4313579797744751,
      "grad_norm": 150.3522491455078,
      "learning_rate": 4.968597221690985e-07,
      "logits/chosen": 0.164788618683815,
      "logits/rejected": 0.13764531910419464,
      "logps/chosen": -57.42375183105469,
      "logps/ref_chosen": -55.503231048583984,
      "logps/ref_rejected": -72.81553649902344,
      "logps/rejected": -75.28810119628906,
      "loss": 1.205,
      "margin_dpo/margin_mean": 0.5520361661911011,
      "margin_dpo/margin_std": 1.16847562789917,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": -1.9835891723632812,
      "KL/mean": -2.3774375915527344,
      "KL/rejected_KL_mean": -2.7712936401367188,
      "KL/std": 1.3897836208343506,
      "epoch": 0.14965986394557823,
      "fcm_dpo/beta": 0.5461075305938721,
      "fcm_dpo/delta": -0.03178960457444191,
      "fcm_dpo/margin": 0.7876995801925659,
      "fcm_dpo/q_t": 0.40874579548835754,
      "grad_norm": 160.76629638671875,
      "learning_rate": 4.966473580761389e-07,
      "logits/chosen": 0.17406132817268372,
      "logits/rejected": 0.1360493004322052,
      "logps/chosen": -60.559226989746094,
      "logps/ref_chosen": -58.57563781738281,
      "logps/ref_rejected": -78.693603515625,
      "logps/rejected": -81.46490478515625,
      "loss": 1.1611,
      "margin_dpo/margin_mean": 0.7876993417739868,
      "margin_dpo/margin_std": 1.554375410079956,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": -2.141376495361328,
      "KL/mean": -2.51546311378479,
      "KL/rejected_KL_mean": -2.8895492553710938,
      "KL/std": 1.2873187065124512,
      "epoch": 0.15117157974300832,
      "fcm_dpo/beta": 0.5401067733764648,
      "fcm_dpo/delta": -0.12405236810445786,
      "fcm_dpo/margin": 0.7481719255447388,
      "fcm_dpo/q_t": 0.41820228099823,
      "grad_norm": 167.9686279296875,
      "learning_rate": 4.964280947263676e-07,
      "logits/chosen": 0.16738124191761017,
      "logits/rejected": 0.15975362062454224,
      "logps/chosen": -81.72480773925781,
      "logps/ref_chosen": -79.58343505859375,
      "logps/ref_rejected": -92.152587890625,
      "logps/rejected": -95.04212951660156,
      "loss": 1.2225,
      "margin_dpo/margin_mean": 0.7481718063354492,
      "margin_dpo/margin_std": 1.7359800338745117,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": -1.9018211364746094,
      "KL/mean": -2.420198440551758,
      "KL/rejected_KL_mean": -2.9385757446289062,
      "KL/std": 1.3791840076446533,
      "epoch": 0.15268329554043839,
      "fcm_dpo/beta": 0.5242752432823181,
      "fcm_dpo/delta": -0.15156486630439758,
      "fcm_dpo/margin": 1.0367605686187744,
      "fcm_dpo/q_t": 0.38136833906173706,
      "grad_norm": 114.09092712402344,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": 0.14855097234249115,
      "logits/rejected": 0.11142145842313766,
      "logps/chosen": -54.2346076965332,
      "logps/ref_chosen": -52.332786560058594,
      "logps/ref_rejected": -69.55589294433594,
      "logps/rejected": -72.49447631835938,
      "loss": 1.0327,
      "margin_dpo/margin_mean": 1.0367603302001953,
      "margin_dpo/margin_std": 1.3791306018829346,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": -2.221982955932617,
      "KL/mean": -2.5972418785095215,
      "KL/rejected_KL_mean": -2.9725074768066406,
      "KL/std": 1.3701460361480713,
      "epoch": 0.15419501133786848,
      "fcm_dpo/beta": 0.5192157030105591,
      "fcm_dpo/delta": 0.01065264642238617,
      "fcm_dpo/margin": 0.7505237460136414,
      "fcm_dpo/q_t": 0.41866227984428406,
      "grad_norm": 141.7440948486328,
      "learning_rate": 4.959688949822748e-07,
      "logits/chosen": 0.09039057046175003,
      "logits/rejected": 0.05150360241532326,
      "logps/chosen": -66.96546936035156,
      "logps/ref_chosen": -64.74348449707031,
      "logps/ref_rejected": -69.06132507324219,
      "logps/rejected": -72.0338363647461,
      "loss": 1.2014,
      "margin_dpo/margin_mean": 0.7505236864089966,
      "margin_dpo/margin_std": 1.66679048538208,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": -2.160694122314453,
      "KL/mean": -2.570974349975586,
      "KL/rejected_KL_mean": -2.9812583923339844,
      "KL/std": 1.454613208770752,
      "epoch": 0.15570672713529857,
      "fcm_dpo/beta": 0.5179067850112915,
      "fcm_dpo/delta": -0.026149997487664223,
      "fcm_dpo/margin": 0.8205588459968567,
      "fcm_dpo/q_t": 0.4109882414340973,
      "grad_norm": 144.24627685546875,
      "learning_rate": 4.957289714327572e-07,
      "logits/chosen": 0.18639464676380157,
      "logits/rejected": 0.15367087721824646,
      "logps/chosen": -65.9973373413086,
      "logps/ref_chosen": -63.83664321899414,
      "logps/ref_rejected": -79.32362365722656,
      "logps/rejected": -82.30488586425781,
      "loss": 1.1471,
      "margin_dpo/margin_mean": 0.8205587863922119,
      "margin_dpo/margin_std": 1.54433274269104,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": -2.1406803131103516,
      "KL/mean": -2.5614709854125977,
      "KL/rejected_KL_mean": -2.982269287109375,
      "KL/std": 1.6537811756134033,
      "epoch": 0.15721844293272866,
      "fcm_dpo/beta": 0.5142132043838501,
      "fcm_dpo/delta": -0.03424425050616264,
      "fcm_dpo/margin": 0.8415879011154175,
      "fcm_dpo/q_t": 0.4116860628128052,
      "grad_norm": 154.5833282470703,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": 0.19082754850387573,
      "logits/rejected": 0.10447197407484055,
      "logps/chosen": -63.139888763427734,
      "logps/ref_chosen": -60.99920654296875,
      "logps/ref_rejected": -98.84645080566406,
      "logps/rejected": -101.82872009277344,
      "loss": 1.1778,
      "margin_dpo/margin_mean": 0.8415881395339966,
      "margin_dpo/margin_std": 1.7412712574005127,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": -2.194263458251953,
      "KL/mean": -2.5530097484588623,
      "KL/rejected_KL_mean": -2.9117507934570312,
      "KL/std": 1.4359549283981323,
      "epoch": 0.15873015873015872,
      "fcm_dpo/beta": 0.5169385671615601,
      "fcm_dpo/delta": 0.029685666784644127,
      "fcm_dpo/margin": 0.7174838781356812,
      "fcm_dpo/q_t": 0.4222378730773926,
      "grad_norm": 156.31329345703125,
      "learning_rate": 4.952285105344791e-07,
      "logits/chosen": 0.1434515416622162,
      "logits/rejected": 0.08881168067455292,
      "logps/chosen": -73.14453125,
      "logps/ref_chosen": -70.95027160644531,
      "logps/ref_rejected": -87.88340759277344,
      "logps/rejected": -90.795166015625,
      "loss": 1.2032,
      "margin_dpo/margin_mean": 0.7174830436706543,
      "margin_dpo/margin_std": 1.5990705490112305,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": -2.0911026000976562,
      "KL/mean": -2.471804618835449,
      "KL/rejected_KL_mean": -2.8525047302246094,
      "KL/std": 1.4120562076568604,
      "epoch": 0.1602418745275888,
      "fcm_dpo/beta": 0.51438307762146,
      "fcm_dpo/delta": 0.00849740020930767,
      "fcm_dpo/margin": 0.7614033818244934,
      "fcm_dpo/q_t": 0.41360223293304443,
      "grad_norm": 146.1759490966797,
      "learning_rate": 4.949679871846857e-07,
      "logits/chosen": 0.14227566123008728,
      "logits/rejected": 0.12906034290790558,
      "logps/chosen": -64.55043029785156,
      "logps/ref_chosen": -62.45933151245117,
      "logps/ref_rejected": -67.00595092773438,
      "logps/rejected": -69.85845184326172,
      "loss": 1.1924,
      "margin_dpo/margin_mean": 0.7614032030105591,
      "margin_dpo/margin_std": 1.6357064247131348,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": -2.3842124938964844,
      "KL/mean": -2.662627935409546,
      "KL/rejected_KL_mean": -2.9410476684570312,
      "KL/std": 1.5985708236694336,
      "epoch": 0.1617535903250189,
      "fcm_dpo/beta": 0.5118233561515808,
      "fcm_dpo/delta": -0.0434018038213253,
      "fcm_dpo/margin": 0.556837260723114,
      "fcm_dpo/q_t": 0.44089895486831665,
      "grad_norm": 181.79139709472656,
      "learning_rate": 4.947006115536947e-07,
      "logits/chosen": 0.07630196213722229,
      "logits/rejected": 0.0557682104408741,
      "logps/chosen": -78.22218322753906,
      "logps/ref_chosen": -75.83796691894531,
      "logps/ref_rejected": -87.74038696289062,
      "logps/rejected": -90.68142700195312,
      "loss": 1.3318,
      "margin_dpo/margin_mean": 0.5568374395370483,
      "margin_dpo/margin_std": 1.8654475212097168,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": -2.119457244873047,
      "KL/mean": -2.583726167678833,
      "KL/rejected_KL_mean": -3.0479888916015625,
      "KL/std": 1.2902718782424927,
      "epoch": 0.16326530612244897,
      "fcm_dpo/beta": 0.508414626121521,
      "fcm_dpo/delta": -0.07572700083255768,
      "fcm_dpo/margin": 0.9285260438919067,
      "fcm_dpo/q_t": 0.40096914768218994,
      "grad_norm": 136.07444763183594,
      "learning_rate": 4.944263911205772e-07,
      "logits/chosen": 0.10524410009384155,
      "logits/rejected": 0.07608610391616821,
      "logps/chosen": -70.5126953125,
      "logps/ref_chosen": -68.39323425292969,
      "logps/ref_rejected": -83.24267578125,
      "logps/rejected": -86.29066467285156,
      "loss": 1.1391,
      "margin_dpo/margin_mean": 0.9285261631011963,
      "margin_dpo/margin_std": 1.71048903465271,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": -2.061033248901367,
      "KL/mean": -2.6250224113464355,
      "KL/rejected_KL_mean": -3.189014434814453,
      "KL/std": 1.577345609664917,
      "epoch": 0.16477702191987906,
      "fcm_dpo/beta": 0.4920162558555603,
      "fcm_dpo/delta": -0.16408663988113403,
      "fcm_dpo/margin": 1.127977967262268,
      "fcm_dpo/q_t": 0.3859930634498596,
      "grad_norm": 118.14586639404297,
      "learning_rate": 4.941453335558681e-07,
      "logits/chosen": 0.1284589171409607,
      "logits/rejected": 0.07558364421129227,
      "logps/chosen": -57.58851623535156,
      "logps/ref_chosen": -55.52748107910156,
      "logps/ref_rejected": -83.55218505859375,
      "logps/rejected": -86.74120330810547,
      "loss": 1.0418,
      "margin_dpo/margin_mean": 1.127976894378662,
      "margin_dpo/margin_std": 1.6730471849441528,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": -2.3157196044921875,
      "KL/mean": -2.638578414916992,
      "KL/rejected_KL_mean": -2.961437225341797,
      "KL/std": 1.4730072021484375,
      "epoch": 0.16628873771730915,
      "fcm_dpo/beta": 0.4954935908317566,
      "fcm_dpo/delta": 0.08220823109149933,
      "fcm_dpo/margin": 0.6457212567329407,
      "fcm_dpo/q_t": 0.4308883547782898,
      "grad_norm": 154.20327758789062,
      "learning_rate": 4.938574467213517e-07,
      "logits/chosen": 0.09752906113862991,
      "logits/rejected": 0.10612943768501282,
      "logps/chosen": -83.4744644165039,
      "logps/ref_chosen": -81.15874481201172,
      "logps/ref_rejected": -72.56021118164062,
      "logps/rejected": -75.52165222167969,
      "loss": 1.239,
      "margin_dpo/margin_mean": 0.6457208395004272,
      "margin_dpo/margin_std": 1.617480754852295,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": -2.4843578338623047,
      "KL/mean": -2.9124999046325684,
      "KL/rejected_KL_mean": -3.3406448364257812,
      "KL/std": 1.544374942779541,
      "epoch": 0.16780045351473924,
      "fcm_dpo/beta": 0.49549514055252075,
      "fcm_dpo/delta": -0.025457965210080147,
      "fcm_dpo/margin": 0.8562860488891602,
      "fcm_dpo/q_t": 0.4100920557975769,
      "grad_norm": 131.57899475097656,
      "learning_rate": 4.935627386698418e-07,
      "logits/chosen": 0.21068426966667175,
      "logits/rejected": 0.1739131659269333,
      "logps/chosen": -54.84334182739258,
      "logps/ref_chosen": -52.358985900878906,
      "logps/ref_rejected": -77.06150817871094,
      "logps/rejected": -80.40214538574219,
      "loss": 1.1888,
      "margin_dpo/margin_mean": 0.8562856912612915,
      "margin_dpo/margin_std": 1.8077609539031982,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": -2.1196937561035156,
      "KL/mean": -2.637922763824463,
      "KL/rejected_KL_mean": -3.1561431884765625,
      "KL/std": 1.472923755645752,
      "epoch": 0.1693121693121693,
      "fcm_dpo/beta": 0.4850374460220337,
      "fcm_dpo/delta": -0.10829277336597443,
      "fcm_dpo/margin": 1.036455512046814,
      "fcm_dpo/q_t": 0.38861560821533203,
      "grad_norm": 134.61207580566406,
      "learning_rate": 4.932612176449559e-07,
      "logits/chosen": 0.13890337944030762,
      "logits/rejected": 0.07483598589897156,
      "logps/chosen": -65.13975524902344,
      "logps/ref_chosen": -63.02006530761719,
      "logps/ref_rejected": -111.36941528320312,
      "logps/rejected": -114.52555847167969,
      "loss": 1.0869,
      "margin_dpo/margin_mean": 1.0364547967910767,
      "margin_dpo/margin_std": 1.6114401817321777,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": -2.397075653076172,
      "KL/mean": -2.8203911781311035,
      "KL/rejected_KL_mean": -3.2437095642089844,
      "KL/std": 1.5809566974639893,
      "epoch": 0.1708238851095994,
      "fcm_dpo/beta": 0.4861123561859131,
      "fcm_dpo/delta": -0.013478599488735199,
      "fcm_dpo/margin": 0.8466259241104126,
      "fcm_dpo/q_t": 0.4091210961341858,
      "grad_norm": 145.45289611816406,
      "learning_rate": 4.929528920808854e-07,
      "logits/chosen": 0.11322137713432312,
      "logits/rejected": 0.07701212167739868,
      "logps/chosen": -58.20473861694336,
      "logps/ref_chosen": -55.80766296386719,
      "logps/ref_rejected": -69.84014129638672,
      "logps/rejected": -73.08384704589844,
      "loss": 1.1936,
      "margin_dpo/margin_mean": 0.846626341342926,
      "margin_dpo/margin_std": 1.7623178958892822,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": -2.044393539428711,
      "KL/mean": -2.769190788269043,
      "KL/rejected_KL_mean": -3.4939918518066406,
      "KL/std": 1.6656452417373657,
      "epoch": 0.17233560090702948,
      "fcm_dpo/beta": 0.45873937010765076,
      "fcm_dpo/delta": -0.2863787114620209,
      "fcm_dpo/margin": 1.449592113494873,
      "fcm_dpo/q_t": 0.35719749331474304,
      "grad_norm": 100.9645004272461,
      "learning_rate": 4.92637770602159e-07,
      "logits/chosen": 0.18304236233234406,
      "logits/rejected": 0.1241585835814476,
      "logps/chosen": -68.37716674804688,
      "logps/ref_chosen": -66.33277130126953,
      "logps/ref_rejected": -71.61489868164062,
      "logps/rejected": -75.10888671875,
      "loss": 0.9644,
      "margin_dpo/margin_mean": 1.4495927095413208,
      "margin_dpo/margin_std": 1.7315881252288818,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": -2.3844070434570312,
      "KL/mean": -2.925072193145752,
      "KL/rejected_KL_mean": -3.465738296508789,
      "KL/std": 1.5188902616500854,
      "epoch": 0.17384731670445955,
      "fcm_dpo/beta": 0.44873154163360596,
      "fcm_dpo/delta": -0.08957144618034363,
      "fcm_dpo/margin": 1.0813267230987549,
      "fcm_dpo/q_t": 0.3982967138290405,
      "grad_norm": 116.77565002441406,
      "learning_rate": 4.923158620234019e-07,
      "logits/chosen": 0.16674408316612244,
      "logits/rejected": 0.10691162198781967,
      "logps/chosen": -58.133445739746094,
      "logps/ref_chosen": -55.74903869628906,
      "logps/ref_rejected": -79.59849548339844,
      "logps/rejected": -83.06423950195312,
      "loss": 1.0894,
      "margin_dpo/margin_mean": 1.081327199935913,
      "margin_dpo/margin_std": 1.734327793121338,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": -2.2533435821533203,
      "KL/mean": -2.827744960784912,
      "KL/rejected_KL_mean": -3.402149200439453,
      "KL/std": 1.5526344776153564,
      "epoch": 0.17535903250188964,
      "fcm_dpo/beta": 0.44074195623397827,
      "fcm_dpo/delta": -0.1118515133857727,
      "fcm_dpo/margin": 1.1488080024719238,
      "fcm_dpo/q_t": 0.38711243867874146,
      "grad_norm": 97.64161682128906,
      "learning_rate": 4.91987175349089e-07,
      "logits/chosen": 0.16603578627109528,
      "logits/rejected": 0.10042545944452286,
      "logps/chosen": -51.618507385253906,
      "logps/ref_chosen": -49.36516571044922,
      "logps/ref_rejected": -72.84671020507812,
      "logps/rejected": -76.24885559082031,
      "loss": 1.0452,
      "margin_dpo/margin_mean": 1.1488078832626343,
      "margin_dpo/margin_std": 1.517305612564087,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": -2.1072540283203125,
      "KL/mean": -2.5920538902282715,
      "KL/rejected_KL_mean": -3.076854705810547,
      "KL/std": 1.4733943939208984,
      "epoch": 0.17687074829931973,
      "fcm_dpo/beta": 0.43148428201675415,
      "fcm_dpo/delta": -0.021056801080703735,
      "fcm_dpo/margin": 0.9695932865142822,
      "fcm_dpo/q_t": 0.4066160023212433,
      "grad_norm": 107.48424530029297,
      "learning_rate": 4.916517197732933e-07,
      "logits/chosen": 0.16764116287231445,
      "logits/rejected": 0.132475346326828,
      "logps/chosen": -59.818153381347656,
      "logps/ref_chosen": -57.710899353027344,
      "logps/ref_rejected": -69.77253723144531,
      "logps/rejected": -72.84939575195312,
      "loss": 1.1438,
      "margin_dpo/margin_mean": 0.9695931673049927,
      "margin_dpo/margin_std": 1.717713475227356,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": -2.0526790618896484,
      "KL/mean": -2.708968162536621,
      "KL/rejected_KL_mean": -3.365253448486328,
      "KL/std": 1.5411814451217651,
      "epoch": 0.17838246409674982,
      "fcm_dpo/beta": 0.4221842288970947,
      "fcm_dpo/delta": -0.16465967893600464,
      "fcm_dpo/margin": 1.3125801086425781,
      "fcm_dpo/q_t": 0.3760074973106384,
      "grad_norm": 100.54457092285156,
      "learning_rate": 4.913095046794281e-07,
      "logits/chosen": 0.23767630755901337,
      "logits/rejected": 0.1974300742149353,
      "logps/chosen": -54.53257751464844,
      "logps/ref_chosen": -52.479896545410156,
      "logps/ref_rejected": -81.359130859375,
      "logps/rejected": -84.72438049316406,
      "loss": 1.0228,
      "margin_dpo/margin_mean": 1.3125793933868408,
      "margin_dpo/margin_std": 1.6644185781478882,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": -2.659524917602539,
      "KL/mean": -3.153452157974243,
      "KL/rejected_KL_mean": -3.6473846435546875,
      "KL/std": 1.64215087890625,
      "epoch": 0.17989417989417988,
      "fcm_dpo/beta": 0.4208963215351105,
      "fcm_dpo/delta": -0.016559874638915062,
      "fcm_dpo/margin": 0.9878571629524231,
      "fcm_dpo/q_t": 0.4095316529273987,
      "grad_norm": 108.7704086303711,
      "learning_rate": 4.909605396399855e-07,
      "logits/chosen": 0.1396723985671997,
      "logits/rejected": 0.10316324234008789,
      "logps/chosen": -64.01719665527344,
      "logps/ref_chosen": -61.35767364501953,
      "logps/ref_rejected": -75.71510314941406,
      "logps/rejected": -79.36248779296875,
      "loss": 1.1536,
      "margin_dpo/margin_mean": 0.9878574013710022,
      "margin_dpo/margin_std": 1.850081205368042,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": -2.236845016479492,
      "KL/mean": -2.9230434894561768,
      "KL/rejected_KL_mean": -3.6092453002929688,
      "KL/std": 1.6300339698791504,
      "epoch": 0.18140589569160998,
      "fcm_dpo/beta": 0.4087931215763092,
      "fcm_dpo/delta": -0.17068202793598175,
      "fcm_dpo/margin": 1.372398853302002,
      "fcm_dpo/q_t": 0.37656670808792114,
      "grad_norm": 95.69251251220703,
      "learning_rate": 4.906048344162676e-07,
      "logits/chosen": 0.15590906143188477,
      "logits/rejected": 0.09781802445650101,
      "logps/chosen": -62.144412994384766,
      "logps/ref_chosen": -59.907569885253906,
      "logps/ref_rejected": -79.6910629272461,
      "logps/rejected": -83.30030822753906,
      "loss": 1.0054,
      "margin_dpo/margin_mean": 1.3723986148834229,
      "margin_dpo/margin_std": 1.6863455772399902,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": -2.52587890625,
      "KL/mean": -3.0839738845825195,
      "KL/rejected_KL_mean": -3.642070770263672,
      "KL/std": 1.6643069982528687,
      "epoch": 0.18291761148904007,
      "fcm_dpo/beta": 0.40134647488594055,
      "fcm_dpo/delta": -0.050486352294683456,
      "fcm_dpo/margin": 1.1161900758743286,
      "fcm_dpo/q_t": 0.4011920094490051,
      "grad_norm": 92.25079345703125,
      "learning_rate": 4.902423989581143e-07,
      "logits/chosen": 0.22740596532821655,
      "logits/rejected": 0.14705073833465576,
      "logps/chosen": -58.19192123413086,
      "logps/ref_chosen": -55.66604232788086,
      "logps/ref_rejected": -101.56233978271484,
      "logps/rejected": -105.20440673828125,
      "loss": 1.0831,
      "margin_dpo/margin_mean": 1.1161892414093018,
      "margin_dpo/margin_std": 1.642409086227417,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": -2.5778884887695312,
      "KL/mean": -3.28173828125,
      "KL/rejected_KL_mean": -3.9855918884277344,
      "KL/std": 1.813812494277954,
      "epoch": 0.18442932728647016,
      "fcm_dpo/beta": 0.3933998644351959,
      "fcm_dpo/delta": -0.16248536109924316,
      "fcm_dpo/margin": 1.4077017307281494,
      "fcm_dpo/q_t": 0.38090598583221436,
      "grad_norm": 104.64237213134766,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": 0.16349929571151733,
      "logits/rejected": 0.12945935130119324,
      "logps/chosen": -65.91226196289062,
      "logps/ref_chosen": -63.334373474121094,
      "logps/ref_rejected": -73.67523193359375,
      "logps/rejected": -77.66082763671875,
      "loss": 1.0353,
      "margin_dpo/margin_mean": 1.4077012538909912,
      "margin_dpo/margin_std": 1.9491944313049316,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": -2.3702030181884766,
      "KL/mean": -2.9709951877593994,
      "KL/rejected_KL_mean": -3.571788787841797,
      "KL/std": 1.6864802837371826,
      "epoch": 0.18594104308390022,
      "fcm_dpo/beta": 0.387717604637146,
      "fcm_dpo/delta": -0.0699365884065628,
      "fcm_dpo/margin": 1.2015814781188965,
      "fcm_dpo/q_t": 0.39549094438552856,
      "grad_norm": 97.17578125,
      "learning_rate": 4.894973780788722e-07,
      "logits/chosen": 0.17901018261909485,
      "logits/rejected": 0.1387392282485962,
      "logps/chosen": -59.268943786621094,
      "logps/ref_chosen": -56.89874267578125,
      "logps/ref_rejected": -78.97028350830078,
      "logps/rejected": -82.54206848144531,
      "loss": 1.1029,
      "margin_dpo/margin_mean": 1.2015812397003174,
      "margin_dpo/margin_std": 1.8984622955322266,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": -2.634672164916992,
      "KL/mean": -3.4132986068725586,
      "KL/rejected_KL_mean": -4.191928863525391,
      "KL/std": 1.7580922842025757,
      "epoch": 0.1874527588813303,
      "fcm_dpo/beta": 0.37133079767227173,
      "fcm_dpo/delta": -0.18995118141174316,
      "fcm_dpo/margin": 1.5572537183761597,
      "fcm_dpo/q_t": 0.3700428009033203,
      "grad_norm": 81.3668212890625,
      "learning_rate": 4.89114813497619e-07,
      "logits/chosen": 0.19634616374969482,
      "logits/rejected": 0.13928548991680145,
      "logps/chosen": -59.750755310058594,
      "logps/ref_chosen": -57.116085052490234,
      "logps/ref_rejected": -87.93074035644531,
      "logps/rejected": -92.12267303466797,
      "loss": 0.9903,
      "margin_dpo/margin_mean": 1.5572538375854492,
      "margin_dpo/margin_std": 1.7779855728149414,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": -2.818014144897461,
      "KL/mean": -3.4605140686035156,
      "KL/rejected_KL_mean": -4.103008270263672,
      "KL/std": 1.9742536544799805,
      "epoch": 0.1889644746787604,
      "fcm_dpo/beta": 0.3633834719657898,
      "fcm_dpo/delta": -0.07090410590171814,
      "fcm_dpo/margin": 1.2849962711334229,
      "fcm_dpo/q_t": 0.3981458246707916,
      "grad_norm": 94.46810150146484,
      "learning_rate": 4.887255603610184e-07,
      "logits/chosen": 0.19845634698867798,
      "logits/rejected": 0.14305856823921204,
      "logps/chosen": -68.5241928100586,
      "logps/ref_chosen": -65.7061767578125,
      "logps/ref_rejected": -91.72711944580078,
      "logps/rejected": -95.83012390136719,
      "loss": 1.086,
      "margin_dpo/margin_mean": 1.2849963903427124,
      "margin_dpo/margin_std": 1.9527506828308105,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": -2.5849876403808594,
      "KL/mean": -3.1301674842834473,
      "KL/rejected_KL_mean": -3.6753482818603516,
      "KL/std": 2.1857380867004395,
      "epoch": 0.19047619047619047,
      "fcm_dpo/beta": 0.36337125301361084,
      "fcm_dpo/delta": 0.003923341631889343,
      "fcm_dpo/margin": 1.0903596878051758,
      "fcm_dpo/q_t": 0.41889941692352295,
      "grad_norm": 88.6172103881836,
      "learning_rate": 4.883296295573176e-07,
      "logits/chosen": 0.037295181304216385,
      "logits/rejected": 0.03130710870027542,
      "logps/chosen": -70.76107788085938,
      "logps/ref_chosen": -68.17608642578125,
      "logps/ref_rejected": -65.1175537109375,
      "logps/rejected": -68.79290008544922,
      "loss": 1.1801,
      "margin_dpo/margin_mean": 1.0903599262237549,
      "margin_dpo/margin_std": 2.366217851638794,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": -2.809690475463867,
      "KL/mean": -3.563180923461914,
      "KL/rejected_KL_mean": -4.316673278808594,
      "KL/std": 1.7346203327178955,
      "epoch": 0.19198790627362056,
      "fcm_dpo/beta": 0.3538532853126526,
      "fcm_dpo/delta": -0.1426788717508316,
      "fcm_dpo/margin": 1.5069831609725952,
      "fcm_dpo/q_t": 0.37896767258644104,
      "grad_norm": 83.89849090576172,
      "learning_rate": 4.87927032161552e-07,
      "logits/chosen": 0.13445480167865753,
      "logits/rejected": 0.10488015413284302,
      "logps/chosen": -64.68992614746094,
      "logps/ref_chosen": -61.88023376464844,
      "logps/ref_rejected": -68.46012878417969,
      "logps/rejected": -72.77679443359375,
      "loss": 1.0118,
      "margin_dpo/margin_mean": 1.5069829225540161,
      "margin_dpo/margin_std": 1.7243682146072388,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": -2.9976940155029297,
      "KL/mean": -3.6348915100097656,
      "KL/rejected_KL_mean": -4.272090911865234,
      "KL/std": 2.0820395946502686,
      "epoch": 0.19349962207105065,
      "fcm_dpo/beta": 0.35042160749435425,
      "fcm_dpo/delta": -0.04892526939511299,
      "fcm_dpo/margin": 1.274397850036621,
      "fcm_dpo/q_t": 0.404270738363266,
      "grad_norm": 92.07938385009766,
      "learning_rate": 4.875177794352363e-07,
      "logits/chosen": 0.1919756680727005,
      "logits/rejected": 0.13632725179195404,
      "logps/chosen": -69.70668029785156,
      "logps/ref_chosen": -66.708984375,
      "logps/ref_rejected": -94.97969055175781,
      "logps/rejected": -99.25178527832031,
      "loss": 1.1419,
      "margin_dpo/margin_mean": 1.274397850036621,
      "margin_dpo/margin_std": 2.3580269813537598,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": -3.179830551147461,
      "KL/mean": -3.748736619949341,
      "KL/rejected_KL_mean": -4.317646026611328,
      "KL/std": 1.8753046989440918,
      "epoch": 0.19501133786848074,
      "fcm_dpo/beta": 0.3501220941543579,
      "fcm_dpo/delta": 0.0016883653588593006,
      "fcm_dpo/margin": 1.1378146409988403,
      "fcm_dpo/q_t": 0.41414761543273926,
      "grad_norm": 96.41793060302734,
      "learning_rate": 4.871018828260491e-07,
      "logits/chosen": 0.14986222982406616,
      "logits/rejected": 0.1429169774055481,
      "logps/chosen": -68.51866149902344,
      "logps/ref_chosen": -65.33882904052734,
      "logps/ref_rejected": -68.06109619140625,
      "logps/rejected": -72.37873840332031,
      "loss": 1.1466,
      "margin_dpo/margin_mean": 1.137814998626709,
      "margin_dpo/margin_std": 2.0920934677124023,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": -3.042226791381836,
      "KL/mean": -3.6935412883758545,
      "KL/rejected_KL_mean": -4.344856262207031,
      "KL/std": 1.837762713432312,
      "epoch": 0.1965230536659108,
      "fcm_dpo/beta": 0.34358179569244385,
      "fcm_dpo/delta": -0.053163111209869385,
      "fcm_dpo/margin": 1.302627682685852,
      "fcm_dpo/q_t": 0.3996545076370239,
      "grad_norm": 98.2292251586914,
      "learning_rate": 4.866793539675126e-07,
      "logits/chosen": 0.11451365798711777,
      "logits/rejected": 0.06797914952039719,
      "logps/chosen": -61.702972412109375,
      "logps/ref_chosen": -58.660743713378906,
      "logps/ref_rejected": -79.24510192871094,
      "logps/rejected": -83.58995819091797,
      "loss": 1.08,
      "margin_dpo/margin_mean": 1.3026278018951416,
      "margin_dpo/margin_std": 1.7226202487945557,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": -2.923948287963867,
      "KL/mean": -3.7199363708496094,
      "KL/rejected_KL_mean": -4.515926361083984,
      "KL/std": 2.1819896697998047,
      "epoch": 0.1980347694633409,
      "fcm_dpo/beta": 0.3391422629356384,
      "fcm_dpo/delta": -0.14788037538528442,
      "fcm_dpo/margin": 1.5919833183288574,
      "fcm_dpo/q_t": 0.3828786015510559,
      "grad_norm": 81.14729309082031,
      "learning_rate": 4.86250204678667e-07,
      "logits/chosen": 0.14033398032188416,
      "logits/rejected": 0.0799434557557106,
      "logps/chosen": -55.43848419189453,
      "logps/ref_chosen": -52.51453399658203,
      "logps/ref_rejected": -85.18299865722656,
      "logps/rejected": -89.69892883300781,
      "loss": 1.0619,
      "margin_dpo/margin_mean": 1.5919833183288574,
      "margin_dpo/margin_std": 2.36844539642334,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": -3.237884521484375,
      "KL/mean": -3.9422736167907715,
      "KL/rejected_KL_mean": -4.646657943725586,
      "KL/std": 2.13301157951355,
      "epoch": 0.19954648526077098,
      "fcm_dpo/beta": 0.3328793942928314,
      "fcm_dpo/delta": -0.07235664129257202,
      "fcm_dpo/margin": 1.4087742567062378,
      "fcm_dpo/q_t": 0.3950890004634857,
      "grad_norm": 85.98615264892578,
      "learning_rate": 4.858144469637408e-07,
      "logits/chosen": 0.2111670970916748,
      "logits/rejected": 0.17998561263084412,
      "logps/chosen": -68.92301940917969,
      "logps/ref_chosen": -65.68513488769531,
      "logps/ref_rejected": -69.54120635986328,
      "logps/rejected": -74.1878662109375,
      "loss": 1.1022,
      "margin_dpo/margin_mean": 1.4087748527526855,
      "margin_dpo/margin_std": 2.288146495819092,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": -3.232057571411133,
      "KL/mean": -3.864650249481201,
      "KL/rejected_KL_mean": -4.497241973876953,
      "KL/std": 2.1068387031555176,
      "epoch": 0.20105820105820105,
      "fcm_dpo/beta": 0.3309246897697449,
      "fcm_dpo/delta": -0.019490830600261688,
      "fcm_dpo/margin": 1.2651793956756592,
      "fcm_dpo/q_t": 0.4062243402004242,
      "grad_norm": 88.84878540039062,
      "learning_rate": 4.853720930118138e-07,
      "logits/chosen": 0.12917156517505646,
      "logits/rejected": 0.11973883211612701,
      "logps/chosen": -66.83016967773438,
      "logps/ref_chosen": -63.598114013671875,
      "logps/ref_rejected": -73.72798156738281,
      "logps/rejected": -78.2252197265625,
      "loss": 1.1271,
      "margin_dpo/margin_mean": 1.2651795148849487,
      "margin_dpo/margin_std": 2.1557090282440186,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": -3.0589447021484375,
      "KL/mean": -3.983780860900879,
      "KL/rejected_KL_mean": -4.908611297607422,
      "KL/std": 2.3829293251037598,
      "epoch": 0.20256991685563114,
      "fcm_dpo/beta": 0.31975215673446655,
      "fcm_dpo/delta": -0.20411178469657898,
      "fcm_dpo/margin": 1.8496692180633545,
      "fcm_dpo/q_t": 0.3709458112716675,
      "grad_norm": 72.68990325927734,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": 0.22880001366138458,
      "logits/rejected": 0.17560896277427673,
      "logps/chosen": -56.853515625,
      "logps/ref_chosen": -53.79457092285156,
      "logps/ref_rejected": -74.16741943359375,
      "logps/rejected": -79.07603454589844,
      "loss": 0.9993,
      "margin_dpo/margin_mean": 1.8496696949005127,
      "margin_dpo/margin_std": 2.3175394535064697,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": -3.0780696868896484,
      "KL/mean": -3.7185373306274414,
      "KL/rejected_KL_mean": -4.3590087890625,
      "KL/std": 2.241361141204834,
      "epoch": 0.20408163265306123,
      "fcm_dpo/beta": 0.3168698251247406,
      "fcm_dpo/delta": -0.006139796227216721,
      "fcm_dpo/margin": 1.2809354066848755,
      "fcm_dpo/q_t": 0.4119170606136322,
      "grad_norm": 72.32381439208984,
      "learning_rate": 4.844676460754862e-07,
      "logits/chosen": 0.1814204305410385,
      "logits/rejected": 0.14914453029632568,
      "logps/chosen": -52.51914596557617,
      "logps/ref_chosen": -49.441078186035156,
      "logps/ref_rejected": -65.96878051757812,
      "logps/rejected": -70.32778930664062,
      "loss": 1.1448,
      "margin_dpo/margin_mean": 1.280935287475586,
      "margin_dpo/margin_std": 2.3456642627716064,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": -3.6158905029296875,
      "KL/mean": -4.410771369934082,
      "KL/rejected_KL_mean": -5.205650329589844,
      "KL/std": 2.502413749694824,
      "epoch": 0.20559334845049132,
      "fcm_dpo/beta": 0.3120737373828888,
      "fcm_dpo/delta": -0.10114330053329468,
      "fcm_dpo/margin": 1.589766025543213,
      "fcm_dpo/q_t": 0.40253955125808716,
      "grad_norm": 91.72471618652344,
      "learning_rate": 4.840055783904106e-07,
      "logits/chosen": 0.17749547958374023,
      "logits/rejected": 0.10814127326011658,
      "logps/chosen": -70.37515258789062,
      "logps/ref_chosen": -66.75926208496094,
      "logps/ref_rejected": -94.61787414550781,
      "logps/rejected": -99.82352447509766,
      "loss": 1.1594,
      "margin_dpo/margin_mean": 1.5897669792175293,
      "margin_dpo/margin_std": 3.2202930450439453,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": -3.4252357482910156,
      "KL/mean": -4.258798599243164,
      "KL/rejected_KL_mean": -5.0923614501953125,
      "KL/std": 2.2802345752716064,
      "epoch": 0.20710506424792138,
      "fcm_dpo/beta": 0.3057171106338501,
      "fcm_dpo/delta": -0.11541862785816193,
      "fcm_dpo/margin": 1.6671223640441895,
      "fcm_dpo/q_t": 0.3904217481613159,
      "grad_norm": 70.61022186279297,
      "learning_rate": 4.835369650662767e-07,
      "logits/chosen": 0.17331115901470184,
      "logits/rejected": 0.14633190631866455,
      "logps/chosen": -60.20903396606445,
      "logps/ref_chosen": -56.78379821777344,
      "logps/ref_rejected": -69.89952087402344,
      "logps/rejected": -74.99188232421875,
      "loss": 1.0777,
      "margin_dpo/margin_mean": 1.6671226024627686,
      "margin_dpo/margin_std": 2.5936641693115234,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": -4.005002975463867,
      "KL/mean": -4.6617021560668945,
      "KL/rejected_KL_mean": -5.318401336669922,
      "KL/std": 2.3480114936828613,
      "epoch": 0.20861678004535147,
      "fcm_dpo/beta": 0.3033827841281891,
      "fcm_dpo/delta": 0.001601765281520784,
      "fcm_dpo/margin": 1.3133952617645264,
      "fcm_dpo/q_t": 0.411385178565979,
      "grad_norm": 76.18244934082031,
      "learning_rate": 4.830618192112065e-07,
      "logits/chosen": 0.16729283332824707,
      "logits/rejected": 0.13266587257385254,
      "logps/chosen": -62.77101516723633,
      "logps/ref_chosen": -58.766014099121094,
      "logps/ref_rejected": -68.12371826171875,
      "logps/rejected": -73.44212341308594,
      "loss": 1.1543,
      "margin_dpo/margin_mean": 1.3133950233459473,
      "margin_dpo/margin_std": 2.4506936073303223,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": -4.019571304321289,
      "KL/mean": -4.849878311157227,
      "KL/rejected_KL_mean": -5.680183410644531,
      "KL/std": 2.256826162338257,
      "epoch": 0.21012849584278157,
      "fcm_dpo/beta": 0.3009493350982666,
      "fcm_dpo/delta": -0.1053197830915451,
      "fcm_dpo/margin": 1.6606104373931885,
      "fcm_dpo/q_t": 0.38920527696609497,
      "grad_norm": 78.60396575927734,
      "learning_rate": 4.825801541160509e-07,
      "logits/chosen": 0.13528969883918762,
      "logits/rejected": 0.10798656940460205,
      "logps/chosen": -75.24516296386719,
      "logps/ref_chosen": -71.2255859375,
      "logps/ref_rejected": -82.1834716796875,
      "logps/rejected": -87.86365509033203,
      "loss": 1.053,
      "margin_dpo/margin_mean": 1.6606101989746094,
      "margin_dpo/margin_std": 2.2152228355407715,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": -3.701915740966797,
      "KL/mean": -4.768045902252197,
      "KL/rejected_KL_mean": -5.834178924560547,
      "KL/std": 3.028330087661743,
      "epoch": 0.21164021164021163,
      "fcm_dpo/beta": 0.2873826324939728,
      "fcm_dpo/delta": -0.2266281247138977,
      "fcm_dpo/margin": 2.132258892059326,
      "fcm_dpo/q_t": 0.36785006523132324,
      "grad_norm": 80.54397583007812,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 0.11437252908945084,
      "logits/rejected": 0.07303556054830551,
      "logps/chosen": -66.97957611083984,
      "logps/ref_chosen": -63.27766418457031,
      "logps/ref_rejected": -83.30647277832031,
      "logps/rejected": -89.14065551757812,
      "loss": 1.0582,
      "margin_dpo/margin_mean": 2.132258653640747,
      "margin_dpo/margin_std": 3.2923696041107178,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": -3.9139461517333984,
      "KL/mean": -4.95443058013916,
      "KL/rejected_KL_mean": -5.9949188232421875,
      "KL/std": 2.543238401412964,
      "epoch": 0.21315192743764172,
      "fcm_dpo/beta": 0.27460581064224243,
      "fcm_dpo/delta": -0.18379811942577362,
      "fcm_dpo/margin": 2.0809688568115234,
      "fcm_dpo/q_t": 0.3807663321495056,
      "grad_norm": 71.58379364013672,
      "learning_rate": 4.815973202802966e-07,
      "logits/chosen": 0.17417730391025543,
      "logits/rejected": 0.1330379694700241,
      "logps/chosen": -65.68071746826172,
      "logps/ref_chosen": -61.76676940917969,
      "logps/ref_rejected": -88.60601806640625,
      "logps/rejected": -94.60093688964844,
      "loss": 1.0527,
      "margin_dpo/margin_mean": 2.0809690952301025,
      "margin_dpo/margin_std": 3.0956361293792725,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": -3.9850540161132812,
      "KL/mean": -4.748014450073242,
      "KL/rejected_KL_mean": -5.5109710693359375,
      "KL/std": 2.450737714767456,
      "epoch": 0.2146636432350718,
      "fcm_dpo/beta": 0.2735764980316162,
      "fcm_dpo/delta": -0.01827201247215271,
      "fcm_dpo/margin": 1.525919795036316,
      "fcm_dpo/q_t": 0.4075871706008911,
      "grad_norm": 69.94967651367188,
      "learning_rate": 4.810961790316729e-07,
      "logits/chosen": 0.19211237132549286,
      "logits/rejected": 0.16759377717971802,
      "logps/chosen": -69.25982666015625,
      "logps/ref_chosen": -65.2747802734375,
      "logps/ref_rejected": -81.1378173828125,
      "logps/rejected": -86.64878845214844,
      "loss": 1.1184,
      "margin_dpo/margin_mean": 1.5259199142456055,
      "margin_dpo/margin_std": 2.500253200531006,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": -4.126497268676758,
      "KL/mean": -4.844476699829102,
      "KL/rejected_KL_mean": -5.5624542236328125,
      "KL/std": 2.646272659301758,
      "epoch": 0.2161753590325019,
      "fcm_dpo/beta": 0.2723914384841919,
      "fcm_dpo/delta": 0.009103547781705856,
      "fcm_dpo/margin": 1.4359560012817383,
      "fcm_dpo/q_t": 0.4125630259513855,
      "grad_norm": 85.05684661865234,
      "learning_rate": 4.805885735261454e-07,
      "logits/chosen": 0.20585831999778748,
      "logits/rejected": 0.18943452835083008,
      "logps/chosen": -66.74432373046875,
      "logps/ref_chosen": -62.617828369140625,
      "logps/ref_rejected": -70.39239501953125,
      "logps/rejected": -75.95484924316406,
      "loss": 1.1894,
      "margin_dpo/margin_mean": 1.4359562397003174,
      "margin_dpo/margin_std": 3.0605721473693848,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": -4.377971649169922,
      "KL/mean": -5.268933296203613,
      "KL/rejected_KL_mean": -6.159893035888672,
      "KL/std": 2.8043086528778076,
      "epoch": 0.21768707482993196,
      "fcm_dpo/beta": 0.26900970935821533,
      "fcm_dpo/delta": -0.0838039442896843,
      "fcm_dpo/margin": 1.7819193601608276,
      "fcm_dpo/q_t": 0.39709940552711487,
      "grad_norm": 73.2460708618164,
      "learning_rate": 4.800745179625307e-07,
      "logits/chosen": 0.16512064635753632,
      "logits/rejected": 0.1372772604227066,
      "logps/chosen": -65.1806640625,
      "logps/ref_chosen": -60.80268859863281,
      "logps/ref_rejected": -79.07284545898438,
      "logps/rejected": -85.23273468017578,
      "loss": 1.1176,
      "margin_dpo/margin_mean": 1.7819199562072754,
      "margin_dpo/margin_std": 3.1004514694213867,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": -4.136661529541016,
      "KL/mean": -5.0256829261779785,
      "KL/rejected_KL_mean": -5.914703369140625,
      "KL/std": 2.8482725620269775,
      "epoch": 0.21919879062736206,
      "fcm_dpo/beta": 0.2655991315841675,
      "fcm_dpo/delta": -0.07585104554891586,
      "fcm_dpo/margin": 1.778045892715454,
      "fcm_dpo/q_t": 0.3986510634422302,
      "grad_norm": 80.55018615722656,
      "learning_rate": 4.795540267200686e-07,
      "logits/chosen": 0.13959573209285736,
      "logits/rejected": 0.15644602477550507,
      "logps/chosen": -78.74812316894531,
      "logps/ref_chosen": -74.61146545410156,
      "logps/ref_rejected": -83.24461364746094,
      "logps/rejected": -89.15931701660156,
      "loss": 1.1304,
      "margin_dpo/margin_mean": 1.7780449390411377,
      "margin_dpo/margin_std": 3.2379260063171387,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": -3.8457603454589844,
      "KL/mean": -4.857473850250244,
      "KL/rejected_KL_mean": -5.8691864013671875,
      "KL/std": 2.8113796710968018,
      "epoch": 0.22071050642479215,
      "fcm_dpo/beta": 0.2600950300693512,
      "fcm_dpo/delta": -0.1330960988998413,
      "fcm_dpo/margin": 2.0234241485595703,
      "fcm_dpo/q_t": 0.3855854272842407,
      "grad_norm": 64.35698699951172,
      "learning_rate": 4.790271143580173e-07,
      "logits/chosen": 0.13517965376377106,
      "logits/rejected": 0.11982344835996628,
      "logps/chosen": -61.686744689941406,
      "logps/ref_chosen": -57.84098434448242,
      "logps/ref_rejected": -67.47422790527344,
      "logps/rejected": -73.34341430664062,
      "loss": 1.0524,
      "margin_dpo/margin_mean": 2.0234241485595703,
      "margin_dpo/margin_std": 2.8947503566741943,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": -4.637237548828125,
      "KL/mean": -5.369053840637207,
      "KL/rejected_KL_mean": -6.100879669189453,
      "KL/std": 2.8983042240142822,
      "epoch": 0.2222222222222222,
      "fcm_dpo/beta": 0.25909751653671265,
      "fcm_dpo/delta": 0.02144131436944008,
      "fcm_dpo/margin": 1.4636409282684326,
      "fcm_dpo/q_t": 0.4160732924938202,
      "grad_norm": 85.97638702392578,
      "learning_rate": 4.784937956152489e-07,
      "logits/chosen": 0.15872755646705627,
      "logits/rejected": 0.11801473796367645,
      "logps/chosen": -71.4507064819336,
      "logps/ref_chosen": -66.81346893310547,
      "logps/ref_rejected": -81.1796875,
      "logps/rejected": -87.28057098388672,
      "loss": 1.1983,
      "margin_dpo/margin_mean": 1.4636404514312744,
      "margin_dpo/margin_std": 3.235443592071533,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": -4.425506591796875,
      "KL/mean": -5.488635063171387,
      "KL/rejected_KL_mean": -6.551761627197266,
      "KL/std": 2.806117534637451,
      "epoch": 0.2237339380196523,
      "fcm_dpo/beta": 0.2542745769023895,
      "fcm_dpo/delta": -0.1484357863664627,
      "fcm_dpo/margin": 2.1262574195861816,
      "fcm_dpo/q_t": 0.38453683257102966,
      "grad_norm": 55.99394226074219,
      "learning_rate": 4.779540854098347e-07,
      "logits/chosen": 0.2799733281135559,
      "logits/rejected": 0.2101047933101654,
      "logps/chosen": -53.11326217651367,
      "logps/ref_chosen": -48.6877555847168,
      "logps/ref_rejected": -67.50503540039062,
      "logps/rejected": -74.05679321289062,
      "loss": 1.068,
      "margin_dpo/margin_mean": 2.1262574195861816,
      "margin_dpo/margin_std": 3.2279231548309326,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": -5.392547607421875,
      "KL/mean": -6.629696846008301,
      "KL/rejected_KL_mean": -7.866847991943359,
      "KL/std": 3.4417757987976074,
      "epoch": 0.2252456538170824,
      "fcm_dpo/beta": 0.24332177639007568,
      "fcm_dpo/delta": -0.21501889824867249,
      "fcm_dpo/margin": 2.474299907684326,
      "fcm_dpo/q_t": 0.3698871433734894,
      "grad_norm": 57.53097152709961,
      "learning_rate": 4.774079988386296e-07,
      "logits/chosen": 0.13680626451969147,
      "logits/rejected": 0.09124539792537689,
      "logps/chosen": -60.53632354736328,
      "logps/ref_chosen": -55.143775939941406,
      "logps/ref_rejected": -64.79888916015625,
      "logps/rejected": -72.66574096679688,
      "loss": 1.0026,
      "margin_dpo/margin_mean": 2.4743001461029053,
      "margin_dpo/margin_std": 3.202667236328125,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": -4.186681747436523,
      "KL/mean": -5.657172679901123,
      "KL/rejected_KL_mean": -7.127662658691406,
      "KL/std": 3.2686009407043457,
      "epoch": 0.22675736961451248,
      "fcm_dpo/beta": 0.23021812736988068,
      "fcm_dpo/delta": -0.2972991466522217,
      "fcm_dpo/margin": 2.940983772277832,
      "fcm_dpo/q_t": 0.3510153293609619,
      "grad_norm": 54.16246032714844,
      "learning_rate": 4.768555511768486e-07,
      "logits/chosen": 0.16799965500831604,
      "logits/rejected": 0.1271965056657791,
      "logps/chosen": -71.65742492675781,
      "logps/ref_chosen": -67.47074890136719,
      "logps/ref_rejected": -89.21170806884766,
      "logps/rejected": -96.33937072753906,
      "loss": 0.9379,
      "margin_dpo/margin_mean": 2.9409842491149902,
      "margin_dpo/margin_std": 3.157912254333496,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": -4.072040557861328,
      "KL/mean": -5.650480270385742,
      "KL/rejected_KL_mean": -7.228923797607422,
      "KL/std": 3.339445114135742,
      "epoch": 0.22826908541194255,
      "fcm_dpo/beta": 0.21683219075202942,
      "fcm_dpo/delta": -0.3053116202354431,
      "fcm_dpo/margin": 3.15687894821167,
      "fcm_dpo/q_t": 0.3510277271270752,
      "grad_norm": 47.109622955322266,
      "learning_rate": 4.762967578776406e-07,
      "logits/chosen": 0.15717440843582153,
      "logits/rejected": 0.10770811885595322,
      "logps/chosen": -56.531585693359375,
      "logps/ref_chosen": -52.45954132080078,
      "logps/ref_rejected": -79.0630111694336,
      "logps/rejected": -86.29193115234375,
      "loss": 0.9328,
      "margin_dpo/margin_mean": 3.1568784713745117,
      "margin_dpo/margin_std": 3.4487314224243164,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": -5.256809234619141,
      "KL/mean": -6.478898048400879,
      "KL/rejected_KL_mean": -7.70098876953125,
      "KL/std": 3.440335750579834,
      "epoch": 0.22978080120937264,
      "fcm_dpo/beta": 0.21084949374198914,
      "fcm_dpo/delta": -0.12167318910360336,
      "fcm_dpo/margin": 2.4441773891448975,
      "fcm_dpo/q_t": 0.3902924954891205,
      "grad_norm": 53.89075469970703,
      "learning_rate": 4.757316345716553e-07,
      "logits/chosen": 0.2499184012413025,
      "logits/rejected": 0.2005215585231781,
      "logps/chosen": -61.81064224243164,
      "logps/ref_chosen": -56.5538330078125,
      "logps/ref_rejected": -76.55074310302734,
      "logps/rejected": -84.2517318725586,
      "loss": 1.0891,
      "margin_dpo/margin_mean": 2.4441769123077393,
      "margin_dpo/margin_std": 3.9026143550872803,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": -5.014923095703125,
      "KL/mean": -6.322737693786621,
      "KL/rejected_KL_mean": -7.630558013916016,
      "KL/std": 3.740640878677368,
      "epoch": 0.23129251700680273,
      "fcm_dpo/beta": 0.2035871297121048,
      "fcm_dpo/delta": -0.14023448526859283,
      "fcm_dpo/margin": 2.6156351566314697,
      "fcm_dpo/q_t": 0.38091546297073364,
      "grad_norm": 49.4394645690918,
      "learning_rate": 4.751601970666064e-07,
      "logits/chosen": 0.16498246788978577,
      "logits/rejected": 0.12865030765533447,
      "logps/chosen": -73.02182006835938,
      "logps/ref_chosen": -68.00689697265625,
      "logps/ref_rejected": -74.83482360839844,
      "logps/rejected": -82.46537780761719,
      "loss": 1.0201,
      "margin_dpo/margin_mean": 2.6156349182128906,
      "margin_dpo/margin_std": 3.229191541671753,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -5.846338272094727,
      "KL/mean": -6.814910888671875,
      "KL/rejected_KL_mean": -7.783485412597656,
      "KL/std": 3.7317049503326416,
      "epoch": 0.2328042328042328,
      "fcm_dpo/beta": 0.20286893844604492,
      "fcm_dpo/delta": 0.007271207869052887,
      "fcm_dpo/margin": 1.937145709991455,
      "fcm_dpo/q_t": 0.4129902124404907,
      "grad_norm": 50.64994812011719,
      "learning_rate": 4.745824613468292e-07,
      "logits/chosen": 0.2471812665462494,
      "logits/rejected": 0.24366626143455505,
      "logps/chosen": -65.06887817382812,
      "logps/ref_chosen": -59.222537994384766,
      "logps/ref_rejected": -64.19131469726562,
      "logps/rejected": -71.97480773925781,
      "loss": 1.1774,
      "margin_dpo/margin_mean": 1.9371455907821655,
      "margin_dpo/margin_std": 3.947800636291504,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -5.801868438720703,
      "KL/mean": -7.106063365936279,
      "KL/rejected_KL_mean": -8.410255432128906,
      "KL/std": 3.671025276184082,
      "epoch": 0.23431594860166288,
      "fcm_dpo/beta": 0.19799765944480896,
      "fcm_dpo/delta": -0.12400149554014206,
      "fcm_dpo/margin": 2.6083898544311523,
      "fcm_dpo/q_t": 0.3912537693977356,
      "grad_norm": 52.61668014526367,
      "learning_rate": 4.7399844357283393e-07,
      "logits/chosen": 0.25896644592285156,
      "logits/rejected": 0.23922425508499146,
      "logps/chosen": -74.25656127929688,
      "logps/ref_chosen": -68.45469665527344,
      "logps/ref_rejected": -77.91763305664062,
      "logps/rejected": -86.32789611816406,
      "loss": 1.1124,
      "margin_dpo/margin_mean": 2.608389377593994,
      "margin_dpo/margin_std": 4.497587203979492,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -5.890584945678711,
      "KL/mean": -7.42003059387207,
      "KL/rejected_KL_mean": -8.949478149414062,
      "KL/std": 3.9841737747192383,
      "epoch": 0.23582766439909297,
      "fcm_dpo/beta": 0.19194073975086212,
      "fcm_dpo/delta": -0.1988871991634369,
      "fcm_dpo/margin": 3.0588910579681396,
      "fcm_dpo/q_t": 0.37220460176467896,
      "grad_norm": 50.9721565246582,
      "learning_rate": 4.7340816008085305e-07,
      "logits/chosen": 0.2043873369693756,
      "logits/rejected": 0.1601003259420395,
      "logps/chosen": -73.16018676757812,
      "logps/ref_chosen": -67.26959991455078,
      "logps/ref_rejected": -86.95914459228516,
      "logps/rejected": -95.90862274169922,
      "loss": 0.999,
      "margin_dpo/margin_mean": 3.0588912963867188,
      "margin_dpo/margin_std": 3.7954955101013184,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -5.535558700561523,
      "KL/mean": -6.865988254547119,
      "KL/rejected_KL_mean": -8.196414947509766,
      "KL/std": 4.143555164337158,
      "epoch": 0.23733938019652306,
      "fcm_dpo/beta": 0.18544289469718933,
      "fcm_dpo/delta": -0.10112221539020538,
      "fcm_dpo/margin": 2.6608569622039795,
      "fcm_dpo/q_t": 0.39114609360694885,
      "grad_norm": 44.861305236816406,
      "learning_rate": 4.728116273823847e-07,
      "logits/chosen": 0.1911221146583557,
      "logits/rejected": 0.17139272391796112,
      "logps/chosen": -60.308433532714844,
      "logps/ref_chosen": -54.77287292480469,
      "logps/ref_rejected": -63.87866973876953,
      "logps/rejected": -72.07508850097656,
      "loss": 1.0696,
      "margin_dpo/margin_mean": 2.6608567237854004,
      "margin_dpo/margin_std": 3.7877914905548096,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -6.122274398803711,
      "KL/mean": -7.417959213256836,
      "KL/rejected_KL_mean": -8.713642120361328,
      "KL/std": 4.003837585449219,
      "epoch": 0.23885109599395313,
      "fcm_dpo/beta": 0.18369705975055695,
      "fcm_dpo/delta": -0.08005285263061523,
      "fcm_dpo/margin": 2.5913643836975098,
      "fcm_dpo/q_t": 0.39448630809783936,
      "grad_norm": 48.89786148071289,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": 0.21630354225635529,
      "logits/rejected": 0.18201735615730286,
      "logps/chosen": -71.04499053955078,
      "logps/ref_chosen": -64.92271423339844,
      "logps/ref_rejected": -82.23789978027344,
      "logps/rejected": -90.9515380859375,
      "loss": 1.0773,
      "margin_dpo/margin_mean": 2.5913643836975098,
      "margin_dpo/margin_std": 3.8079347610473633,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -6.438285827636719,
      "KL/mean": -7.947805881500244,
      "KL/rejected_KL_mean": -9.457328796386719,
      "KL/std": 4.387810707092285,
      "epoch": 0.24036281179138322,
      "fcm_dpo/beta": 0.18046115338802338,
      "fcm_dpo/delta": -0.1531095951795578,
      "fcm_dpo/margin": 3.0190439224243164,
      "fcm_dpo/q_t": 0.37977373600006104,
      "grad_norm": 52.29972839355469,
      "learning_rate": 4.715998812855304e-07,
      "logits/chosen": 0.23897811770439148,
      "logits/rejected": 0.20274843275547028,
      "logps/chosen": -63.48527908325195,
      "logps/ref_chosen": -57.046993255615234,
      "logps/ref_rejected": -73.32441711425781,
      "logps/rejected": -82.78174591064453,
      "loss": 1.0672,
      "margin_dpo/margin_mean": 3.0190439224243164,
      "margin_dpo/margin_std": 4.550737380981445,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": -7.259920120239258,
      "KL/mean": -8.699539184570312,
      "KL/rejected_KL_mean": -10.139163970947266,
      "KL/std": 4.2198638916015625,
      "epoch": 0.2418745275888133,
      "fcm_dpo/beta": 0.17518454790115356,
      "fcm_dpo/delta": -0.10981732606887817,
      "fcm_dpo/margin": 2.879239082336426,
      "fcm_dpo/q_t": 0.39520591497421265,
      "grad_norm": 41.98582077026367,
      "learning_rate": 4.7098470178228755e-07,
      "logits/chosen": 0.09705978631973267,
      "logits/rejected": 0.0565880686044693,
      "logps/chosen": -57.06683349609375,
      "logps/ref_chosen": -49.806915283203125,
      "logps/ref_rejected": -68.3370132446289,
      "logps/rejected": -78.47618103027344,
      "loss": 1.1023,
      "margin_dpo/margin_mean": 2.879239082336426,
      "margin_dpo/margin_std": 4.888503074645996,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": -7.292133331298828,
      "KL/mean": -8.760808944702148,
      "KL/rejected_KL_mean": -10.229486465454102,
      "KL/std": 4.264138698577881,
      "epoch": 0.24338624338624337,
      "fcm_dpo/beta": 0.171233668923378,
      "fcm_dpo/delta": -0.10836784541606903,
      "fcm_dpo/margin": 2.937352180480957,
      "fcm_dpo/q_t": 0.3911857604980469,
      "grad_norm": 41.31275939941406,
      "learning_rate": 4.703633408618955e-07,
      "logits/chosen": 0.22426341474056244,
      "logits/rejected": 0.1872980296611786,
      "logps/chosen": -59.79262161254883,
      "logps/ref_chosen": -52.50048828125,
      "logps/ref_rejected": -66.04540252685547,
      "logps/rejected": -76.27488708496094,
      "loss": 1.0814,
      "margin_dpo/margin_mean": 2.937352180480957,
      "margin_dpo/margin_std": 4.569244861602783,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": -7.956607818603516,
      "KL/mean": -10.08245849609375,
      "KL/rejected_KL_mean": -12.208309173583984,
      "KL/std": 4.87081241607666,
      "epoch": 0.24489795918367346,
      "fcm_dpo/beta": 0.16229870915412903,
      "fcm_dpo/delta": -0.3119698464870453,
      "fcm_dpo/margin": 4.251701354980469,
      "fcm_dpo/q_t": 0.34716495871543884,
      "grad_norm": 40.224891662597656,
      "learning_rate": 4.697358159051549e-07,
      "logits/chosen": 0.25031372904777527,
      "logits/rejected": 0.20408298075199127,
      "logps/chosen": -77.42579650878906,
      "logps/ref_chosen": -69.46919250488281,
      "logps/ref_rejected": -92.00952911376953,
      "logps/rejected": -104.21783447265625,
      "loss": 0.9289,
      "margin_dpo/margin_mean": 4.251701354980469,
      "margin_dpo/margin_std": 4.525267124176025,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": -7.32349967956543,
      "KL/mean": -9.388936996459961,
      "KL/rejected_KL_mean": -11.454376220703125,
      "KL/std": 4.648595809936523,
      "epoch": 0.24640967498110355,
      "fcm_dpo/beta": 0.15538102388381958,
      "fcm_dpo/delta": -0.2578536868095398,
      "fcm_dpo/margin": 4.13087272644043,
      "fcm_dpo/q_t": 0.36116883158683777,
      "grad_norm": 37.86701583862305,
      "learning_rate": 4.691021444652876e-07,
      "logits/chosen": 0.18093985319137573,
      "logits/rejected": 0.1371062844991684,
      "logps/chosen": -57.93733215332031,
      "logps/ref_chosen": -50.613834381103516,
      "logps/ref_rejected": -74.62033081054688,
      "logps/rejected": -86.07470703125,
      "loss": 0.9936,
      "margin_dpo/margin_mean": 4.13087272644043,
      "margin_dpo/margin_std": 5.091652870178223,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -8.09062385559082,
      "KL/mean": -10.102151870727539,
      "KL/rejected_KL_mean": -12.11368179321289,
      "KL/std": 4.975480079650879,
      "epoch": 0.24792139077853365,
      "fcm_dpo/beta": 0.14765475690364838,
      "fcm_dpo/delta": -0.20589160919189453,
      "fcm_dpo/margin": 4.023059368133545,
      "fcm_dpo/q_t": 0.37176260352134705,
      "grad_norm": 35.98881912231445,
      "learning_rate": 4.6846234426744624e-07,
      "logits/chosen": 0.18997550010681152,
      "logits/rejected": 0.13003680109977722,
      "logps/chosen": -62.93873596191406,
      "logps/ref_chosen": -54.848114013671875,
      "logps/ref_rejected": -79.0630111694336,
      "logps/rejected": -91.17669677734375,
      "loss": 1.0214,
      "margin_dpo/margin_mean": 4.023058891296387,
      "margin_dpo/margin_std": 5.317191123962402,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -8.894065856933594,
      "KL/mean": -10.737652778625488,
      "KL/rejected_KL_mean": -12.58123779296875,
      "KL/std": 5.082514762878418,
      "epoch": 0.2494331065759637,
      "fcm_dpo/beta": 0.14372721314430237,
      "fcm_dpo/delta": -0.13719907402992249,
      "fcm_dpo/margin": 3.6871719360351562,
      "fcm_dpo/q_t": 0.38155514001846313,
      "grad_norm": 36.49126052856445,
      "learning_rate": 4.678164332082175e-07,
      "logits/chosen": 0.268466055393219,
      "logits/rejected": 0.21414814889431,
      "logps/chosen": -59.9832763671875,
      "logps/ref_chosen": -51.089210510253906,
      "logps/ref_rejected": -71.23370361328125,
      "logps/rejected": -83.81494140625,
      "loss": 1.048,
      "margin_dpo/margin_mean": 3.687171697616577,
      "margin_dpo/margin_std": 4.927584648132324,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -8.764341354370117,
      "KL/mean": -10.186882019042969,
      "KL/rejected_KL_mean": -11.609416961669922,
      "KL/std": 4.883334159851074,
      "epoch": 0.2509448223733938,
      "fcm_dpo/beta": 0.14125752449035645,
      "fcm_dpo/delta": -0.0022036507725715637,
      "fcm_dpo/margin": 2.8450818061828613,
      "fcm_dpo/q_t": 0.4125140905380249,
      "grad_norm": 41.16147232055664,
      "learning_rate": 4.6716442935512214e-07,
      "logits/chosen": 0.2231883555650711,
      "logits/rejected": 0.13836176693439484,
      "logps/chosen": -71.95515441894531,
      "logps/ref_chosen": -63.19081115722656,
      "logps/ref_rejected": -93.8402099609375,
      "logps/rejected": -105.44963073730469,
      "loss": 1.1297,
      "margin_dpo/margin_mean": 2.8450818061828613,
      "margin_dpo/margin_std": 4.802867889404297,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -8.12190055847168,
      "KL/mean": -10.33206844329834,
      "KL/rejected_KL_mean": -12.542236328125,
      "KL/std": 5.092068672180176,
      "epoch": 0.25245653817082386,
      "fcm_dpo/beta": 0.13622060418128967,
      "fcm_dpo/delta": -0.21773764491081238,
      "fcm_dpo/margin": 4.4203338623046875,
      "fcm_dpo/q_t": 0.3652680814266205,
      "grad_norm": 31.98584747314453,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": 0.1831911951303482,
      "logits/rejected": 0.14837321639060974,
      "logps/chosen": -67.04617309570312,
      "logps/ref_chosen": -58.92427062988281,
      "logps/ref_rejected": -72.97377014160156,
      "logps/rejected": -85.51600646972656,
      "loss": 0.9763,
      "margin_dpo/margin_mean": 4.420334339141846,
      "margin_dpo/margin_std": 4.931003570556641,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": -9.720163345336914,
      "KL/mean": -11.258673667907715,
      "KL/rejected_KL_mean": -12.797183990478516,
      "KL/std": 5.373922348022461,
      "epoch": 0.25396825396825395,
      "fcm_dpo/beta": 0.13527539372444153,
      "fcm_dpo/delta": -0.016960913315415382,
      "fcm_dpo/margin": 3.0770163536071777,
      "fcm_dpo/q_t": 0.4077424705028534,
      "grad_norm": 38.734954833984375,
      "learning_rate": 4.6584221638904767e-07,
      "logits/chosen": 0.21366257965564728,
      "logits/rejected": 0.18088281154632568,
      "logps/chosen": -75.37154388427734,
      "logps/ref_chosen": -65.65138244628906,
      "logps/ref_rejected": -79.71418762207031,
      "logps/rejected": -92.51136779785156,
      "loss": 1.1092,
      "margin_dpo/margin_mean": 3.0770161151885986,
      "margin_dpo/margin_std": 4.752354621887207,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -8.624734878540039,
      "KL/mean": -10.732074737548828,
      "KL/rejected_KL_mean": -12.839412689208984,
      "KL/std": 5.623780250549316,
      "epoch": 0.25547996976568405,
      "fcm_dpo/beta": 0.13188880681991577,
      "fcm_dpo/delta": -0.16511370241641998,
      "fcm_dpo/margin": 4.21467399597168,
      "fcm_dpo/q_t": 0.38392937183380127,
      "grad_norm": 35.97867965698242,
      "learning_rate": 4.651720442612075e-07,
      "logits/chosen": 0.2866262197494507,
      "logits/rejected": 0.2534254193305969,
      "logps/chosen": -70.05059814453125,
      "logps/ref_chosen": -61.425865173339844,
      "logps/ref_rejected": -76.09590148925781,
      "logps/rejected": -88.93531036376953,
      "loss": 1.0504,
      "margin_dpo/margin_mean": 4.21467399597168,
      "margin_dpo/margin_std": 6.357587814331055,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": -9.009101867675781,
      "KL/mean": -10.950794219970703,
      "KL/rejected_KL_mean": -12.89248275756836,
      "KL/std": 5.535174369812012,
      "epoch": 0.25699168556311414,
      "fcm_dpo/beta": 0.1291724443435669,
      "fcm_dpo/delta": -0.10685983300209045,
      "fcm_dpo/margin": 3.883380174636841,
      "fcm_dpo/q_t": 0.3895862400531769,
      "grad_norm": 30.346723556518555,
      "learning_rate": 4.6449585330874425e-07,
      "logits/chosen": 0.22347985208034515,
      "logits/rejected": 0.22092100977897644,
      "logps/chosen": -65.66229248046875,
      "logps/ref_chosen": -56.65319061279297,
      "logps/ref_rejected": -63.45965576171875,
      "logps/rejected": -76.35213470458984,
      "loss": 1.095,
      "margin_dpo/margin_mean": 3.883380651473999,
      "margin_dpo/margin_std": 6.250423431396484,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -9.56930160522461,
      "KL/mean": -11.856922149658203,
      "KL/rejected_KL_mean": -14.144546508789062,
      "KL/std": 6.232220649719238,
      "epoch": 0.2585034013605442,
      "fcm_dpo/beta": 0.1235651969909668,
      "fcm_dpo/delta": -0.17861855030059814,
      "fcm_dpo/margin": 4.575247287750244,
      "fcm_dpo/q_t": 0.37708625197410583,
      "grad_norm": 34.1002311706543,
      "learning_rate": 4.6381366244617224e-07,
      "logits/chosen": 0.27739018201828003,
      "logits/rejected": 0.2282651960849762,
      "logps/chosen": -73.30406188964844,
      "logps/ref_chosen": -63.73476028442383,
      "logps/ref_rejected": -78.50328063964844,
      "logps/rejected": -92.6478271484375,
      "loss": 1.0601,
      "margin_dpo/margin_mean": 4.575246810913086,
      "margin_dpo/margin_std": 6.797544956207275,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": -10.548381805419922,
      "KL/mean": -12.787707328796387,
      "KL/rejected_KL_mean": -15.027034759521484,
      "KL/std": 6.131152153015137,
      "epoch": 0.2600151171579743,
      "fcm_dpo/beta": 0.12173713743686676,
      "fcm_dpo/delta": -0.15364830195903778,
      "fcm_dpo/margin": 4.478647708892822,
      "fcm_dpo/q_t": 0.3781545162200928,
      "grad_norm": 30.768226623535156,
      "learning_rate": 4.631254907558365e-07,
      "logits/chosen": 0.3069169521331787,
      "logits/rejected": 0.2507067918777466,
      "logps/chosen": -62.75014114379883,
      "logps/ref_chosen": -52.201759338378906,
      "logps/ref_rejected": -82.85285949707031,
      "logps/rejected": -97.87989807128906,
      "loss": 1.0419,
      "margin_dpo/margin_mean": 4.478647232055664,
      "margin_dpo/margin_std": 6.057438850402832,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -10.424421310424805,
      "KL/mean": -12.854316711425781,
      "KL/rejected_KL_mean": -15.28421401977539,
      "KL/std": 6.542934894561768,
      "epoch": 0.2615268329554044,
      "fcm_dpo/beta": 0.11532153189182281,
      "fcm_dpo/delta": -0.17521372437477112,
      "fcm_dpo/margin": 4.859795093536377,
      "fcm_dpo/q_t": 0.3852936327457428,
      "grad_norm": 29.455821990966797,
      "learning_rate": 4.624313574873786e-07,
      "logits/chosen": 0.29886192083358765,
      "logits/rejected": 0.21168309450149536,
      "logps/chosen": -65.85914611816406,
      "logps/ref_chosen": -55.434722900390625,
      "logps/ref_rejected": -77.81967163085938,
      "logps/rejected": -93.10388946533203,
      "loss": 1.1048,
      "margin_dpo/margin_mean": 4.859795570373535,
      "margin_dpo/margin_std": 8.13494873046875,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -11.51085090637207,
      "KL/mean": -13.983785629272461,
      "KL/rejected_KL_mean": -16.456722259521484,
      "KL/std": 6.39737606048584,
      "epoch": 0.26303854875283444,
      "fcm_dpo/beta": 0.11249849945306778,
      "fcm_dpo/delta": -0.16640028357505798,
      "fcm_dpo/margin": 4.945873260498047,
      "fcm_dpo/q_t": 0.38104724884033203,
      "grad_norm": 31.90245819091797,
      "learning_rate": 4.61731282057198e-07,
      "logits/chosen": 0.25333988666534424,
      "logits/rejected": 0.18902552127838135,
      "logps/chosen": -68.68280029296875,
      "logps/ref_chosen": -57.17195129394531,
      "logps/ref_rejected": -85.47578430175781,
      "logps/rejected": -101.93250274658203,
      "loss": 1.052,
      "margin_dpo/margin_mean": 4.9458723068237305,
      "margin_dpo/margin_std": 7.298829078674316,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -11.260446548461914,
      "KL/mean": -14.006481170654297,
      "KL/rejected_KL_mean": -16.752525329589844,
      "KL/std": 6.899945259094238,
      "epoch": 0.26455026455026454,
      "fcm_dpo/beta": 0.10868742316961288,
      "fcm_dpo/delta": -0.20911765098571777,
      "fcm_dpo/margin": 5.492076873779297,
      "fcm_dpo/q_t": 0.3743385672569275,
      "grad_norm": 30.493921279907227,
      "learning_rate": 4.6102528404790965e-07,
      "logits/chosen": 0.31453484296798706,
      "logits/rejected": 0.2838860750198364,
      "logps/chosen": -78.92607116699219,
      "logps/ref_chosen": -67.6656265258789,
      "logps/ref_rejected": -84.36766815185547,
      "logps/rejected": -101.12019348144531,
      "loss": 1.0364,
      "margin_dpo/margin_mean": 5.492076873779297,
      "margin_dpo/margin_std": 7.835512638092041,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -12.608375549316406,
      "KL/mean": -14.528512954711914,
      "KL/rejected_KL_mean": -16.448657989501953,
      "KL/std": 7.345946311950684,
      "epoch": 0.2660619803476946,
      "fcm_dpo/beta": 0.10646377503871918,
      "fcm_dpo/delta": -0.01017729565501213,
      "fcm_dpo/margin": 3.840282678604126,
      "fcm_dpo/q_t": 0.4146023094654083,
      "grad_norm": 36.30823516845703,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": 0.25232359766960144,
      "logits/rejected": 0.22588184475898743,
      "logps/chosen": -90.46713256835938,
      "logps/ref_chosen": -77.8587646484375,
      "logps/ref_rejected": -81.08732604980469,
      "logps/rejected": -97.53598022460938,
      "loss": 1.178,
      "margin_dpo/margin_mean": 3.840282440185547,
      "margin_dpo/margin_std": 7.908246994018555,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -10.896930694580078,
      "KL/mean": -14.61917495727539,
      "KL/rejected_KL_mean": -18.341419219970703,
      "KL/std": 7.5128865242004395,
      "epoch": 0.2675736961451247,
      "fcm_dpo/beta": 0.10073349624872208,
      "fcm_dpo/delta": -0.38075220584869385,
      "fcm_dpo/margin": 7.444479942321777,
      "fcm_dpo/q_t": 0.3359594941139221,
      "grad_norm": 31.873899459838867,
      "learning_rate": 4.5959559945025183e-07,
      "logits/chosen": 0.3715853691101074,
      "logits/rejected": 0.27700120210647583,
      "logps/chosen": -66.11732482910156,
      "logps/ref_chosen": -55.22039794921875,
      "logps/ref_rejected": -92.54973602294922,
      "logps/rejected": -110.89115905761719,
      "loss": 0.895,
      "margin_dpo/margin_mean": 7.444479465484619,
      "margin_dpo/margin_std": 7.599752426147461,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -12.055704116821289,
      "KL/mean": -14.323225975036621,
      "KL/rejected_KL_mean": -16.590744018554688,
      "KL/std": 7.222278594970703,
      "epoch": 0.2690854119425548,
      "fcm_dpo/beta": 0.09768117219209671,
      "fcm_dpo/delta": -0.04705891013145447,
      "fcm_dpo/margin": 4.535033226013184,
      "fcm_dpo/q_t": 0.4012787938117981,
      "grad_norm": 29.5404109954834,
      "learning_rate": 4.588719528532341e-07,
      "logits/chosen": 0.2326379418373108,
      "logits/rejected": 0.1847991943359375,
      "logps/chosen": -72.86619567871094,
      "logps/ref_chosen": -60.81049346923828,
      "logps/ref_rejected": -81.12973022460938,
      "logps/rejected": -97.72047424316406,
      "loss": 1.1063,
      "margin_dpo/margin_mean": 4.535033702850342,
      "margin_dpo/margin_std": 7.025606155395508,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -13.087169647216797,
      "KL/mean": -15.591184616088867,
      "KL/rejected_KL_mean": -18.095199584960938,
      "KL/std": 7.325949668884277,
      "epoch": 0.2705971277399849,
      "fcm_dpo/beta": 0.0970505028963089,
      "fcm_dpo/delta": -0.09039415419101715,
      "fcm_dpo/margin": 5.008024215698242,
      "fcm_dpo/q_t": 0.39517539739608765,
      "grad_norm": 29.013471603393555,
      "learning_rate": 4.581424636586928e-07,
      "logits/chosen": 0.3003222346305847,
      "logits/rejected": 0.2833176553249359,
      "logps/chosen": -78.75888061523438,
      "logps/ref_chosen": -65.67171478271484,
      "logps/ref_rejected": -75.32586669921875,
      "logps/rejected": -93.42106628417969,
      "loss": 1.1098,
      "margin_dpo/margin_mean": 5.008025169372559,
      "margin_dpo/margin_std": 8.527783393859863,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -10.796808242797852,
      "KL/mean": -13.163284301757812,
      "KL/rejected_KL_mean": -15.529760360717773,
      "KL/std": 7.749887943267822,
      "epoch": 0.272108843537415,
      "fcm_dpo/beta": 0.09602123498916626,
      "fcm_dpo/delta": -0.057060666382312775,
      "fcm_dpo/margin": 4.732954025268555,
      "fcm_dpo/q_t": 0.40494978427886963,
      "grad_norm": 27.684741973876953,
      "learning_rate": 4.5740715227200897e-07,
      "logits/chosen": 0.1134958416223526,
      "logits/rejected": 0.09461627900600433,
      "logps/chosen": -67.4796142578125,
      "logps/ref_chosen": -56.68280792236328,
      "logps/ref_rejected": -64.94414520263672,
      "logps/rejected": -80.47390747070312,
      "loss": 1.1506,
      "margin_dpo/margin_mean": 4.732954502105713,
      "margin_dpo/margin_std": 9.071979522705078,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -10.026453018188477,
      "KL/mean": -13.574191093444824,
      "KL/rejected_KL_mean": -17.121925354003906,
      "KL/std": 8.484979629516602,
      "epoch": 0.273620559334845,
      "fcm_dpo/beta": 0.09160138666629791,
      "fcm_dpo/delta": -0.26784011721611023,
      "fcm_dpo/margin": 7.095474720001221,
      "fcm_dpo/q_t": 0.35501545667648315,
      "grad_norm": 25.291221618652344,
      "learning_rate": 4.566660392614228e-07,
      "logits/chosen": 0.30669811367988586,
      "logits/rejected": 0.2673833668231964,
      "logps/chosen": -70.80250549316406,
      "logps/ref_chosen": -60.77604675292969,
      "logps/ref_rejected": -83.98361206054688,
      "logps/rejected": -101.10554504394531,
      "loss": 0.9419,
      "margin_dpo/margin_mean": 7.095475196838379,
      "margin_dpo/margin_std": 7.501391410827637,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -11.768840789794922,
      "KL/mean": -15.67117691040039,
      "KL/rejected_KL_mean": -19.573516845703125,
      "KL/std": 8.88789176940918,
      "epoch": 0.2751322751322751,
      "fcm_dpo/beta": 0.08631753921508789,
      "fcm_dpo/delta": -0.2947568893432617,
      "fcm_dpo/margin": 7.804677963256836,
      "fcm_dpo/q_t": 0.3566606938838959,
      "grad_norm": 24.507036209106445,
      "learning_rate": 4.5591914535745817e-07,
      "logits/chosen": 0.2883094251155853,
      "logits/rejected": 0.2092103213071823,
      "logps/chosen": -72.02262878417969,
      "logps/ref_chosen": -60.2537841796875,
      "logps/ref_rejected": -89.7706298828125,
      "logps/rejected": -109.34414672851562,
      "loss": 0.9841,
      "margin_dpo/margin_mean": 7.804677963256836,
      "margin_dpo/margin_std": 10.020936012268066,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -14.355695724487305,
      "KL/mean": -15.877723693847656,
      "KL/rejected_KL_mean": -17.399749755859375,
      "KL/std": 8.516490936279297,
      "epoch": 0.2766439909297052,
      "fcm_dpo/beta": 0.08607832342386246,
      "fcm_dpo/delta": 0.044956937432289124,
      "fcm_dpo/margin": 3.0440587997436523,
      "fcm_dpo/q_t": 0.44098007678985596,
      "grad_norm": 27.24208641052246,
      "learning_rate": 4.551664914523433e-07,
      "logits/chosen": 0.2489510476589203,
      "logits/rejected": 0.22813934087753296,
      "logps/chosen": -76.11711120605469,
      "logps/ref_chosen": -61.76142120361328,
      "logps/ref_rejected": -72.54627990722656,
      "logps/rejected": -89.94602966308594,
      "loss": 1.2561,
      "margin_dpo/margin_mean": 3.044058322906494,
      "margin_dpo/margin_std": 8.112913131713867,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -10.535205841064453,
      "KL/mean": -13.59359359741211,
      "KL/rejected_KL_mean": -16.6519775390625,
      "KL/std": 7.746424674987793,
      "epoch": 0.2781557067271353,
      "fcm_dpo/beta": 0.08390414714813232,
      "fcm_dpo/delta": -0.12142601609230042,
      "fcm_dpo/margin": 6.116772651672363,
      "fcm_dpo/q_t": 0.38632309436798096,
      "grad_norm": 21.602025985717773,
      "learning_rate": 4.544080985994258e-07,
      "logits/chosen": 0.3624228537082672,
      "logits/rejected": 0.29974132776260376,
      "logps/chosen": -57.37592697143555,
      "logps/ref_chosen": -46.840721130371094,
      "logps/ref_rejected": -69.3609390258789,
      "logps/rejected": -86.0129165649414,
      "loss": 1.0354,
      "margin_dpo/margin_mean": 6.116772651672363,
      "margin_dpo/margin_std": 7.672127723693848,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -12.417903900146484,
      "KL/mean": -15.590404510498047,
      "KL/rejected_KL_mean": -18.76290512084961,
      "KL/std": 8.948210716247559,
      "epoch": 0.2796674225245654,
      "fcm_dpo/beta": 0.08195741474628448,
      "fcm_dpo/delta": -0.12801620364189148,
      "fcm_dpo/margin": 6.345008850097656,
      "fcm_dpo/q_t": 0.39112916588783264,
      "grad_norm": 22.322933197021484,
      "learning_rate": 4.5364398801258394e-07,
      "logits/chosen": 0.28738462924957275,
      "logits/rejected": 0.24117065966129303,
      "logps/chosen": -64.73904418945312,
      "logps/ref_chosen": -52.32114028930664,
      "logps/ref_rejected": -68.3885726928711,
      "logps/rejected": -87.15147399902344,
      "loss": 1.1172,
      "margin_dpo/margin_mean": 6.3450093269348145,
      "margin_dpo/margin_std": 11.064637184143066,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -11.585432052612305,
      "KL/mean": -15.133550643920898,
      "KL/rejected_KL_mean": -18.681671142578125,
      "KL/std": 9.106042861938477,
      "epoch": 0.2811791383219955,
      "fcm_dpo/beta": 0.07970194518566132,
      "fcm_dpo/delta": -0.17570821940898895,
      "fcm_dpo/margin": 7.096240043640137,
      "fcm_dpo/q_t": 0.38234925270080566,
      "grad_norm": 27.01889991760254,
      "learning_rate": 4.5287418106563354e-07,
      "logits/chosen": 0.23089167475700378,
      "logits/rejected": 0.18934544920921326,
      "logps/chosen": -79.00556182861328,
      "logps/ref_chosen": -67.42012786865234,
      "logps/ref_rejected": -82.50968933105469,
      "logps/rejected": -101.19136047363281,
      "loss": 1.076,
      "margin_dpo/margin_mean": 7.096240043640137,
      "margin_dpo/margin_std": 11.376433372497559,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -13.08731460571289,
      "KL/mean": -16.411659240722656,
      "KL/rejected_KL_mean": -19.736003875732422,
      "KL/std": 9.400962829589844,
      "epoch": 0.28269085411942557,
      "fcm_dpo/beta": 0.07743757218122482,
      "fcm_dpo/delta": -0.1215682178735733,
      "fcm_dpo/margin": 6.648694038391113,
      "fcm_dpo/q_t": 0.38729268312454224,
      "grad_norm": 26.019197463989258,
      "learning_rate": 4.520986992917297e-07,
      "logits/chosen": 0.288669228553772,
      "logits/rejected": 0.23321621119976044,
      "logps/chosen": -88.61280822753906,
      "logps/ref_chosen": -75.52549743652344,
      "logps/ref_rejected": -94.76289367675781,
      "logps/rejected": -114.4989013671875,
      "loss": 1.0867,
      "margin_dpo/margin_mean": 6.648694038391113,
      "margin_dpo/margin_std": 10.502693176269531,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -12.019424438476562,
      "KL/mean": -15.445846557617188,
      "KL/rejected_KL_mean": -18.87226104736328,
      "KL/std": 9.891624450683594,
      "epoch": 0.2842025699168556,
      "fcm_dpo/beta": 0.0757642388343811,
      "fcm_dpo/delta": -0.12568299472332,
      "fcm_dpo/margin": 6.85283088684082,
      "fcm_dpo/q_t": 0.3881131708621979,
      "grad_norm": 25.153697967529297,
      "learning_rate": 4.5131756438276466e-07,
      "logits/chosen": 0.32440823316574097,
      "logits/rejected": 0.27967768907546997,
      "logps/chosen": -83.54275512695312,
      "logps/ref_chosen": -71.52333068847656,
      "logps/ref_rejected": -78.29949951171875,
      "logps/rejected": -97.17176055908203,
      "loss": 1.076,
      "margin_dpo/margin_mean": 6.85283088684082,
      "margin_dpo/margin_std": 10.685548782348633,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -11.275667190551758,
      "KL/mean": -14.627676963806152,
      "KL/rejected_KL_mean": -17.979686737060547,
      "KL/std": 9.685689926147461,
      "epoch": 0.2857142857142857,
      "fcm_dpo/beta": 0.07335545122623444,
      "fcm_dpo/delta": -0.09989577531814575,
      "fcm_dpo/margin": 6.704021453857422,
      "fcm_dpo/q_t": 0.3903145492076874,
      "grad_norm": 24.36782455444336,
      "learning_rate": 4.5053079818876096e-07,
      "logits/chosen": 0.31160449981689453,
      "logits/rejected": 0.32390740513801575,
      "logps/chosen": -83.45193481445312,
      "logps/ref_chosen": -72.17626953125,
      "logps/ref_rejected": -75.26313781738281,
      "logps/rejected": -93.2428207397461,
      "loss": 1.0889,
      "margin_dpo/margin_mean": 6.7040228843688965,
      "margin_dpo/margin_std": 10.087574005126953,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -10.780826568603516,
      "KL/mean": -15.475550651550293,
      "KL/rejected_KL_mean": -20.170272827148438,
      "KL/std": 10.145885467529297,
      "epoch": 0.2872260015117158,
      "fcm_dpo/beta": 0.07069344073534012,
      "fcm_dpo/delta": -0.28270792961120605,
      "fcm_dpo/margin": 9.389444351196289,
      "fcm_dpo/q_t": 0.3553423285484314,
      "grad_norm": 24.150226593017578,
      "learning_rate": 4.4973842271726024e-07,
      "logits/chosen": 0.3613849878311157,
      "logits/rejected": 0.2206803560256958,
      "logps/chosen": -65.40509796142578,
      "logps/ref_chosen": -54.624271392822266,
      "logps/ref_rejected": -101.47068786621094,
      "logps/rejected": -121.64096069335938,
      "loss": 0.9544,
      "margin_dpo/margin_mean": 9.389444351196289,
      "margin_dpo/margin_std": 10.760894775390625,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -13.82766342163086,
      "KL/mean": -17.24479866027832,
      "KL/rejected_KL_mean": -20.66193389892578,
      "KL/std": 10.255237579345703,
      "epoch": 0.2887377173091459,
      "fcm_dpo/beta": 0.0689966082572937,
      "fcm_dpo/delta": -0.07502906769514084,
      "fcm_dpo/margin": 6.834271430969238,
      "fcm_dpo/q_t": 0.39443153142929077,
      "grad_norm": 24.53253173828125,
      "learning_rate": 4.48940460132708e-07,
      "logits/chosen": 0.35218584537506104,
      "logits/rejected": 0.3250824511051178,
      "logps/chosen": -86.76017761230469,
      "logps/ref_chosen": -72.93251037597656,
      "logps/ref_rejected": -89.95103454589844,
      "logps/rejected": -110.61296844482422,
      "loss": 1.1062,
      "margin_dpo/margin_mean": 6.834270477294922,
      "margin_dpo/margin_std": 11.15610122680664,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -14.251951217651367,
      "KL/mean": -16.651016235351562,
      "KL/rejected_KL_mean": -19.050079345703125,
      "KL/std": 10.874744415283203,
      "epoch": 0.29024943310657597,
      "fcm_dpo/beta": 0.0695391297340393,
      "fcm_dpo/delta": 0.06834352016448975,
      "fcm_dpo/margin": 4.798130035400391,
      "fcm_dpo/q_t": 0.4251948595046997,
      "grad_norm": 19.9398136138916,
      "learning_rate": 4.481369327558329e-07,
      "logits/chosen": 0.3311668038368225,
      "logits/rejected": 0.3057538866996765,
      "logps/chosen": -68.25306701660156,
      "logps/ref_chosen": -54.001121520996094,
      "logps/ref_rejected": -63.531551361083984,
      "logps/rejected": -82.58163452148438,
      "loss": 1.1854,
      "margin_dpo/margin_mean": 4.798130035400391,
      "margin_dpo/margin_std": 9.688655853271484,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -12.110715866088867,
      "KL/mean": -16.520183563232422,
      "KL/rejected_KL_mean": -20.929645538330078,
      "KL/std": 10.359651565551758,
      "epoch": 0.29176114890400606,
      "fcm_dpo/beta": 0.06718215346336365,
      "fcm_dpo/delta": -0.20611168444156647,
      "fcm_dpo/margin": 8.818931579589844,
      "fcm_dpo/q_t": 0.3694593608379364,
      "grad_norm": 19.043062210083008,
      "learning_rate": 4.47327863063023e-07,
      "logits/chosen": 0.26940101385116577,
      "logits/rejected": 0.2465055286884308,
      "logps/chosen": -68.85999298095703,
      "logps/ref_chosen": -56.74927520751953,
      "logps/ref_rejected": -58.80629348754883,
      "logps/rejected": -79.7359390258789,
      "loss": 0.9925,
      "margin_dpo/margin_mean": 8.818931579589844,
      "margin_dpo/margin_std": 10.676626205444336,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -12.245641708374023,
      "KL/mean": -15.736579895019531,
      "KL/rejected_KL_mean": -19.227519989013672,
      "KL/std": 10.281841278076172,
      "epoch": 0.29327286470143615,
      "fcm_dpo/beta": 0.06571200489997864,
      "fcm_dpo/delta": -0.06307755410671234,
      "fcm_dpo/margin": 6.981878280639648,
      "fcm_dpo/q_t": 0.3988710045814514,
      "grad_norm": 20.552404403686523,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 0.3422006368637085,
      "logits/rejected": 0.31276822090148926,
      "logps/chosen": -68.89508056640625,
      "logps/ref_chosen": -56.64944076538086,
      "logps/ref_rejected": -69.98954772949219,
      "logps/rejected": -89.2170639038086,
      "loss": 1.1452,
      "margin_dpo/margin_mean": 6.981878280639648,
      "margin_dpo/margin_std": 12.815977096557617,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -14.283248901367188,
      "KL/mean": -18.73421287536621,
      "KL/rejected_KL_mean": -23.1851806640625,
      "KL/std": 11.819705963134766,
      "epoch": 0.2947845804988662,
      "fcm_dpo/beta": 0.06387071311473846,
      "fcm_dpo/delta": -0.18032635748386383,
      "fcm_dpo/margin": 8.901932716369629,
      "fcm_dpo/q_t": 0.3765624761581421,
      "grad_norm": 21.156545639038086,
      "learning_rate": 4.4569318740967043e-07,
      "logits/chosen": 0.25106382369995117,
      "logits/rejected": 0.25328803062438965,
      "logps/chosen": -84.69302368164062,
      "logps/ref_chosen": -70.40977478027344,
      "logps/ref_rejected": -74.39448547363281,
      "logps/rejected": -97.57966613769531,
      "loss": 1.0292,
      "margin_dpo/margin_mean": 8.901932716369629,
      "margin_dpo/margin_std": 12.040631294250488,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -13.364896774291992,
      "KL/mean": -17.0338191986084,
      "KL/rejected_KL_mean": -20.702739715576172,
      "KL/std": 11.776092529296875,
      "epoch": 0.2962962962962963,
      "fcm_dpo/beta": 0.06328917294740677,
      "fcm_dpo/delta": -0.06768125295639038,
      "fcm_dpo/margin": 7.3378376960754395,
      "fcm_dpo/q_t": 0.3948385417461395,
      "grad_norm": 19.727270126342773,
      "learning_rate": 4.448676271745197e-07,
      "logits/chosen": 0.3381340205669403,
      "logits/rejected": 0.2969015836715698,
      "logps/chosen": -72.59246826171875,
      "logps/ref_chosen": -59.227577209472656,
      "logps/ref_rejected": -83.54757690429688,
      "logps/rejected": -104.25031280517578,
      "loss": 1.0878,
      "margin_dpo/margin_mean": 7.3378376960754395,
      "margin_dpo/margin_std": 10.919742584228516,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -11.704835891723633,
      "KL/mean": -16.28235626220703,
      "KL/rejected_KL_mean": -20.859878540039062,
      "KL/std": 11.915338516235352,
      "epoch": 0.29780801209372637,
      "fcm_dpo/beta": 0.061614636331796646,
      "fcm_dpo/delta": -0.1736968606710434,
      "fcm_dpo/margin": 9.155037879943848,
      "fcm_dpo/q_t": 0.37907886505126953,
      "grad_norm": 19.98828125,
      "learning_rate": 4.440366160729392e-07,
      "logits/chosen": 0.43114370107650757,
      "logits/rejected": 0.38091135025024414,
      "logps/chosen": -63.23396682739258,
      "logps/ref_chosen": -51.52912902832031,
      "logps/ref_rejected": -73.70631408691406,
      "logps/rejected": -94.56619262695312,
      "loss": 1.0893,
      "margin_dpo/margin_mean": 9.155037879943848,
      "margin_dpo/margin_std": 14.674212455749512,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -12.288036346435547,
      "KL/mean": -17.261886596679688,
      "KL/rejected_KL_mean": -22.235740661621094,
      "KL/std": 11.151510238647461,
      "epoch": 0.29931972789115646,
      "fcm_dpo/beta": 0.059206273406744,
      "fcm_dpo/delta": -0.20032742619514465,
      "fcm_dpo/margin": 9.947700500488281,
      "fcm_dpo/q_t": 0.3675551116466522,
      "grad_norm": 19.296764373779297,
      "learning_rate": 4.432001773500957e-07,
      "logits/chosen": 0.3754596710205078,
      "logits/rejected": 0.33579397201538086,
      "logps/chosen": -72.07071685791016,
      "logps/ref_chosen": -59.78268051147461,
      "logps/ref_rejected": -72.24533081054688,
      "logps/rejected": -94.48106384277344,
      "loss": 0.9862,
      "margin_dpo/margin_mean": 9.947700500488281,
      "margin_dpo/margin_std": 11.30981159210205,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -14.60167121887207,
      "KL/mean": -18.459152221679688,
      "KL/rejected_KL_mean": -22.316627502441406,
      "KL/std": 12.058280944824219,
      "epoch": 0.30083144368858655,
      "fcm_dpo/beta": 0.05836878716945648,
      "fcm_dpo/delta": -0.05422385782003403,
      "fcm_dpo/margin": 7.714962959289551,
      "fcm_dpo/q_t": 0.4011952877044678,
      "grad_norm": 19.802335739135742,
      "learning_rate": 4.4235833440297856e-07,
      "logits/chosen": 0.3471040725708008,
      "logits/rejected": 0.2592379152774811,
      "logps/chosen": -70.98844146728516,
      "logps/ref_chosen": -56.38677215576172,
      "logps/ref_rejected": -74.56779479980469,
      "logps/rejected": -96.88442993164062,
      "loss": 1.1597,
      "margin_dpo/margin_mean": 7.714962959289551,
      "margin_dpo/margin_std": 14.490645408630371,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -11.44668197631836,
      "KL/mean": -16.932655334472656,
      "KL/rejected_KL_mean": -22.418624877929688,
      "KL/std": 12.822843551635742,
      "epoch": 0.30234315948601664,
      "fcm_dpo/beta": 0.055415768176317215,
      "fcm_dpo/delta": -0.22348003089427948,
      "fcm_dpo/margin": 10.971942901611328,
      "fcm_dpo/q_t": 0.3693169951438904,
      "grad_norm": 18.33708953857422,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": 0.39051544666290283,
      "logits/rejected": 0.3189677298069,
      "logps/chosen": -69.27101135253906,
      "logps/ref_chosen": -57.82432556152344,
      "logps/ref_rejected": -89.28246307373047,
      "logps/rejected": -111.70108795166016,
      "loss": 1.0304,
      "margin_dpo/margin_mean": 10.971942901611328,
      "margin_dpo/margin_std": 15.467931747436523,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -15.586231231689453,
      "KL/mean": -20.176849365234375,
      "KL/rejected_KL_mean": -24.767475128173828,
      "KL/std": 13.329109191894531,
      "epoch": 0.30385487528344673,
      "fcm_dpo/beta": 0.05406852066516876,
      "fcm_dpo/delta": -0.10211023688316345,
      "fcm_dpo/margin": 9.181241035461426,
      "fcm_dpo/q_t": 0.39290472865104675,
      "grad_norm": 20.03122329711914,
      "learning_rate": 4.4065853017905953e-07,
      "logits/chosen": 0.4268413186073303,
      "logits/rejected": 0.3806511163711548,
      "logps/chosen": -74.58599090576172,
      "logps/ref_chosen": -58.999759674072266,
      "logps/ref_rejected": -84.67575073242188,
      "logps/rejected": -109.44322204589844,
      "loss": 1.0898,
      "margin_dpo/margin_mean": 9.18124008178711,
      "margin_dpo/margin_std": 14.563886642456055,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -12.942483901977539,
      "KL/mean": -18.29604148864746,
      "KL/rejected_KL_mean": -23.64959716796875,
      "KL/std": 13.108734130859375,
      "epoch": 0.30536659108087677,
      "fcm_dpo/beta": 0.05283664911985397,
      "fcm_dpo/delta": -0.17529305815696716,
      "fcm_dpo/margin": 10.707110404968262,
      "fcm_dpo/q_t": 0.3732600510120392,
      "grad_norm": 18.544675827026367,
      "learning_rate": 4.3980061644943575e-07,
      "logits/chosen": 0.3339017629623413,
      "logits/rejected": 0.2624325156211853,
      "logps/chosen": -60.60313415527344,
      "logps/ref_chosen": -47.660648345947266,
      "logps/ref_rejected": -73.63249969482422,
      "logps/rejected": -97.28209686279297,
      "loss": 1.0304,
      "margin_dpo/margin_mean": 10.707110404968262,
      "margin_dpo/margin_std": 14.182441711425781,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -15.660724639892578,
      "KL/mean": -20.505842208862305,
      "KL/rejected_KL_mean": -25.3509521484375,
      "KL/std": 13.342029571533203,
      "epoch": 0.30687830687830686,
      "fcm_dpo/beta": 0.05144822597503662,
      "fcm_dpo/delta": -0.10358630120754242,
      "fcm_dpo/margin": 9.69023323059082,
      "fcm_dpo/q_t": 0.391654908657074,
      "grad_norm": 21.113204956054688,
      "learning_rate": 4.3893739358856455e-07,
      "logits/chosen": 0.3900166153907776,
      "logits/rejected": 0.31723517179489136,
      "logps/chosen": -77.98625946044922,
      "logps/ref_chosen": -62.32553482055664,
      "logps/ref_rejected": -99.37226104736328,
      "logps/rejected": -124.72321319580078,
      "loss": 1.0751,
      "margin_dpo/margin_mean": 9.69023323059082,
      "margin_dpo/margin_std": 14.619604110717773,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -14.742902755737305,
      "KL/mean": -19.905223846435547,
      "KL/rejected_KL_mean": -25.067546844482422,
      "KL/std": 14.210126876831055,
      "epoch": 0.30839002267573695,
      "fcm_dpo/beta": 0.04956476390361786,
      "fcm_dpo/delta": -0.12190810590982437,
      "fcm_dpo/margin": 10.324640274047852,
      "fcm_dpo/q_t": 0.38725700974464417,
      "grad_norm": 17.99201202392578,
      "learning_rate": 4.380688857426449e-07,
      "logits/chosen": 0.3529035151004791,
      "logits/rejected": 0.28449898958206177,
      "logps/chosen": -65.37222290039062,
      "logps/ref_chosen": -50.62931823730469,
      "logps/ref_rejected": -66.60475158691406,
      "logps/rejected": -91.67230224609375,
      "loss": 1.0641,
      "margin_dpo/margin_mean": 10.324640274047852,
      "margin_dpo/margin_std": 14.458605766296387,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -16.37442398071289,
      "KL/mean": -21.547502517700195,
      "KL/rejected_KL_mean": -26.7205810546875,
      "KL/std": 14.305099487304688,
      "epoch": 0.30990173847316704,
      "fcm_dpo/beta": 0.04888454079627991,
      "fcm_dpo/delta": -0.11180345714092255,
      "fcm_dpo/margin": 10.346155166625977,
      "fcm_dpo/q_t": 0.38996249437332153,
      "grad_norm": 22.623018264770508,
      "learning_rate": 4.3719511720570814e-07,
      "logits/chosen": 0.4121706783771515,
      "logits/rejected": 0.34857797622680664,
      "logps/chosen": -86.73060607910156,
      "logps/ref_chosen": -70.3561782836914,
      "logps/ref_rejected": -93.39848327636719,
      "logps/rejected": -120.11906433105469,
      "loss": 1.0888,
      "margin_dpo/margin_mean": 10.346155166625977,
      "margin_dpo/margin_std": 16.493879318237305,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -17.43178939819336,
      "KL/mean": -21.19215202331543,
      "KL/rejected_KL_mean": -24.9525146484375,
      "KL/std": 14.40170669555664,
      "epoch": 0.31141345427059713,
      "fcm_dpo/beta": 0.049201615154743195,
      "fcm_dpo/delta": 0.030412331223487854,
      "fcm_dpo/margin": 7.52072286605835,
      "fcm_dpo/q_t": 0.42180708050727844,
      "grad_norm": 20.029573440551758,
      "learning_rate": 4.363161124189387e-07,
      "logits/chosen": 0.4177933931350708,
      "logits/rejected": 0.40190303325653076,
      "logps/chosen": -85.0772705078125,
      "logps/ref_chosen": -67.64547729492188,
      "logps/ref_rejected": -79.89584350585938,
      "logps/rejected": -104.84835815429688,
      "loss": 1.2222,
      "margin_dpo/margin_mean": 7.52072286605835,
      "margin_dpo/margin_std": 17.54438591003418,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -20.065610885620117,
      "KL/mean": -25.459985733032227,
      "KL/rejected_KL_mean": -30.854358673095703,
      "KL/std": 15.306570053100586,
      "epoch": 0.3129251700680272,
      "fcm_dpo/beta": 0.048022348433732986,
      "fcm_dpo/delta": -0.12489670515060425,
      "fcm_dpo/margin": 10.788747787475586,
      "fcm_dpo/q_t": 0.3892369568347931,
      "grad_norm": 18.09482765197754,
      "learning_rate": 4.3543189596998986e-07,
      "logits/chosen": 0.3574819564819336,
      "logits/rejected": 0.2902287244796753,
      "logps/chosen": -87.72980499267578,
      "logps/ref_chosen": -67.66419219970703,
      "logps/ref_rejected": -85.10249328613281,
      "logps/rejected": -115.95684814453125,
      "loss": 1.0639,
      "margin_dpo/margin_mean": 10.788747787475586,
      "margin_dpo/margin_std": 16.01801300048828,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -15.293437957763672,
      "KL/mean": -18.562654495239258,
      "KL/rejected_KL_mean": -21.831867218017578,
      "KL/std": 14.139419555664062,
      "epoch": 0.3144368858654573,
      "fcm_dpo/beta": 0.0484270378947258,
      "fcm_dpo/delta": 0.08609728515148163,
      "fcm_dpo/margin": 6.538424968719482,
      "fcm_dpo/q_t": 0.4310414791107178,
      "grad_norm": 21.07671356201172,
      "learning_rate": 4.3454249259229664e-07,
      "logits/chosen": 0.3809185326099396,
      "logits/rejected": 0.35520946979522705,
      "logps/chosen": -73.025146484375,
      "logps/ref_chosen": -57.731712341308594,
      "logps/ref_rejected": -74.19276428222656,
      "logps/rejected": -96.02462768554688,
      "loss": 1.2329,
      "margin_dpo/margin_mean": 6.538425445556641,
      "margin_dpo/margin_std": 15.793625831604004,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -15.615686416625977,
      "KL/mean": -22.246536254882812,
      "KL/rejected_KL_mean": -28.877395629882812,
      "KL/std": 16.236427307128906,
      "epoch": 0.31594860166288735,
      "fcm_dpo/beta": 0.04701051115989685,
      "fcm_dpo/delta": -0.23806017637252808,
      "fcm_dpo/margin": 13.261709213256836,
      "fcm_dpo/q_t": 0.3655932545661926,
      "grad_norm": 20.175548553466797,
      "learning_rate": 4.336479271643833e-07,
      "logits/chosen": 0.3561670184135437,
      "logits/rejected": 0.30066242814064026,
      "logps/chosen": -84.165771484375,
      "logps/ref_chosen": -68.55007934570312,
      "logps/ref_rejected": -87.90541076660156,
      "logps/rejected": -116.78280639648438,
      "loss": 1.0317,
      "margin_dpo/margin_mean": 13.261709213256836,
      "margin_dpo/margin_std": 19.039752960205078,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -15.559123992919922,
      "KL/mean": -21.865951538085938,
      "KL/rejected_KL_mean": -28.172779083251953,
      "KL/std": 17.181137084960938,
      "epoch": 0.31746031746031744,
      "fcm_dpo/beta": 0.04532770439982414,
      "fcm_dpo/delta": -0.18176668882369995,
      "fcm_dpo/margin": 12.613653182983398,
      "fcm_dpo/q_t": 0.3786957859992981,
      "grad_norm": 17.807655334472656,
      "learning_rate": 4.327482247091679e-07,
      "logits/chosen": 0.4548831582069397,
      "logits/rejected": 0.3575727939605713,
      "logps/chosen": -72.82740020751953,
      "logps/ref_chosen": -57.268272399902344,
      "logps/ref_rejected": -85.72807312011719,
      "logps/rejected": -113.9008560180664,
      "loss": 1.049,
      "margin_dpo/margin_mean": 12.613653182983398,
      "margin_dpo/margin_std": 18.374156951904297,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -13.578191757202148,
      "KL/mean": -19.461641311645508,
      "KL/rejected_KL_mean": -25.345096588134766,
      "KL/std": 15.827226638793945,
      "epoch": 0.31897203325774753,
      "fcm_dpo/beta": 0.04412417858839035,
      "fcm_dpo/delta": -0.1256234496831894,
      "fcm_dpo/margin": 11.766897201538086,
      "fcm_dpo/q_t": 0.38870713114738464,
      "grad_norm": 20.859329223632812,
      "learning_rate": 4.3184341039326217e-07,
      "logits/chosen": 0.45018890500068665,
      "logits/rejected": 0.35748744010925293,
      "logps/chosen": -67.21890258789062,
      "logps/ref_chosen": -53.640708923339844,
      "logps/ref_rejected": -93.0387954711914,
      "logps/rejected": -118.38389587402344,
      "loss": 1.064,
      "margin_dpo/margin_mean": 11.766897201538086,
      "margin_dpo/margin_std": 17.286218643188477,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -16.12069320678711,
      "KL/mean": -22.743148803710938,
      "KL/rejected_KL_mean": -29.365604400634766,
      "KL/std": 17.04791259765625,
      "epoch": 0.3204837490551776,
      "fcm_dpo/beta": 0.04247160255908966,
      "fcm_dpo/delta": -0.1724153459072113,
      "fcm_dpo/margin": 13.244913101196289,
      "fcm_dpo/q_t": 0.3759151101112366,
      "grad_norm": 15.934540748596191,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": 0.4377868175506592,
      "logits/rejected": 0.36682993173599243,
      "logps/chosen": -73.48743438720703,
      "logps/ref_chosen": -57.36674499511719,
      "logps/ref_rejected": -79.89643096923828,
      "logps/rejected": -109.26203918457031,
      "loss": 1.0409,
      "margin_dpo/margin_mean": 13.244912147521973,
      "margin_dpo/margin_std": 18.538911819458008,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -11.125129699707031,
      "KL/mean": -18.71963882446289,
      "KL/rejected_KL_mean": -26.31414794921875,
      "KL/std": 17.344621658325195,
      "epoch": 0.3219954648526077,
      "fcm_dpo/beta": 0.04053671658039093,
      "fcm_dpo/delta": -0.23129788041114807,
      "fcm_dpo/margin": 15.189022064208984,
      "fcm_dpo/q_t": 0.3656018376350403,
      "grad_norm": 14.400700569152832,
      "learning_rate": 4.3001854756006724e-07,
      "logits/chosen": 0.4481104016304016,
      "logits/rejected": 0.4241155683994293,
      "logps/chosen": -76.34624481201172,
      "logps/ref_chosen": -65.22111511230469,
      "logps/ref_rejected": -80.1810302734375,
      "logps/rejected": -106.49517822265625,
      "loss": 1.0033,
      "margin_dpo/margin_mean": 15.189022064208984,
      "margin_dpo/margin_std": 19.745624542236328,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -13.558965682983398,
      "KL/mean": -21.037841796875,
      "KL/rejected_KL_mean": -28.516714096069336,
      "KL/std": 17.805932998657227,
      "epoch": 0.3235071806500378,
      "fcm_dpo/beta": 0.03931838646531105,
      "fcm_dpo/delta": -0.19944192469120026,
      "fcm_dpo/margin": 14.957748413085938,
      "fcm_dpo/q_t": 0.37324780225753784,
      "grad_norm": 19.317140579223633,
      "learning_rate": 4.290985500881143e-07,
      "logits/chosen": 0.32487252354621887,
      "logits/rejected": 0.3027455508708954,
      "logps/chosen": -74.8512954711914,
      "logps/ref_chosen": -61.292327880859375,
      "logps/ref_rejected": -67.69841003417969,
      "logps/rejected": -96.21511840820312,
      "loss": 1.0322,
      "margin_dpo/margin_mean": 14.957748413085938,
      "margin_dpo/margin_std": 20.531984329223633,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -17.458284378051758,
      "KL/mean": -25.071035385131836,
      "KL/rejected_KL_mean": -32.68378448486328,
      "KL/std": 18.632884979248047,
      "epoch": 0.3250188964474679,
      "fcm_dpo/beta": 0.037889935076236725,
      "fcm_dpo/delta": -0.18748575448989868,
      "fcm_dpo/margin": 15.225502967834473,
      "fcm_dpo/q_t": 0.3767107129096985,
      "grad_norm": 17.0438175201416,
      "learning_rate": 4.281735428447157e-07,
      "logits/chosen": 0.3328137993812561,
      "logits/rejected": 0.22789113223552704,
      "logps/chosen": -81.32742309570312,
      "logps/ref_chosen": -63.869136810302734,
      "logps/ref_rejected": -98.7657241821289,
      "logps/rejected": -131.4495086669922,
      "loss": 1.0456,
      "margin_dpo/margin_mean": 15.225502967834473,
      "margin_dpo/margin_std": 21.498851776123047,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -15.768959045410156,
      "KL/mean": -23.88794708251953,
      "KL/rejected_KL_mean": -32.00693130493164,
      "KL/std": 19.378738403320312,
      "epoch": 0.32653061224489793,
      "fcm_dpo/beta": 0.036197736859321594,
      "fcm_dpo/delta": -0.1992907077074051,
      "fcm_dpo/margin": 16.23797035217285,
      "fcm_dpo/q_t": 0.3708665370941162,
      "grad_norm": 20.10349464416504,
      "learning_rate": 4.2724355170431247e-07,
      "logits/chosen": 0.49180224537849426,
      "logits/rejected": 0.40338221192359924,
      "logps/chosen": -83.59391784667969,
      "logps/ref_chosen": -67.824951171875,
      "logps/ref_rejected": -96.40231323242188,
      "logps/rejected": -128.40924072265625,
      "loss": 1.0051,
      "margin_dpo/margin_mean": 16.23796844482422,
      "margin_dpo/margin_std": 20.70318031311035,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -18.353816986083984,
      "KL/mean": -26.814929962158203,
      "KL/rejected_KL_mean": -35.27604675292969,
      "KL/std": 20.04430389404297,
      "epoch": 0.328042328042328,
      "fcm_dpo/beta": 0.0346650592982769,
      "fcm_dpo/delta": -0.19878257811069489,
      "fcm_dpo/margin": 16.92223358154297,
      "fcm_dpo/q_t": 0.3714277148246765,
      "grad_norm": 15.19613265991211,
      "learning_rate": 4.26308602680756e-07,
      "logits/chosen": 0.41574960947036743,
      "logits/rejected": 0.3108539581298828,
      "logps/chosen": -78.85881042480469,
      "logps/ref_chosen": -60.5049934387207,
      "logps/ref_rejected": -84.26618194580078,
      "logps/rejected": -119.54222869873047,
      "loss": 1.0045,
      "margin_dpo/margin_mean": 16.92223358154297,
      "margin_dpo/margin_std": 21.368816375732422,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -19.374526977539062,
      "KL/mean": -24.803550720214844,
      "KL/rejected_KL_mean": -30.232582092285156,
      "KL/std": 18.930479049682617,
      "epoch": 0.3295540438397581,
      "fcm_dpo/beta": 0.03415830060839653,
      "fcm_dpo/delta": -0.06700804829597473,
      "fcm_dpo/margin": 10.85805606842041,
      "fcm_dpo/q_t": 0.4185621738433838,
      "grad_norm": 17.009702682495117,
      "learning_rate": 4.253687219265803e-07,
      "logits/chosen": 0.31673234701156616,
      "logits/rejected": 0.3123531937599182,
      "logps/chosen": -89.96884155273438,
      "logps/ref_chosen": -70.59431457519531,
      "logps/ref_rejected": -73.89038848876953,
      "logps/rejected": -104.12297058105469,
      "loss": 1.205,
      "margin_dpo/margin_mean": 10.858057022094727,
      "margin_dpo/margin_std": 23.331634521484375,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -18.319480895996094,
      "KL/mean": -24.112728118896484,
      "KL/rejected_KL_mean": -29.905981063842773,
      "KL/std": 20.181316375732422,
      "epoch": 0.3310657596371882,
      "fcm_dpo/beta": 0.03382644057273865,
      "fcm_dpo/delta": 0.008064381778240204,
      "fcm_dpo/margin": 11.586502075195312,
      "fcm_dpo/q_t": 0.41295433044433594,
      "grad_norm": 17.34720230102539,
      "learning_rate": 4.2442393573227043e-07,
      "logits/chosen": 0.3763273358345032,
      "logits/rejected": 0.3354039788246155,
      "logps/chosen": -78.8104248046875,
      "logps/ref_chosen": -60.490943908691406,
      "logps/ref_rejected": -75.85001373291016,
      "logps/rejected": -105.75599670410156,
      "loss": 1.1397,
      "margin_dpo/margin_mean": 11.586501121520996,
      "margin_dpo/margin_std": 19.970802307128906,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -15.745569229125977,
      "KL/mean": -22.564374923706055,
      "KL/rejected_KL_mean": -29.383182525634766,
      "KL/std": 20.509496688842773,
      "epoch": 0.3325774754346183,
      "fcm_dpo/beta": 0.03348580747842789,
      "fcm_dpo/delta": -0.06041298806667328,
      "fcm_dpo/margin": 13.637612342834473,
      "fcm_dpo/q_t": 0.4020610749721527,
      "grad_norm": 14.368889808654785,
      "learning_rate": 4.234742705255272e-07,
      "logits/chosen": 0.4836348295211792,
      "logits/rejected": 0.42029207944869995,
      "logps/chosen": -60.758968353271484,
      "logps/ref_chosen": -45.013397216796875,
      "logps/ref_rejected": -70.49369812011719,
      "logps/rejected": -99.87688446044922,
      "loss": 1.1246,
      "margin_dpo/margin_mean": 13.637613296508789,
      "margin_dpo/margin_std": 23.508586883544922,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -15.578283309936523,
      "KL/mean": -23.014617919921875,
      "KL/rejected_KL_mean": -30.450958251953125,
      "KL/std": 20.340774536132812,
      "epoch": 0.3340891912320484,
      "fcm_dpo/beta": 0.03313559293746948,
      "fcm_dpo/delta": -0.09753476083278656,
      "fcm_dpo/margin": 14.872676849365234,
      "fcm_dpo/q_t": 0.39384615421295166,
      "grad_norm": 16.941404342651367,
      "learning_rate": 4.22519752870528e-07,
      "logits/chosen": 0.45701926946640015,
      "logits/rejected": 0.38429608941078186,
      "logps/chosen": -74.67411804199219,
      "logps/ref_chosen": -59.09584045410156,
      "logps/ref_rejected": -88.64388275146484,
      "logps/rejected": -119.09484100341797,
      "loss": 1.0863,
      "margin_dpo/margin_mean": 14.872674942016602,
      "margin_dpo/margin_std": 23.3742733001709,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -16.549724578857422,
      "KL/mean": -26.46912956237793,
      "KL/rejected_KL_mean": -36.38853454589844,
      "KL/std": 22.84616470336914,
      "epoch": 0.3356009070294785,
      "fcm_dpo/beta": 0.03170529007911682,
      "fcm_dpo/delta": -0.24479737877845764,
      "fcm_dpo/margin": 19.838809967041016,
      "fcm_dpo/q_t": 0.36175861954689026,
      "grad_norm": 16.72490119934082,
      "learning_rate": 4.2156040946718343e-07,
      "logits/chosen": 0.48086023330688477,
      "logits/rejected": 0.39596283435821533,
      "logps/chosen": -72.54741668701172,
      "logps/ref_chosen": -55.9976921081543,
      "logps/ref_rejected": -111.94727325439453,
      "logps/rejected": -148.3358154296875,
      "loss": 0.993,
      "margin_dpo/margin_mean": 19.838809967041016,
      "margin_dpo/margin_std": 24.805423736572266,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -19.845706939697266,
      "KL/mean": -28.89853858947754,
      "KL/rejected_KL_mean": -37.95137405395508,
      "KL/std": 22.876976013183594,
      "epoch": 0.3371126228269085,
      "fcm_dpo/beta": 0.030458718538284302,
      "fcm_dpo/delta": -0.16162584722042084,
      "fcm_dpo/margin": 18.105667114257812,
      "fcm_dpo/q_t": 0.3780610263347626,
      "grad_norm": 15.224184036254883,
      "learning_rate": 4.2059626715039065e-07,
      "logits/chosen": 0.4890958368778229,
      "logits/rejected": 0.43254202604293823,
      "logps/chosen": -79.73712921142578,
      "logps/ref_chosen": -59.891422271728516,
      "logps/ref_rejected": -86.28954315185547,
      "logps/rejected": -124.24092102050781,
      "loss": 1.0133,
      "margin_dpo/margin_mean": 18.105669021606445,
      "margin_dpo/margin_std": 22.11848258972168,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -23.249916076660156,
      "KL/mean": -28.278770446777344,
      "KL/rejected_KL_mean": -33.307621002197266,
      "KL/std": 22.561180114746094,
      "epoch": 0.3386243386243386,
      "fcm_dpo/beta": 0.030742764472961426,
      "fcm_dpo/delta": 0.0937860757112503,
      "fcm_dpo/margin": 10.057705879211426,
      "fcm_dpo/q_t": 0.43336811661720276,
      "grad_norm": 20.546825408935547,
      "learning_rate": 4.1962735288928304e-07,
      "logits/chosen": 0.5040819644927979,
      "logits/rejected": 0.48309725522994995,
      "logps/chosen": -87.2945556640625,
      "logps/ref_chosen": -64.04463195800781,
      "logps/ref_rejected": -75.05450439453125,
      "logps/rejected": -108.36212158203125,
      "loss": 1.2213,
      "margin_dpo/margin_mean": 10.05770492553711,
      "margin_dpo/margin_std": 23.49422264099121,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -21.998910903930664,
      "KL/mean": -31.384471893310547,
      "KL/rejected_KL_mean": -40.7700309753418,
      "KL/std": 25.406606674194336,
      "epoch": 0.3401360544217687,
      "fcm_dpo/beta": 0.029996603727340698,
      "fcm_dpo/delta": -0.1741228997707367,
      "fcm_dpo/margin": 18.7711181640625,
      "fcm_dpo/q_t": 0.3778340220451355,
      "grad_norm": 16.333887100219727,
      "learning_rate": 4.186536937864752e-07,
      "logits/chosen": 0.5071430802345276,
      "logits/rejected": 0.39010632038116455,
      "logps/chosen": -88.09473419189453,
      "logps/ref_chosen": -66.0958251953125,
      "logps/ref_rejected": -97.68675231933594,
      "logps/rejected": -138.456787109375,
      "loss": 1.0246,
      "margin_dpo/margin_mean": 18.7711181640625,
      "margin_dpo/margin_std": 24.778152465820312,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -20.758808135986328,
      "KL/mean": -28.343278884887695,
      "KL/rejected_KL_mean": -35.9277458190918,
      "KL/std": 24.265933990478516,
      "epoch": 0.3416477702191988,
      "fcm_dpo/beta": 0.029768429696559906,
      "fcm_dpo/delta": -0.05416828766465187,
      "fcm_dpo/margin": 15.168935775756836,
      "fcm_dpo/q_t": 0.4019937515258789,
      "grad_norm": 15.020020484924316,
      "learning_rate": 4.176753170773052e-07,
      "logits/chosen": 0.5398536920547485,
      "logits/rejected": 0.4917876124382019,
      "logps/chosen": -72.1756820678711,
      "logps/ref_chosen": -51.4168701171875,
      "logps/ref_rejected": -66.30068969726562,
      "logps/rejected": -102.22843170166016,
      "loss": 1.1472,
      "margin_dpo/margin_mean": 15.168935775756836,
      "margin_dpo/margin_std": 28.176733016967773,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -22.506391525268555,
      "KL/mean": -30.736263275146484,
      "KL/rejected_KL_mean": -38.966129302978516,
      "KL/std": 26.21303939819336,
      "epoch": 0.3431594860166289,
      "fcm_dpo/beta": 0.02910151518881321,
      "fcm_dpo/delta": -0.08339697122573853,
      "fcm_dpo/margin": 16.45973777770996,
      "fcm_dpo/q_t": 0.39864617586135864,
      "grad_norm": 16.029760360717773,
      "learning_rate": 4.166922501290729e-07,
      "logits/chosen": 0.5619155168533325,
      "logits/rejected": 0.522531270980835,
      "logps/chosen": -80.49617004394531,
      "logps/ref_chosen": -57.989776611328125,
      "logps/ref_rejected": -75.05464172363281,
      "logps/rejected": -114.0207748413086,
      "loss": 1.124,
      "margin_dpo/margin_mean": 16.459735870361328,
      "margin_dpo/margin_std": 28.955650329589844,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -25.3358154296875,
      "KL/mean": -33.85264587402344,
      "KL/rejected_KL_mean": -42.36947250366211,
      "KL/std": 25.331405639648438,
      "epoch": 0.34467120181405897,
      "fcm_dpo/beta": 0.028743447735905647,
      "fcm_dpo/delta": -0.09411942958831787,
      "fcm_dpo/margin": 17.033653259277344,
      "fcm_dpo/q_t": 0.39314448833465576,
      "grad_norm": 16.72762107849121,
      "learning_rate": 4.1570452044027405e-07,
      "logits/chosen": 0.5379786491394043,
      "logits/rejected": 0.45798879861831665,
      "logps/chosen": -80.89518737792969,
      "logps/ref_chosen": -55.55936813354492,
      "logps/ref_rejected": -77.02364349365234,
      "logps/rejected": -119.39311218261719,
      "loss": 1.0818,
      "margin_dpo/margin_mean": 17.033653259277344,
      "margin_dpo/margin_std": 25.905319213867188,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -20.280107498168945,
      "KL/mean": -28.22270393371582,
      "KL/rejected_KL_mean": -36.16529846191406,
      "KL/std": 24.416088104248047,
      "epoch": 0.34618291761148906,
      "fcm_dpo/beta": 0.028269220143556595,
      "fcm_dpo/delta": -0.051458459347486496,
      "fcm_dpo/margin": 15.88519287109375,
      "fcm_dpo/q_t": 0.4015154242515564,
      "grad_norm": 28.57234764099121,
      "learning_rate": 4.147121556398312e-07,
      "logits/chosen": 0.635587215423584,
      "logits/rejected": 0.5650753974914551,
      "logps/chosen": -71.07476806640625,
      "logps/ref_chosen": -50.79466247558594,
      "logps/ref_rejected": -78.4474105834961,
      "logps/rejected": -114.61270904541016,
      "loss": 1.1506,
      "margin_dpo/margin_mean": 15.885190963745117,
      "margin_dpo/margin_std": 30.102184295654297,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -23.39543914794922,
      "KL/mean": -32.73040771484375,
      "KL/rejected_KL_mean": -42.06538009643555,
      "KL/std": 27.682418823242188,
      "epoch": 0.3476946334089191,
      "fcm_dpo/beta": 0.027965370565652847,
      "fcm_dpo/delta": -0.12953221797943115,
      "fcm_dpo/margin": 18.66994285583496,
      "fcm_dpo/q_t": 0.38660961389541626,
      "grad_norm": 16.53853988647461,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 0.515990138053894,
      "logits/rejected": 0.5156873464584351,
      "logps/chosen": -80.12466430664062,
      "logps/ref_chosen": -56.729225158691406,
      "logps/ref_rejected": -62.99180603027344,
      "logps/rejected": -105.05718994140625,
      "loss": 1.0596,
      "margin_dpo/margin_mean": 18.669940948486328,
      "margin_dpo/margin_std": 26.313983917236328,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -28.046634674072266,
      "KL/mean": -41.93617248535156,
      "KL/rejected_KL_mean": -55.825721740722656,
      "KL/std": 27.473129272460938,
      "epoch": 0.3492063492063492,
      "fcm_dpo/beta": 0.02596151828765869,
      "fcm_dpo/delta": -0.349088579416275,
      "fcm_dpo/margin": 27.77908706665039,
      "fcm_dpo/q_t": 0.3383832573890686,
      "grad_norm": 16.58981704711914,
      "learning_rate": 4.1271363186719835e-07,
      "logits/chosen": 0.45076966285705566,
      "logits/rejected": 0.441531240940094,
      "logps/chosen": -100.64373779296875,
      "logps/ref_chosen": -72.59709930419922,
      "logps/ref_rejected": -86.2322998046875,
      "logps/rejected": -142.05801391601562,
      "loss": 0.9186,
      "margin_dpo/margin_mean": 27.77908706665039,
      "margin_dpo/margin_std": 28.995311737060547,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -28.168758392333984,
      "KL/mean": -37.593505859375,
      "KL/rejected_KL_mean": -47.01825714111328,
      "KL/std": 29.821605682373047,
      "epoch": 0.3507180650037793,
      "fcm_dpo/beta": 0.025313373655080795,
      "fcm_dpo/delta": -0.08113664388656616,
      "fcm_dpo/margin": 18.84949493408203,
      "fcm_dpo/q_t": 0.39923810958862305,
      "grad_norm": 15.988265037536621,
      "learning_rate": 4.1170752879801436e-07,
      "logits/chosen": 0.46044355630874634,
      "logits/rejected": 0.4326399564743042,
      "logps/chosen": -96.28729248046875,
      "logps/ref_chosen": -68.1185302734375,
      "logps/ref_rejected": -83.79415893554688,
      "logps/rejected": -130.8124237060547,
      "loss": 1.1242,
      "margin_dpo/margin_mean": 18.84949493408203,
      "margin_dpo/margin_std": 33.42100143432617,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -33.43086242675781,
      "KL/mean": -40.904014587402344,
      "KL/rejected_KL_mean": -48.37717056274414,
      "KL/std": 28.984731674194336,
      "epoch": 0.35222978080120937,
      "fcm_dpo/beta": 0.024750979617238045,
      "fcm_dpo/delta": -0.09754282236099243,
      "fcm_dpo/margin": 14.946308135986328,
      "fcm_dpo/q_t": 0.4192150831222534,
      "grad_norm": 15.90912914276123,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": 0.5378991365432739,
      "logits/rejected": 0.480247437953949,
      "logps/chosen": -88.50101470947266,
      "logps/ref_chosen": -55.070152282714844,
      "logps/ref_rejected": -66.61845397949219,
      "logps/rejected": -114.99562072753906,
      "loss": 1.1719,
      "margin_dpo/margin_mean": 14.946308135986328,
      "margin_dpo/margin_std": 28.04265785217285,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -30.95333480834961,
      "KL/mean": -38.52400207519531,
      "KL/rejected_KL_mean": -46.09467697143555,
      "KL/std": 28.024137496948242,
      "epoch": 0.35374149659863946,
      "fcm_dpo/beta": 0.024464137852191925,
      "fcm_dpo/delta": -0.1172548457980156,
      "fcm_dpo/margin": 15.141345977783203,
      "fcm_dpo/q_t": 0.4203869700431824,
      "grad_norm": 18.991703033447266,
      "learning_rate": 4.09681781007452e-07,
      "logits/chosen": 0.47061771154403687,
      "logits/rejected": 0.46105387806892395,
      "logps/chosen": -86.87922668457031,
      "logps/ref_chosen": -55.92589569091797,
      "logps/ref_rejected": -51.11608123779297,
      "logps/rejected": -97.21075439453125,
      "loss": 1.1982,
      "margin_dpo/margin_mean": 15.141345977783203,
      "margin_dpo/margin_std": 30.72395133972168,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -25.921781539916992,
      "KL/mean": -38.711524963378906,
      "KL/rejected_KL_mean": -51.50127410888672,
      "KL/std": 30.172622680664062,
      "epoch": 0.35525321239606955,
      "fcm_dpo/beta": 0.02348637580871582,
      "fcm_dpo/delta": -0.21319061517715454,
      "fcm_dpo/margin": 25.579490661621094,
      "fcm_dpo/q_t": 0.3651096224784851,
      "grad_norm": 15.18369197845459,
      "learning_rate": 4.08662192950594e-07,
      "logits/chosen": 0.5647023916244507,
      "logits/rejected": 0.5489069223403931,
      "logps/chosen": -90.46150207519531,
      "logps/ref_chosen": -64.53972625732422,
      "logps/ref_rejected": -77.69151306152344,
      "logps/rejected": -129.19277954101562,
      "loss": 0.9831,
      "margin_dpo/margin_mean": 25.579490661621094,
      "margin_dpo/margin_std": 29.648242950439453,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -41.41307830810547,
      "KL/mean": -51.20012283325195,
      "KL/rejected_KL_mean": -60.98716735839844,
      "KL/std": 30.337989807128906,
      "epoch": 0.35676492819349964,
      "fcm_dpo/beta": 0.02293534204363823,
      "fcm_dpo/delta": -0.051485203206539154,
      "fcm_dpo/margin": 19.57408905029297,
      "fcm_dpo/q_t": 0.4013304114341736,
      "grad_norm": 14.13412094116211,
      "learning_rate": 4.076381667711306e-07,
      "logits/chosen": 0.5349459648132324,
      "logits/rejected": 0.5214509963989258,
      "logps/chosen": -112.56781768798828,
      "logps/ref_chosen": -71.15473937988281,
      "logps/ref_rejected": -84.88541412353516,
      "logps/rejected": -145.87258911132812,
      "loss": 1.1275,
      "margin_dpo/margin_mean": 19.57408905029297,
      "margin_dpo/margin_std": 34.05792999267578,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -36.039955139160156,
      "KL/mean": -47.605323791503906,
      "KL/rejected_KL_mean": -59.170692443847656,
      "KL/std": 30.790592193603516,
      "epoch": 0.35827664399092973,
      "fcm_dpo/beta": 0.02254084311425686,
      "fcm_dpo/delta": -0.12789805233478546,
      "fcm_dpo/margin": 23.130735397338867,
      "fcm_dpo/q_t": 0.3853898048400879,
      "grad_norm": 17.299875259399414,
      "learning_rate": 4.066097311132753e-07,
      "logits/chosen": 0.5635801553726196,
      "logits/rejected": 0.5527620315551758,
      "logps/chosen": -112.18196868896484,
      "logps/ref_chosen": -76.14201354980469,
      "logps/ref_rejected": -80.88479614257812,
      "logps/rejected": -140.05548095703125,
      "loss": 1.074,
      "margin_dpo/margin_mean": 23.130735397338867,
      "margin_dpo/margin_std": 34.4949951171875,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -33.41836929321289,
      "KL/mean": -44.90141296386719,
      "KL/rejected_KL_mean": -56.38445281982422,
      "KL/std": 32.43263244628906,
      "epoch": 0.35978835978835977,
      "fcm_dpo/beta": 0.021894235163927078,
      "fcm_dpo/delta": -0.10915926098823547,
      "fcm_dpo/margin": 22.96609115600586,
      "fcm_dpo/q_t": 0.3888513445854187,
      "grad_norm": 21.488059997558594,
      "learning_rate": 4.0557691474458414e-07,
      "logits/chosen": 0.501011073589325,
      "logits/rejected": 0.4915581941604614,
      "logps/chosen": -102.30322265625,
      "logps/ref_chosen": -68.88484954833984,
      "logps/ref_rejected": -75.8946304321289,
      "logps/rejected": -132.27908325195312,
      "loss": 1.0694,
      "margin_dpo/margin_mean": 22.96609115600586,
      "margin_dpo/margin_std": 33.38800811767578,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -40.47924041748047,
      "KL/mean": -51.81416320800781,
      "KL/rejected_KL_mean": -63.149078369140625,
      "KL/std": 34.226661682128906,
      "epoch": 0.36130007558578986,
      "fcm_dpo/beta": 0.021715857088565826,
      "fcm_dpo/delta": -0.09747522324323654,
      "fcm_dpo/margin": 22.669843673706055,
      "fcm_dpo/q_t": 0.3927465081214905,
      "grad_norm": 17.716434478759766,
      "learning_rate": 4.045397465551513e-07,
      "logits/chosen": 0.6732344627380371,
      "logits/rejected": 0.539535403251648,
      "logps/chosen": -97.25106811523438,
      "logps/ref_chosen": -56.771827697753906,
      "logps/ref_rejected": -116.23050689697266,
      "logps/rejected": -179.37957763671875,
      "loss": 1.0906,
      "margin_dpo/margin_mean": 22.669845581054688,
      "margin_dpo/margin_std": 35.027000427246094,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -39.25141143798828,
      "KL/mean": -54.110137939453125,
      "KL/rejected_KL_mean": -68.96885681152344,
      "KL/std": 34.705718994140625,
      "epoch": 0.36281179138321995,
      "fcm_dpo/beta": 0.020736213773489,
      "fcm_dpo/delta": -0.2303335964679718,
      "fcm_dpo/margin": 29.717445373535156,
      "fcm_dpo/q_t": 0.3637624979019165,
      "grad_norm": 13.503387451171875,
      "learning_rate": 4.0349825555680045e-07,
      "logits/chosen": 0.5842655897140503,
      "logits/rejected": 0.48873424530029297,
      "logps/chosen": -92.60552215576172,
      "logps/ref_chosen": -53.35411071777344,
      "logps/ref_rejected": -80.12019348144531,
      "logps/rejected": -149.08905029296875,
      "loss": 0.9824,
      "margin_dpo/margin_mean": 29.717445373535156,
      "margin_dpo/margin_std": 35.28871154785156,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -39.388832092285156,
      "KL/mean": -49.43465042114258,
      "KL/rejected_KL_mean": -59.480464935302734,
      "KL/std": 32.912261962890625,
      "epoch": 0.36432350718065004,
      "fcm_dpo/beta": 0.020505176857113838,
      "fcm_dpo/delta": -0.012648653239011765,
      "fcm_dpo/margin": 20.091632843017578,
      "fcm_dpo/q_t": 0.40955421328544617,
      "grad_norm": 16.11968421936035,
      "learning_rate": 4.0245247088227377e-07,
      "logits/chosen": 0.5211039781570435,
      "logits/rejected": 0.485470712184906,
      "logps/chosen": -111.28424072265625,
      "logps/ref_chosen": -71.89541625976562,
      "logps/ref_rejected": -83.03492736816406,
      "logps/rejected": -142.51539611816406,
      "loss": 1.1351,
      "margin_dpo/margin_mean": 20.091632843017578,
      "margin_dpo/margin_std": 34.95091247558594,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -40.57394790649414,
      "KL/mean": -54.1338996887207,
      "KL/rejected_KL_mean": -67.69384765625,
      "KL/std": 35.485565185546875,
      "epoch": 0.36583522297808013,
      "fcm_dpo/beta": 0.01981888711452484,
      "fcm_dpo/delta": -0.14796458184719086,
      "fcm_dpo/margin": 27.119897842407227,
      "fcm_dpo/q_t": 0.38168632984161377,
      "grad_norm": 12.919242858886719,
      "learning_rate": 4.0140242178441665e-07,
      "logits/chosen": 0.5120102167129517,
      "logits/rejected": 0.4930839240550995,
      "logps/chosen": -98.50138092041016,
      "logps/ref_chosen": -57.927433013916016,
      "logps/ref_rejected": -67.838623046875,
      "logps/rejected": -135.532470703125,
      "loss": 1.0436,
      "margin_dpo/margin_mean": 27.119895935058594,
      "margin_dpo/margin_std": 36.915733337402344,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -40.39060592651367,
      "KL/mean": -52.39445495605469,
      "KL/rejected_KL_mean": -64.39830017089844,
      "KL/std": 35.495384216308594,
      "epoch": 0.3673469387755102,
      "fcm_dpo/beta": 0.019632235169410706,
      "fcm_dpo/delta": -0.07480161637067795,
      "fcm_dpo/margin": 24.007692337036133,
      "fcm_dpo/q_t": 0.3948795199394226,
      "grad_norm": 16.458721160888672,
      "learning_rate": 4.003481376353596e-07,
      "logits/chosen": 0.5681760311126709,
      "logits/rejected": 0.574451744556427,
      "logps/chosen": -114.66728210449219,
      "logps/ref_chosen": -74.27667236328125,
      "logps/ref_rejected": -73.24340057373047,
      "logps/rejected": -137.64169311523438,
      "loss": 1.0823,
      "margin_dpo/margin_mean": 24.007692337036133,
      "margin_dpo/margin_std": 35.69834518432617,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -40.54975891113281,
      "KL/mean": -56.06737518310547,
      "KL/rejected_KL_mean": -71.58499145507812,
      "KL/std": 34.8726806640625,
      "epoch": 0.3688586545729403,
      "fcm_dpo/beta": 0.019023999571800232,
      "fcm_dpo/delta": -0.20192870497703552,
      "fcm_dpo/margin": 31.035232543945312,
      "fcm_dpo/q_t": 0.367572546005249,
      "grad_norm": 15.303215026855469,
      "learning_rate": 3.9928964792569654e-07,
      "logits/chosen": 0.6046304106712341,
      "logits/rejected": 0.5188884735107422,
      "logps/chosen": -93.91366577148438,
      "logps/ref_chosen": -53.36390686035156,
      "logps/ref_rejected": -71.10276794433594,
      "logps/rejected": -142.68777465820312,
      "loss": 0.9804,
      "margin_dpo/margin_mean": 31.035232543945312,
      "margin_dpo/margin_std": 34.62377166748047,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -42.96038055419922,
      "KL/mean": -60.99906539916992,
      "KL/rejected_KL_mean": -79.03775024414062,
      "KL/std": 36.55558776855469,
      "epoch": 0.37037037037037035,
      "fcm_dpo/beta": 0.018024669960141182,
      "fcm_dpo/delta": -0.26787251234054565,
      "fcm_dpo/margin": 36.07737350463867,
      "fcm_dpo/q_t": 0.35281607508659363,
      "grad_norm": 20.350332260131836,
      "learning_rate": 3.982269822636601e-07,
      "logits/chosen": 0.6260539293289185,
      "logits/rejected": 0.6000999808311462,
      "logps/chosen": -114.15548706054688,
      "logps/ref_chosen": -71.19510650634766,
      "logps/ref_rejected": -80.76235961914062,
      "logps/rejected": -159.80010986328125,
      "loss": 0.9344,
      "margin_dpo/margin_mean": 36.077369689941406,
      "margin_dpo/margin_std": 36.25225067138672,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -50.89258575439453,
      "KL/mean": -65.32125854492188,
      "KL/rejected_KL_mean": -79.74992370605469,
      "KL/std": 36.9205322265625,
      "epoch": 0.37188208616780044,
      "fcm_dpo/beta": 0.017585981637239456,
      "fcm_dpo/delta": -0.11321959644556046,
      "fcm_dpo/margin": 28.857349395751953,
      "fcm_dpo/q_t": 0.3887024521827698,
      "grad_norm": 15.056567192077637,
      "learning_rate": 3.971601703742932e-07,
      "logits/chosen": 0.6686552166938782,
      "logits/rejected": 0.6054153442382812,
      "logps/chosen": -122.51363372802734,
      "logps/ref_chosen": -71.62104797363281,
      "logps/ref_rejected": -94.03392028808594,
      "logps/rejected": -173.78384399414062,
      "loss": 1.0857,
      "margin_dpo/margin_mean": 28.85734748840332,
      "margin_dpo/margin_std": 44.590126037597656,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -56.60285186767578,
      "KL/mean": -64.69072723388672,
      "KL/rejected_KL_mean": -72.77860260009766,
      "KL/std": 36.552886962890625,
      "epoch": 0.37339380196523053,
      "fcm_dpo/beta": 0.017424512654542923,
      "fcm_dpo/delta": 0.020572219043970108,
      "fcm_dpo/margin": 16.175758361816406,
      "fcm_dpo/q_t": 0.4364478886127472,
      "grad_norm": 17.090055465698242,
      "learning_rate": 3.960892420986177e-07,
      "logits/chosen": 0.6290233731269836,
      "logits/rejected": 0.6191028356552124,
      "logps/chosen": -136.62539672851562,
      "logps/ref_chosen": -80.02254486083984,
      "logps/ref_rejected": -89.22705841064453,
      "logps/rejected": -162.0056610107422,
      "loss": 1.2325,
      "margin_dpo/margin_mean": 16.175758361816406,
      "margin_dpo/margin_std": 38.545249938964844,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -47.576560974121094,
      "KL/mean": -62.861209869384766,
      "KL/rejected_KL_mean": -78.14585876464844,
      "KL/std": 40.5474739074707,
      "epoch": 0.3749055177626606,
      "fcm_dpo/beta": 0.017187952995300293,
      "fcm_dpo/delta": -0.1321752369403839,
      "fcm_dpo/margin": 30.569297790527344,
      "fcm_dpo/q_t": 0.3867141902446747,
      "grad_norm": 14.95384407043457,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": 0.6109728813171387,
      "logits/rejected": 0.6641882061958313,
      "logps/chosen": -112.95452117919922,
      "logps/ref_chosen": -65.37796020507812,
      "logps/ref_rejected": -61.365787506103516,
      "logps/rejected": -139.5116424560547,
      "loss": 1.0665,
      "margin_dpo/margin_mean": 30.56929588317871,
      "margin_dpo/margin_std": 45.22699737548828,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -61.29655838012695,
      "KL/mean": -67.18286895751953,
      "KL/rejected_KL_mean": -73.06918334960938,
      "KL/std": 38.33504867553711,
      "epoch": 0.3764172335600907,
      "fcm_dpo/beta": 0.017067905515432358,
      "fcm_dpo/delta": 0.036949530243873596,
      "fcm_dpo/margin": 11.772629737854004,
      "fcm_dpo/q_t": 0.45426398515701294,
      "grad_norm": 18.01552963256836,
      "learning_rate": 3.9393515632731094e-07,
      "logits/chosen": 0.5990445613861084,
      "logits/rejected": 0.637617290019989,
      "logps/chosen": -135.89801025390625,
      "logps/ref_chosen": -74.60145568847656,
      "logps/ref_rejected": -63.79338455200195,
      "logps/rejected": -136.86256408691406,
      "loss": 1.3192,
      "margin_dpo/margin_mean": 11.772629737854004,
      "margin_dpo/margin_std": 41.29820251464844,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -52.64606475830078,
      "KL/mean": -67.76353454589844,
      "KL/rejected_KL_mean": -82.88101196289062,
      "KL/std": 40.518585205078125,
      "epoch": 0.3779289493575208,
      "fcm_dpo/beta": 0.016897017136216164,
      "fcm_dpo/delta": -0.11669476330280304,
      "fcm_dpo/margin": 30.234954833984375,
      "fcm_dpo/q_t": 0.38494789600372314,
      "grad_norm": 14.85726261138916,
      "learning_rate": 3.9285205908608934e-07,
      "logits/chosen": 0.6990875005722046,
      "logits/rejected": 0.6553751826286316,
      "logps/chosen": -114.58427429199219,
      "logps/ref_chosen": -61.938209533691406,
      "logps/ref_rejected": -72.21602630615234,
      "logps/rejected": -155.0970458984375,
      "loss": 1.053,
      "margin_dpo/margin_mean": 30.234954833984375,
      "margin_dpo/margin_std": 41.59199523925781,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -59.0046272277832,
      "KL/mean": -68.8511734008789,
      "KL/rejected_KL_mean": -78.69772338867188,
      "KL/std": 37.51115417480469,
      "epoch": 0.3794406651549509,
      "fcm_dpo/beta": 0.01688208617269993,
      "fcm_dpo/delta": 0.06992226839065552,
      "fcm_dpo/margin": 19.69308853149414,
      "fcm_dpo/q_t": 0.4261128604412079,
      "grad_norm": 20.584993362426758,
      "learning_rate": 3.9176496596569265e-07,
      "logits/chosen": 0.6441947817802429,
      "logits/rejected": 0.6038833856582642,
      "logps/chosen": -125.861572265625,
      "logps/ref_chosen": -66.85694885253906,
      "logps/ref_rejected": -84.83396911621094,
      "logps/rejected": -163.5316925048828,
      "loss": 1.2037,
      "margin_dpo/margin_mean": 19.69308853149414,
      "margin_dpo/margin_std": 42.82395553588867,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -54.52249526977539,
      "KL/mean": -63.388526916503906,
      "KL/rejected_KL_mean": -72.25457000732422,
      "KL/std": 40.418739318847656,
      "epoch": 0.38095238095238093,
      "fcm_dpo/beta": 0.01684136688709259,
      "fcm_dpo/delta": -0.07725033164024353,
      "fcm_dpo/margin": 17.73206901550293,
      "fcm_dpo/q_t": 0.4335172474384308,
      "grad_norm": 22.192724227905273,
      "learning_rate": 3.9067390737445254e-07,
      "logits/chosen": 0.573256254196167,
      "logits/rejected": 0.5198137164115906,
      "logps/chosen": -110.74642944335938,
      "logps/ref_chosen": -56.22393035888672,
      "logps/ref_rejected": -77.1136245727539,
      "logps/rejected": -149.36819458007812,
      "loss": 1.2559,
      "margin_dpo/margin_mean": 17.73206901550293,
      "margin_dpo/margin_std": 44.369590759277344,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -55.13286590576172,
      "KL/mean": -65.97904968261719,
      "KL/rejected_KL_mean": -76.82524108886719,
      "KL/std": 39.877471923828125,
      "epoch": 0.382464096749811,
      "fcm_dpo/beta": 0.016513584181666374,
      "fcm_dpo/delta": -0.05949968472123146,
      "fcm_dpo/margin": 21.692380905151367,
      "fcm_dpo/q_t": 0.4199674129486084,
      "grad_norm": 17.629150390625,
      "learning_rate": 3.8957891383162304e-07,
      "logits/chosen": 0.6932963132858276,
      "logits/rejected": 0.6514720916748047,
      "logps/chosen": -107.34288024902344,
      "logps/ref_chosen": -52.21001434326172,
      "logps/ref_rejected": -58.75764846801758,
      "logps/rejected": -135.5828857421875,
      "loss": 1.1667,
      "margin_dpo/margin_mean": 21.692380905151367,
      "margin_dpo/margin_std": 39.838340759277344,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -57.78590393066406,
      "KL/mean": -70.54551696777344,
      "KL/rejected_KL_mean": -83.30513000488281,
      "KL/std": 41.56895065307617,
      "epoch": 0.3839758125472411,
      "fcm_dpo/beta": 0.01639086753129959,
      "fcm_dpo/delta": -0.019591979682445526,
      "fcm_dpo/margin": 25.519224166870117,
      "fcm_dpo/q_t": 0.40863853693008423,
      "grad_norm": 14.444862365722656,
      "learning_rate": 3.884800159665276e-07,
      "logits/chosen": 0.6223227977752686,
      "logits/rejected": 0.5706925392150879,
      "logps/chosen": -123.42222595214844,
      "logps/ref_chosen": -65.63632202148438,
      "logps/ref_rejected": -82.34425354003906,
      "logps/rejected": -165.64938354492188,
      "loss": 1.1239,
      "margin_dpo/margin_mean": 25.519224166870117,
      "margin_dpo/margin_std": 42.410675048828125,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -54.50482940673828,
      "KL/mean": -68.82572937011719,
      "KL/rejected_KL_mean": -83.14663696289062,
      "KL/std": 42.355289459228516,
      "epoch": 0.3854875283446712,
      "fcm_dpo/beta": 0.016250912100076675,
      "fcm_dpo/delta": -0.06885148584842682,
      "fcm_dpo/margin": 28.64180564880371,
      "fcm_dpo/q_t": 0.39793136715888977,
      "grad_norm": 22.218046188354492,
      "learning_rate": 3.873772445177015e-07,
      "logits/chosen": 0.59206622838974,
      "logits/rejected": 0.5635826587677002,
      "logps/chosen": -122.41592407226562,
      "logps/ref_chosen": -67.91108703613281,
      "logps/ref_rejected": -83.89114379882812,
      "logps/rejected": -167.03778076171875,
      "loss": 1.1038,
      "margin_dpo/margin_mean": 28.64180564880371,
      "margin_dpo/margin_std": 46.61860275268555,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -62.24885177612305,
      "KL/mean": -76.39161682128906,
      "KL/rejected_KL_mean": -90.53438568115234,
      "KL/std": 39.0150146484375,
      "epoch": 0.3869992441421013,
      "fcm_dpo/beta": 0.01603306457400322,
      "fcm_dpo/delta": -0.05648089200258255,
      "fcm_dpo/margin": 28.285526275634766,
      "fcm_dpo/q_t": 0.4011594355106354,
      "grad_norm": 17.703187942504883,
      "learning_rate": 3.862706303320329e-07,
      "logits/chosen": 0.6131513118743896,
      "logits/rejected": 0.551064133644104,
      "logps/chosen": -125.74884033203125,
      "logps/ref_chosen": -63.49998474121094,
      "logps/ref_rejected": -90.77104187011719,
      "logps/rejected": -181.305419921875,
      "loss": 1.1329,
      "margin_dpo/margin_mean": 28.2855224609375,
      "margin_dpo/margin_std": 50.50141525268555,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -60.9831428527832,
      "KL/mean": -77.10617065429688,
      "KL/rejected_KL_mean": -93.22918701171875,
      "KL/std": 43.73507308959961,
      "epoch": 0.3885109599395314,
      "fcm_dpo/beta": 0.015743490308523178,
      "fcm_dpo/delta": -0.1137080192565918,
      "fcm_dpo/margin": 32.24604797363281,
      "fcm_dpo/q_t": 0.38974249362945557,
      "grad_norm": 16.30253791809082,
      "learning_rate": 3.851602043638994e-07,
      "logits/chosen": 0.6148316860198975,
      "logits/rejected": 0.5495343208312988,
      "logps/chosen": -131.58380126953125,
      "logps/ref_chosen": -70.60064697265625,
      "logps/ref_rejected": -108.58313751220703,
      "logps/rejected": -201.81231689453125,
      "loss": 1.077,
      "margin_dpo/margin_mean": 32.24604797363281,
      "margin_dpo/margin_std": 49.422523498535156,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -58.98999786376953,
      "KL/mean": -72.73417663574219,
      "KL/rejected_KL_mean": -86.47834777832031,
      "KL/std": 38.587520599365234,
      "epoch": 0.3900226757369615,
      "fcm_dpo/beta": 0.015620948746800423,
      "fcm_dpo/delta": -0.030701272189617157,
      "fcm_dpo/margin": 27.488344192504883,
      "fcm_dpo/q_t": 0.40082675218582153,
      "grad_norm": 15.351493835449219,
      "learning_rate": 3.840459976743023e-07,
      "logits/chosen": 0.6455204486846924,
      "logits/rejected": 0.5953073501586914,
      "logps/chosen": -118.2441635131836,
      "logps/ref_chosen": -59.25416564941406,
      "logps/ref_rejected": -85.58709716796875,
      "logps/rejected": -172.06544494628906,
      "loss": 1.0803,
      "margin_dpo/margin_mean": 27.488344192504883,
      "margin_dpo/margin_std": 37.24540328979492,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -48.836265563964844,
      "KL/mean": -69.62240600585938,
      "KL/rejected_KL_mean": -90.40853881835938,
      "KL/std": 43.766815185546875,
      "epoch": 0.3915343915343915,
      "fcm_dpo/beta": 0.014955306425690651,
      "fcm_dpo/delta": -0.23853763937950134,
      "fcm_dpo/margin": 41.5722770690918,
      "fcm_dpo/q_t": 0.3603108823299408,
      "grad_norm": 13.786149024963379,
      "learning_rate": 3.8292804142999796e-07,
      "logits/chosen": 0.5523971319198608,
      "logits/rejected": 0.4531182646751404,
      "logps/chosen": -114.2711410522461,
      "logps/ref_chosen": -65.43487548828125,
      "logps/ref_rejected": -95.41731262207031,
      "logps/rejected": -185.8258514404297,
      "loss": 0.9843,
      "margin_dpo/margin_mean": 41.5722770690918,
      "margin_dpo/margin_std": 49.170082092285156,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -49.102508544921875,
      "KL/mean": -65.42601013183594,
      "KL/rejected_KL_mean": -81.74950408935547,
      "KL/std": 41.56929397583008,
      "epoch": 0.3930461073318216,
      "fcm_dpo/beta": 0.014612874016165733,
      "fcm_dpo/delta": -0.08172280341386795,
      "fcm_dpo/margin": 32.64699935913086,
      "fcm_dpo/q_t": 0.3949354588985443,
      "grad_norm": 14.47229290008545,
      "learning_rate": 3.818063669026256e-07,
      "logits/chosen": 0.6209584474563599,
      "logits/rejected": 0.5350062251091003,
      "logps/chosen": -98.19209289550781,
      "logps/ref_chosen": -49.08958435058594,
      "logps/ref_rejected": -79.01708221435547,
      "logps/rejected": -160.76658630371094,
      "loss": 1.0936,
      "margin_dpo/margin_mean": 32.64699935913086,
      "margin_dpo/margin_std": 50.892677307128906,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -49.21405792236328,
      "KL/mean": -62.99930191040039,
      "KL/rejected_KL_mean": -76.78453826904297,
      "KL/std": 40.187191009521484,
      "epoch": 0.3945578231292517,
      "fcm_dpo/beta": 0.014624063856899738,
      "fcm_dpo/delta": -0.0033402051776647568,
      "fcm_dpo/margin": 27.570484161376953,
      "fcm_dpo/q_t": 0.4106593132019043,
      "grad_norm": 16.098180770874023,
      "learning_rate": 3.806810054678331e-07,
      "logits/chosen": 0.49886083602905273,
      "logits/rejected": 0.5313537120819092,
      "logps/chosen": -120.08645629882812,
      "logps/ref_chosen": -70.87239074707031,
      "logps/ref_rejected": -65.01522064208984,
      "logps/rejected": -141.7997589111328,
      "loss": 1.1226,
      "margin_dpo/margin_mean": 27.570484161376953,
      "margin_dpo/margin_std": 44.75124740600586,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -50.91869354248047,
      "KL/mean": -64.60671997070312,
      "KL/rejected_KL_mean": -78.29474639892578,
      "KL/std": 39.187896728515625,
      "epoch": 0.3960695389266818,
      "fcm_dpo/beta": 0.014638787135481834,
      "fcm_dpo/delta": -0.0008811671286821365,
      "fcm_dpo/margin": 27.37605857849121,
      "fcm_dpo/q_t": 0.40981292724609375,
      "grad_norm": 16.295869827270508,
      "learning_rate": 3.7955198860439887e-07,
      "logits/chosen": 0.6594116687774658,
      "logits/rejected": 0.596439003944397,
      "logps/chosen": -118.78932189941406,
      "logps/ref_chosen": -67.8706283569336,
      "logps/ref_rejected": -88.7205810546875,
      "logps/rejected": -167.01531982421875,
      "loss": 1.1153,
      "margin_dpo/margin_mean": 27.376060485839844,
      "margin_dpo/margin_std": 42.29436492919922,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -48.147857666015625,
      "KL/mean": -61.33314514160156,
      "KL/rejected_KL_mean": -74.51841735839844,
      "KL/std": 39.83005142211914,
      "epoch": 0.3975812547241119,
      "fcm_dpo/beta": 0.014664757996797562,
      "fcm_dpo/delta": 0.013708971440792084,
      "fcm_dpo/margin": 26.370563507080078,
      "fcm_dpo/q_t": 0.41229403018951416,
      "grad_norm": 13.509973526000977,
      "learning_rate": 3.784193478933516e-07,
      "logits/chosen": 0.5683990120887756,
      "logits/rejected": 0.4630658030509949,
      "logps/chosen": -103.34243774414062,
      "logps/ref_chosen": -55.194583892822266,
      "logps/ref_rejected": -80.54048156738281,
      "logps/rejected": -155.05889892578125,
      "loss": 1.1326,
      "margin_dpo/margin_mean": 26.370563507080078,
      "margin_dpo/margin_std": 43.668113708496094,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -51.94562530517578,
      "KL/mean": -66.57713317871094,
      "KL/rejected_KL_mean": -81.20864868164062,
      "KL/std": 42.38758850097656,
      "epoch": 0.39909297052154197,
      "fcm_dpo/beta": 0.014563208445906639,
      "fcm_dpo/delta": -0.027422528713941574,
      "fcm_dpo/margin": 29.263023376464844,
      "fcm_dpo/q_t": 0.4048900306224823,
      "grad_norm": 14.484800338745117,
      "learning_rate": 3.7728311501708674e-07,
      "logits/chosen": 0.5261293053627014,
      "logits/rejected": 0.48067325353622437,
      "logps/chosen": -135.11630249023438,
      "logps/ref_chosen": -83.17068481445312,
      "logps/ref_rejected": -88.33625793457031,
      "logps/rejected": -169.54490661621094,
      "loss": 1.1104,
      "margin_dpo/margin_mean": 29.263023376464844,
      "margin_dpo/margin_std": 46.34964370727539,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -49.31029510498047,
      "KL/mean": -65.97465515136719,
      "KL/rejected_KL_mean": -82.63902282714844,
      "KL/std": 44.29258728027344,
      "epoch": 0.40060468631897206,
      "fcm_dpo/beta": 0.014456374570727348,
      "fcm_dpo/delta": -0.08606263995170593,
      "fcm_dpo/margin": 33.32872772216797,
      "fcm_dpo/q_t": 0.39422452449798584,
      "grad_norm": 14.318184852600098,
      "learning_rate": 3.7614332175848027e-07,
      "logits/chosen": 0.6983447670936584,
      "logits/rejected": 0.6320916414260864,
      "logps/chosen": -100.97314453125,
      "logps/ref_chosen": -51.66284942626953,
      "logps/ref_rejected": -67.1720962524414,
      "logps/rejected": -149.81112670898438,
      "loss": 1.1056,
      "margin_dpo/margin_mean": 33.32872772216797,
      "margin_dpo/margin_std": 54.2000617980957,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -47.9881591796875,
      "KL/mean": -63.58972930908203,
      "KL/rejected_KL_mean": -79.19129943847656,
      "KL/std": 44.739112854003906,
      "epoch": 0.4021164021164021,
      "fcm_dpo/beta": 0.014198727905750275,
      "fcm_dpo/delta": -0.04516395553946495,
      "fcm_dpo/margin": 31.203144073486328,
      "fcm_dpo/q_t": 0.4013225734233856,
      "grad_norm": 16.357166290283203,
      "learning_rate": 3.75e-07,
      "logits/chosen": 0.627079963684082,
      "logits/rejected": 0.5522067546844482,
      "logps/chosen": -105.43865203857422,
      "logps/ref_chosen": -57.45049285888672,
      "logps/ref_rejected": -77.60826110839844,
      "logps/rejected": -156.799560546875,
      "loss": 1.0993,
      "margin_dpo/margin_mean": 31.203144073486328,
      "margin_dpo/margin_std": 48.260189056396484,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -42.61259460449219,
      "KL/mean": -54.70775604248047,
      "KL/rejected_KL_mean": -66.80291748046875,
      "KL/std": 41.03778076171875,
      "epoch": 0.4036281179138322,
      "fcm_dpo/beta": 0.014107579365372658,
      "fcm_dpo/delta": -0.0573669970035553,
      "fcm_dpo/margin": 24.190322875976562,
      "fcm_dpo/q_t": 0.4226904511451721,
      "grad_norm": 14.8142671585083,
      "learning_rate": 3.738531817228131e-07,
      "logits/chosen": 0.6663018465042114,
      "logits/rejected": 0.6482110023498535,
      "logps/chosen": -97.64794921875,
      "logps/ref_chosen": -55.03535079956055,
      "logps/ref_rejected": -66.0953369140625,
      "logps/rejected": -132.89825439453125,
      "loss": 1.194,
      "margin_dpo/margin_mean": 24.190324783325195,
      "margin_dpo/margin_std": 48.79133605957031,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -43.24398422241211,
      "KL/mean": -58.16192626953125,
      "KL/rejected_KL_mean": -73.07987213134766,
      "KL/std": 44.10837173461914,
      "epoch": 0.4051398337112623,
      "fcm_dpo/beta": 0.014020204544067383,
      "fcm_dpo/delta": -0.019175250083208084,
      "fcm_dpo/margin": 29.835880279541016,
      "fcm_dpo/q_t": 0.40609800815582275,
      "grad_norm": 13.555010795593262,
      "learning_rate": 3.7270289900589204e-07,
      "logits/chosen": 0.5056520104408264,
      "logits/rejected": 0.49157899618148804,
      "logps/chosen": -108.31572723388672,
      "logps/ref_chosen": -65.07174682617188,
      "logps/ref_rejected": -71.42485809326172,
      "logps/rejected": -144.50473022460938,
      "loss": 1.0911,
      "margin_dpo/margin_mean": 29.835880279541016,
      "margin_dpo/margin_std": 41.757545471191406,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -49.64472961425781,
      "KL/mean": -67.10590362548828,
      "KL/rejected_KL_mean": -84.56707763671875,
      "KL/std": 46.25409698486328,
      "epoch": 0.40665154950869237,
      "fcm_dpo/beta": 0.013886158354580402,
      "fcm_dpo/delta": -0.0896507278084755,
      "fcm_dpo/margin": 34.92235565185547,
      "fcm_dpo/q_t": 0.3912719786167145,
      "grad_norm": 13.464406967163086,
      "learning_rate": 3.7154918402511714e-07,
      "logits/chosen": 0.7307313084602356,
      "logits/rejected": 0.6821566820144653,
      "logps/chosen": -116.78093719482422,
      "logps/ref_chosen": -67.1362075805664,
      "logps/ref_rejected": -82.55778503417969,
      "logps/rejected": -167.12486267089844,
      "loss": 1.0593,
      "margin_dpo/margin_mean": 34.92235565185547,
      "margin_dpo/margin_std": 46.31365203857422,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -50.8516731262207,
      "KL/mean": -64.68014526367188,
      "KL/rejected_KL_mean": -78.50861358642578,
      "KL/std": 45.99193572998047,
      "epoch": 0.40816326530612246,
      "fcm_dpo/beta": 0.013720536604523659,
      "fcm_dpo/delta": 0.021170198917388916,
      "fcm_dpo/margin": 27.656940460205078,
      "fcm_dpo/q_t": 0.4142289161682129,
      "grad_norm": 14.118720054626465,
      "learning_rate": 3.7039206905237656e-07,
      "logits/chosen": 0.6752135157585144,
      "logits/rejected": 0.5939148664474487,
      "logps/chosen": -117.54037475585938,
      "logps/ref_chosen": -66.6886978149414,
      "logps/ref_rejected": -85.16129302978516,
      "logps/rejected": -163.66990661621094,
      "loss": 1.1477,
      "margin_dpo/margin_mean": 27.65694236755371,
      "margin_dpo/margin_std": 48.92210388183594,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -50.66046142578125,
      "KL/mean": -61.81583023071289,
      "KL/rejected_KL_mean": -72.97119140625,
      "KL/std": 46.73677062988281,
      "epoch": 0.40967498110355255,
      "fcm_dpo/beta": 0.013790830969810486,
      "fcm_dpo/delta": 0.002880556508898735,
      "fcm_dpo/margin": 22.31073760986328,
      "fcm_dpo/q_t": 0.4354283809661865,
      "grad_norm": 16.642648696899414,
      "learning_rate": 3.692315864546635e-07,
      "logits/chosen": 0.6743849515914917,
      "logits/rejected": 0.6103301644325256,
      "logps/chosen": -123.06800842285156,
      "logps/ref_chosen": -72.40754699707031,
      "logps/ref_rejected": -92.06311798095703,
      "logps/rejected": -165.0343017578125,
      "loss": 1.2363,
      "margin_dpo/margin_mean": 22.310733795166016,
      "margin_dpo/margin_std": 55.137351989746094,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -42.49518585205078,
      "KL/mean": -65.01316833496094,
      "KL/rejected_KL_mean": -87.53114318847656,
      "KL/std": 45.20049285888672,
      "epoch": 0.41118669690098264,
      "fcm_dpo/beta": 0.013394663110375404,
      "fcm_dpo/delta": -0.2158459573984146,
      "fcm_dpo/margin": 45.03594970703125,
      "fcm_dpo/q_t": 0.36177968978881836,
      "grad_norm": 15.147326469421387,
      "learning_rate": 3.6806776869317067e-07,
      "logits/chosen": 0.6594383716583252,
      "logits/rejected": 0.6793452501296997,
      "logps/chosen": -109.09658813476562,
      "logps/ref_chosen": -66.60140228271484,
      "logps/ref_rejected": -67.74340057373047,
      "logps/rejected": -155.2745361328125,
      "loss": 0.9511,
      "margin_dpo/margin_mean": 45.035953521728516,
      "margin_dpo/margin_std": 43.84568786621094,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -59.83015441894531,
      "KL/mean": -75.74633026123047,
      "KL/rejected_KL_mean": -91.6624984741211,
      "KL/std": 47.8607177734375,
      "epoch": 0.4126984126984127,
      "fcm_dpo/beta": 0.013139687478542328,
      "fcm_dpo/delta": -0.019220881164073944,
      "fcm_dpo/margin": 31.832345962524414,
      "fcm_dpo/q_t": 0.4068758189678192,
      "grad_norm": 16.181289672851562,
      "learning_rate": 3.669006483223828e-07,
      "logits/chosen": 0.6526553630828857,
      "logits/rejected": 0.584166407585144,
      "logps/chosen": -117.18502807617188,
      "logps/ref_chosen": -57.35487747192383,
      "logps/ref_rejected": -84.17168426513672,
      "logps/rejected": -175.8341827392578,
      "loss": 1.1591,
      "margin_dpo/margin_mean": 31.832345962524414,
      "margin_dpo/margin_std": 60.78428649902344,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -58.55799865722656,
      "KL/mean": -76.35200500488281,
      "KL/rejected_KL_mean": -94.14601135253906,
      "KL/std": 48.54522705078125,
      "epoch": 0.41421012849584277,
      "fcm_dpo/beta": 0.013040488585829735,
      "fcm_dpo/delta": -0.06715575605630875,
      "fcm_dpo/margin": 35.5880012512207,
      "fcm_dpo/q_t": 0.3975376486778259,
      "grad_norm": 13.54541301727295,
      "learning_rate": 3.657302579891656e-07,
      "logits/chosen": 0.5605419874191284,
      "logits/rejected": 0.5445349812507629,
      "logps/chosen": -118.19949340820312,
      "logps/ref_chosen": -59.64149475097656,
      "logps/ref_rejected": -68.29348754882812,
      "logps/rejected": -162.43948364257812,
      "loss": 1.1076,
      "margin_dpo/margin_mean": 35.5880012512207,
      "margin_dpo/margin_std": 58.20866394042969,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -58.40193176269531,
      "KL/mean": -75.99978637695312,
      "KL/rejected_KL_mean": -93.5976333618164,
      "KL/std": 47.385040283203125,
      "epoch": 0.41572184429327286,
      "fcm_dpo/beta": 0.012910742312669754,
      "fcm_dpo/delta": -0.05700352042913437,
      "fcm_dpo/margin": 35.195701599121094,
      "fcm_dpo/q_t": 0.3971477746963501,
      "grad_norm": 14.352291107177734,
      "learning_rate": 3.645566304318526e-07,
      "logits/chosen": 0.6138721108436584,
      "logits/rejected": 0.5329569578170776,
      "logps/chosen": -111.6685791015625,
      "logps/ref_chosen": -53.26664352416992,
      "logps/ref_rejected": -73.84062194824219,
      "logps/rejected": -167.43826293945312,
      "loss": 1.0804,
      "margin_dpo/margin_mean": 35.19570541381836,
      "margin_dpo/margin_std": 50.1209716796875,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -54.633445739746094,
      "KL/mean": -71.94046020507812,
      "KL/rejected_KL_mean": -89.24748229980469,
      "KL/std": 45.49829864501953,
      "epoch": 0.41723356009070295,
      "fcm_dpo/beta": 0.01276165060698986,
      "fcm_dpo/delta": -0.04363919422030449,
      "fcm_dpo/margin": 34.61402893066406,
      "fcm_dpo/q_t": 0.4001271426677704,
      "grad_norm": 15.740699768066406,
      "learning_rate": 3.633797984793294e-07,
      "logits/chosen": 0.6114457845687866,
      "logits/rejected": 0.5790101289749146,
      "logps/chosen": -107.65424346923828,
      "logps/ref_chosen": -53.02079772949219,
      "logps/ref_rejected": -61.56678771972656,
      "logps/rejected": -150.81427001953125,
      "loss": 1.088,
      "margin_dpo/margin_mean": 34.61402893066406,
      "margin_dpo/margin_std": 50.05985641479492,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -62.1165771484375,
      "KL/mean": -72.11553955078125,
      "KL/rejected_KL_mean": -82.114501953125,
      "KL/std": 46.5863037109375,
      "epoch": 0.41874527588813304,
      "fcm_dpo/beta": 0.012991832569241524,
      "fcm_dpo/delta": 0.14401455223560333,
      "fcm_dpo/margin": 19.997928619384766,
      "fcm_dpo/q_t": 0.4420499801635742,
      "grad_norm": 19.145910263061523,
      "learning_rate": 3.6219979505011555e-07,
      "logits/chosen": 0.692374587059021,
      "logits/rejected": 0.7201675176620483,
      "logps/chosen": -133.549560546875,
      "logps/ref_chosen": -71.43299102783203,
      "logps/ref_rejected": -67.65852355957031,
      "logps/rejected": -149.7730255126953,
      "loss": 1.2577,
      "margin_dpo/margin_mean": 19.997926712036133,
      "margin_dpo/margin_std": 53.601715087890625,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -65.80406188964844,
      "KL/mean": -82.9516372680664,
      "KL/rejected_KL_mean": -100.09921264648438,
      "KL/std": 51.23931121826172,
      "epoch": 0.42025699168556313,
      "fcm_dpo/beta": 0.013081016018986702,
      "fcm_dpo/delta": -0.051438432186841965,
      "fcm_dpo/margin": 34.29515075683594,
      "fcm_dpo/q_t": 0.3994791507720947,
      "grad_norm": 18.11107635498047,
      "learning_rate": 3.6101665315144353e-07,
      "logits/chosen": 0.574435293674469,
      "logits/rejected": 0.5230345726013184,
      "logps/chosen": -132.91482543945312,
      "logps/ref_chosen": -67.11076354980469,
      "logps/ref_rejected": -88.74851989746094,
      "logps/rejected": -188.8477325439453,
      "loss": 1.1069,
      "margin_dpo/margin_mean": 34.2951545715332,
      "margin_dpo/margin_std": 54.25193786621094,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -50.36063003540039,
      "KL/mean": -74.28724670410156,
      "KL/rejected_KL_mean": -98.2138671875,
      "KL/std": 50.72193908691406,
      "epoch": 0.4217687074829932,
      "fcm_dpo/beta": 0.01258824486285448,
      "fcm_dpo/delta": -0.21493816375732422,
      "fcm_dpo/margin": 47.853233337402344,
      "fcm_dpo/q_t": 0.36314916610717773,
      "grad_norm": 17.61539077758789,
      "learning_rate": 3.5983040587833563e-07,
      "logits/chosen": 0.6193308234214783,
      "logits/rejected": 0.5832624435424805,
      "logps/chosen": -104.85811614990234,
      "logps/ref_chosen": -54.49748611450195,
      "logps/ref_rejected": -70.42373657226562,
      "logps/rejected": -168.63760375976562,
      "loss": 0.9625,
      "margin_dpo/margin_mean": 47.853233337402344,
      "margin_dpo/margin_std": 49.155887603759766,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -55.05585479736328,
      "KL/mean": -79.03872680664062,
      "KL/rejected_KL_mean": -103.0216064453125,
      "KL/std": 53.03904342651367,
      "epoch": 0.42328042328042326,
      "fcm_dpo/beta": 0.01207013800740242,
      "fcm_dpo/delta": -0.18996167182922363,
      "fcm_dpo/margin": 47.96575164794922,
      "fcm_dpo/q_t": 0.3691137135028839,
      "grad_norm": 12.14430046081543,
      "learning_rate": 3.586410864126781e-07,
      "logits/chosen": 0.6724978685379028,
      "logits/rejected": 0.6316500902175903,
      "logps/chosen": -115.48866271972656,
      "logps/ref_chosen": -60.43281173706055,
      "logps/ref_rejected": -78.39051818847656,
      "logps/rejected": -181.41212463378906,
      "loss": 0.9676,
      "margin_dpo/margin_mean": 47.96575164794922,
      "margin_dpo/margin_std": 49.17424774169922,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -59.39515686035156,
      "KL/mean": -80.96408081054688,
      "KL/rejected_KL_mean": -102.53300476074219,
      "KL/std": 52.042930603027344,
      "epoch": 0.42479213907785335,
      "fcm_dpo/beta": 0.01177662331610918,
      "fcm_dpo/delta": -0.1136535257101059,
      "fcm_dpo/margin": 43.137847900390625,
      "fcm_dpo/q_t": 0.3859631419181824,
      "grad_norm": 13.771058082580566,
      "learning_rate": 3.574487280222929e-07,
      "logits/chosen": 0.64704430103302,
      "logits/rejected": 0.672046422958374,
      "logps/chosen": -119.67724609375,
      "logps/ref_chosen": -60.2820930480957,
      "logps/ref_rejected": -62.04009246826172,
      "logps/rejected": -164.57308959960938,
      "loss": 1.044,
      "margin_dpo/margin_mean": 43.137847900390625,
      "margin_dpo/margin_std": 56.448631286621094,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -66.66419219970703,
      "KL/mean": -86.35317993164062,
      "KL/rejected_KL_mean": -106.04218292236328,
      "KL/std": 52.45392608642578,
      "epoch": 0.42630385487528344,
      "fcm_dpo/beta": 0.011679998598992825,
      "fcm_dpo/delta": -0.06393231451511383,
      "fcm_dpo/margin": 39.37797927856445,
      "fcm_dpo/q_t": 0.3986932039260864,
      "grad_norm": 16.130067825317383,
      "learning_rate": 3.562533640600075e-07,
      "logits/chosen": 0.5881474614143372,
      "logits/rejected": 0.54173743724823,
      "logps/chosen": -127.28811645507812,
      "logps/ref_chosen": -60.623924255371094,
      "logps/ref_rejected": -68.67400360107422,
      "logps/rejected": -174.7161865234375,
      "loss": 1.1015,
      "margin_dpo/margin_mean": 39.37797927856445,
      "margin_dpo/margin_std": 60.376380920410156,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -66.44935607910156,
      "KL/mean": -84.42182159423828,
      "KL/rejected_KL_mean": -102.394287109375,
      "KL/std": 51.58662033081055,
      "epoch": 0.42781557067271353,
      "fcm_dpo/beta": 0.011540468782186508,
      "fcm_dpo/delta": -0.015572082251310349,
      "fcm_dpo/margin": 35.94493865966797,
      "fcm_dpo/q_t": 0.40672242641448975,
      "grad_norm": 15.746005058288574,
      "learning_rate": 3.550550279627215e-07,
      "logits/chosen": 0.6343629360198975,
      "logits/rejected": 0.5318249464035034,
      "logps/chosen": -134.09710693359375,
      "logps/ref_chosen": -67.64775085449219,
      "logps/ref_rejected": -99.96835327148438,
      "logps/rejected": -202.36264038085938,
      "loss": 1.1187,
      "margin_dpo/margin_mean": 35.9449348449707,
      "margin_dpo/margin_std": 57.89904022216797,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -64.0386734008789,
      "KL/mean": -83.11166381835938,
      "KL/rejected_KL_mean": -102.18466186523438,
      "KL/std": 55.27910614013672,
      "epoch": 0.4293272864701436,
      "fcm_dpo/beta": 0.011395130306482315,
      "fcm_dpo/delta": -0.03666817396879196,
      "fcm_dpo/margin": 38.14597702026367,
      "fcm_dpo/q_t": 0.40096110105514526,
      "grad_norm": 13.227509498596191,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": 0.6858668327331543,
      "logits/rejected": 0.6233437061309814,
      "logps/chosen": -121.006103515625,
      "logps/ref_chosen": -56.96742630004883,
      "logps/ref_rejected": -86.36236572265625,
      "logps/rejected": -188.54702758789062,
      "loss": 1.0842,
      "margin_dpo/margin_mean": 38.14597702026367,
      "margin_dpo/margin_std": 53.011199951171875,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -77.92655944824219,
      "KL/mean": -93.79136657714844,
      "KL/rejected_KL_mean": -109.65617370605469,
      "KL/std": 49.799896240234375,
      "epoch": 0.4308390022675737,
      "fcm_dpo/beta": 0.011514578014612198,
      "fcm_dpo/delta": 0.03562067821621895,
      "fcm_dpo/margin": 31.729633331298828,
      "fcm_dpo/q_t": 0.41822776198387146,
      "grad_norm": 17.575851440429688,
      "learning_rate": 3.5264957352549375e-07,
      "logits/chosen": 0.6727806329727173,
      "logits/rejected": 0.6466799974441528,
      "logps/chosen": -149.58267211914062,
      "logps/ref_chosen": -71.65611267089844,
      "logps/ref_rejected": -81.63829803466797,
      "logps/rejected": -191.2944793701172,
      "loss": 1.1496,
      "margin_dpo/margin_mean": 31.729633331298828,
      "margin_dpo/margin_std": 55.26091384887695,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -70.88116455078125,
      "KL/mean": -96.23809814453125,
      "KL/rejected_KL_mean": -121.59503173828125,
      "KL/std": 53.773773193359375,
      "epoch": 0.4323507180650038,
      "fcm_dpo/beta": 0.011181243695318699,
      "fcm_dpo/delta": -0.17741291224956512,
      "fcm_dpo/margin": 50.713863372802734,
      "fcm_dpo/q_t": 0.3722303509712219,
      "grad_norm": 13.359155654907227,
      "learning_rate": 3.514425224712835e-07,
      "logits/chosen": 0.5950964689254761,
      "logits/rejected": 0.5027275085449219,
      "logps/chosen": -131.960693359375,
      "logps/ref_chosen": -61.07952117919922,
      "logps/ref_rejected": -91.28128051757812,
      "logps/rejected": -212.87631225585938,
      "loss": 0.9865,
      "margin_dpo/margin_mean": 50.713863372802734,
      "margin_dpo/margin_std": 56.22399139404297,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -58.15245819091797,
      "KL/mean": -83.52070617675781,
      "KL/rejected_KL_mean": -108.88896179199219,
      "KL/std": 56.466026306152344,
      "epoch": 0.43386243386243384,
      "fcm_dpo/beta": 0.010851925238966942,
      "fcm_dpo/delta": -0.15918992459774017,
      "fcm_dpo/margin": 50.73650360107422,
      "fcm_dpo/q_t": 0.3756788969039917,
      "grad_norm": 13.096345901489258,
      "learning_rate": 3.502326338516534e-07,
      "logits/chosen": 0.6736834049224854,
      "logits/rejected": 0.636581540107727,
      "logps/chosen": -104.18824768066406,
      "logps/ref_chosen": -46.035789489746094,
      "logps/ref_rejected": -59.95293426513672,
      "logps/rejected": -168.84188842773438,
      "loss": 1.0036,
      "margin_dpo/margin_mean": 50.73650360107422,
      "margin_dpo/margin_std": 59.042449951171875,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -73.77389526367188,
      "KL/mean": -92.34275817871094,
      "KL/rejected_KL_mean": -110.91160583496094,
      "KL/std": 53.420928955078125,
      "epoch": 0.43537414965986393,
      "fcm_dpo/beta": 0.010760816745460033,
      "fcm_dpo/delta": 0.00036012567579746246,
      "fcm_dpo/margin": 37.137718200683594,
      "fcm_dpo/q_t": 0.409721314907074,
      "grad_norm": 14.71628475189209,
      "learning_rate": 3.490199415097892e-07,
      "logits/chosen": 0.5556157827377319,
      "logits/rejected": 0.5000091195106506,
      "logps/chosen": -139.16473388671875,
      "logps/ref_chosen": -65.3908462524414,
      "logps/ref_rejected": -88.53607940673828,
      "logps/rejected": -199.44769287109375,
      "loss": 1.1213,
      "margin_dpo/margin_mean": 37.137718200683594,
      "margin_dpo/margin_std": 59.299896240234375,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -75.05809783935547,
      "KL/mean": -92.1280288696289,
      "KL/rejected_KL_mean": -109.19795227050781,
      "KL/std": 53.392269134521484,
      "epoch": 0.436885865457294,
      "fcm_dpo/beta": 0.01084593590348959,
      "fcm_dpo/delta": 0.030338387936353683,
      "fcm_dpo/margin": 34.13987350463867,
      "fcm_dpo/q_t": 0.41851770877838135,
      "grad_norm": 18.15667724609375,
      "learning_rate": 3.4780447936730247e-07,
      "logits/chosen": 0.7534016370773315,
      "logits/rejected": 0.7147485017776489,
      "logps/chosen": -129.6517333984375,
      "logps/ref_chosen": -54.5936279296875,
      "logps/ref_rejected": -67.20855712890625,
      "logps/rejected": -176.40650939941406,
      "loss": 1.1586,
      "margin_dpo/margin_mean": 34.13987350463867,
      "margin_dpo/margin_std": 62.12736129760742,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -83.91240692138672,
      "KL/mean": -104.92753601074219,
      "KL/rejected_KL_mean": -125.94265747070312,
      "KL/std": 52.57867431640625,
      "epoch": 0.4383975812547241,
      "fcm_dpo/beta": 0.01068640872836113,
      "fcm_dpo/delta": -0.052049390971660614,
      "fcm_dpo/margin": 42.03026580810547,
      "fcm_dpo/q_t": 0.39872145652770996,
      "grad_norm": 16.8378963470459,
      "learning_rate": 3.465862814232821e-07,
      "logits/chosen": 0.7585524320602417,
      "logits/rejected": 0.6853688955307007,
      "logps/chosen": -145.29698181152344,
      "logps/ref_chosen": -61.38457489013672,
      "logps/ref_rejected": -91.92778015136719,
      "logps/rejected": -217.8704376220703,
      "loss": 1.096,
      "margin_dpo/margin_mean": 42.03026580810547,
      "margin_dpo/margin_std": 63.9581298828125,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -79.92852020263672,
      "KL/mean": -102.51646423339844,
      "KL/rejected_KL_mean": -125.10440063476562,
      "KL/std": 52.22700500488281,
      "epoch": 0.4399092970521542,
      "fcm_dpo/beta": 0.010655292309820652,
      "fcm_dpo/delta": -0.08616377413272858,
      "fcm_dpo/margin": 45.17587661743164,
      "fcm_dpo/q_t": 0.3925040364265442,
      "grad_norm": 15.182485580444336,
      "learning_rate": 3.4536538175334343e-07,
      "logits/chosen": 0.8224391937255859,
      "logits/rejected": 0.7500874996185303,
      "logps/chosen": -130.79156494140625,
      "logps/ref_chosen": -50.863037109375,
      "logps/ref_rejected": -82.20868682861328,
      "logps/rejected": -207.31307983398438,
      "loss": 1.06,
      "margin_dpo/margin_mean": 45.17587661743164,
      "margin_dpo/margin_std": 59.43260192871094,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -78.54954528808594,
      "KL/mean": -96.63288879394531,
      "KL/rejected_KL_mean": -114.71622467041016,
      "KL/std": 55.45445251464844,
      "epoch": 0.4414210128495843,
      "fcm_dpo/beta": 0.010552434250712395,
      "fcm_dpo/delta": 0.01907689869403839,
      "fcm_dpo/margin": 36.16666793823242,
      "fcm_dpo/q_t": 0.41435399651527405,
      "grad_norm": 15.718446731567383,
      "learning_rate": 3.4414181450867465e-07,
      "logits/chosen": 0.6965575218200684,
      "logits/rejected": 0.6475476026535034,
      "logps/chosen": -142.8984375,
      "logps/ref_chosen": -64.34888458251953,
      "logps/ref_rejected": -72.86434173583984,
      "logps/rejected": -187.58056640625,
      "loss": 1.1482,
      "margin_dpo/margin_mean": 36.166664123535156,
      "margin_dpo/margin_std": 64.25723266601562,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -78.22496032714844,
      "KL/mean": -103.93822479248047,
      "KL/rejected_KL_mean": -129.6514892578125,
      "KL/std": 57.0059928894043,
      "epoch": 0.4429327286470144,
      "fcm_dpo/beta": 0.010385725647211075,
      "fcm_dpo/delta": -0.14143896102905273,
      "fcm_dpo/margin": 51.426517486572266,
      "fcm_dpo/q_t": 0.38087648153305054,
      "grad_norm": 11.961139678955078,
      "learning_rate": 3.4291561391508185e-07,
      "logits/chosen": 0.7971653938293457,
      "logits/rejected": 0.7070008516311646,
      "logps/chosen": -133.09442138671875,
      "logps/ref_chosen": -54.869468688964844,
      "logps/ref_rejected": -81.858642578125,
      "logps/rejected": -211.5101318359375,
      "loss": 1.045,
      "margin_dpo/margin_mean": 51.426513671875,
      "margin_dpo/margin_std": 69.60263061523438,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -82.18350219726562,
      "KL/mean": -99.19888305664062,
      "KL/rejected_KL_mean": -116.21426391601562,
      "KL/std": 58.05199432373047,
      "epoch": 0.4444444444444444,
      "fcm_dpo/beta": 0.010307633318006992,
      "fcm_dpo/delta": 0.05093620717525482,
      "fcm_dpo/margin": 34.030757904052734,
      "fcm_dpo/q_t": 0.42151233553886414,
      "grad_norm": 12.948081970214844,
      "learning_rate": 3.4168681427203153e-07,
      "logits/chosen": 0.7250140905380249,
      "logits/rejected": 0.6781303882598877,
      "logps/chosen": -138.85440063476562,
      "logps/ref_chosen": -56.670902252197266,
      "logps/ref_rejected": -70.32819366455078,
      "logps/rejected": -186.54244995117188,
      "loss": 1.1413,
      "margin_dpo/margin_mean": 34.030757904052734,
      "margin_dpo/margin_std": 54.95783233642578,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -87.12187194824219,
      "KL/mean": -103.36134338378906,
      "KL/rejected_KL_mean": -119.60082244873047,
      "KL/std": 56.014801025390625,
      "epoch": 0.4459561602418745,
      "fcm_dpo/beta": 0.010448349639773369,
      "fcm_dpo/delta": 0.06282395124435425,
      "fcm_dpo/margin": 32.47895431518555,
      "fcm_dpo/q_t": 0.4243730306625366,
      "grad_norm": 18.652545928955078,
      "learning_rate": 3.4045544995169125e-07,
      "logits/chosen": 0.7032138109207153,
      "logits/rejected": 0.5987756848335266,
      "logps/chosen": -137.5227508544922,
      "logps/ref_chosen": -50.40088653564453,
      "logps/ref_rejected": -83.43521881103516,
      "logps/rejected": -203.03604125976562,
      "loss": 1.1691,
      "margin_dpo/margin_mean": 32.47895812988281,
      "margin_dpo/margin_std": 60.507484436035156,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -87.24478149414062,
      "KL/mean": -107.73387145996094,
      "KL/rejected_KL_mean": -128.22296142578125,
      "KL/std": 58.011165618896484,
      "epoch": 0.4474678760393046,
      "fcm_dpo/beta": 0.010410955175757408,
      "fcm_dpo/delta": -0.028351018205285072,
      "fcm_dpo/margin": 40.978172302246094,
      "fcm_dpo/q_t": 0.4043758809566498,
      "grad_norm": 13.773336410522461,
      "learning_rate": 3.392215553979679e-07,
      "logits/chosen": 0.6533064246177673,
      "logits/rejected": 0.6085612773895264,
      "logps/chosen": -156.3951416015625,
      "logps/ref_chosen": -69.15034484863281,
      "logps/ref_rejected": -89.60166931152344,
      "logps/rejected": -217.82461547851562,
      "loss": 1.1126,
      "margin_dpo/margin_mean": 40.978172302246094,
      "margin_dpo/margin_std": 65.01152038574219,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -92.09687805175781,
      "KL/mean": -114.38128662109375,
      "KL/rejected_KL_mean": -136.66571044921875,
      "KL/std": 51.77814483642578,
      "epoch": 0.4489795918367347,
      "fcm_dpo/beta": 0.010388961061835289,
      "fcm_dpo/delta": -0.06621909141540527,
      "fcm_dpo/margin": 44.56882095336914,
      "fcm_dpo/q_t": 0.39432087540626526,
      "grad_norm": 13.58420467376709,
      "learning_rate": 3.3798516512554485e-07,
      "logits/chosen": 0.6658183932304382,
      "logits/rejected": 0.6103072166442871,
      "logps/chosen": -150.11318969726562,
      "logps/ref_chosen": -58.01630401611328,
      "logps/ref_rejected": -69.95780944824219,
      "logps/rejected": -206.62350463867188,
      "loss": 1.0559,
      "margin_dpo/margin_mean": 44.568824768066406,
      "margin_dpo/margin_std": 55.705467224121094,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -90.31517028808594,
      "KL/mean": -107.02474975585938,
      "KL/rejected_KL_mean": -123.73432922363281,
      "KL/std": 54.960777282714844,
      "epoch": 0.4504913076341648,
      "fcm_dpo/beta": 0.010392475873231888,
      "fcm_dpo/delta": 0.05457156524062157,
      "fcm_dpo/margin": 33.419158935546875,
      "fcm_dpo/q_t": 0.42312532663345337,
      "grad_norm": 13.5759916305542,
      "learning_rate": 3.367463137189156e-07,
      "logits/chosen": 0.797134518623352,
      "logits/rejected": 0.7383297085762024,
      "logps/chosen": -146.48448181152344,
      "logps/ref_chosen": -56.1693115234375,
      "logps/ref_rejected": -68.55052185058594,
      "logps/rejected": -192.28485107421875,
      "loss": 1.181,
      "margin_dpo/margin_mean": 33.419158935546875,
      "margin_dpo/margin_std": 66.36347961425781,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -89.1030502319336,
      "KL/mean": -103.92301177978516,
      "KL/rejected_KL_mean": -118.74298095703125,
      "KL/std": 53.572784423828125,
      "epoch": 0.4520030234315949,
      "fcm_dpo/beta": 0.010414022952318192,
      "fcm_dpo/delta": -0.0021413981448858976,
      "fcm_dpo/margin": 29.63991355895996,
      "fcm_dpo/q_t": 0.4309845566749573,
      "grad_norm": 17.80376625061035,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": 0.5932430028915405,
      "logits/rejected": 0.5652042627334595,
      "logps/chosen": -151.4208526611328,
      "logps/ref_chosen": -62.31780242919922,
      "logps/ref_rejected": -72.60028839111328,
      "logps/rejected": -191.34326171875,
      "loss": 1.2229,
      "margin_dpo/margin_mean": 29.639911651611328,
      "margin_dpo/margin_std": 67.74291229248047,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -90.56175231933594,
      "KL/mean": -109.30287170410156,
      "KL/rejected_KL_mean": -128.04400634765625,
      "KL/std": 54.03219985961914,
      "epoch": 0.45351473922902497,
      "fcm_dpo/beta": 0.010439357720315456,
      "fcm_dpo/delta": 0.009030385874211788,
      "fcm_dpo/margin": 37.48223876953125,
      "fcm_dpo/q_t": 0.41219377517700195,
      "grad_norm": 14.686261177062988,
      "learning_rate": 3.3426136618426043e-07,
      "logits/chosen": 0.7061352729797363,
      "logits/rejected": 0.6380654573440552,
      "logps/chosen": -150.94332885742188,
      "logps/ref_chosen": -60.38157653808594,
      "logps/ref_rejected": -75.45442199707031,
      "logps/rejected": -203.49842834472656,
      "loss": 1.1422,
      "margin_dpo/margin_mean": 37.48223876953125,
      "margin_dpo/margin_std": 65.19305419921875,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -87.87615966796875,
      "KL/mean": -104.57803344726562,
      "KL/rejected_KL_mean": -121.2798843383789,
      "KL/std": 53.78392791748047,
      "epoch": 0.455026455026455,
      "fcm_dpo/beta": 0.010482998564839363,
      "fcm_dpo/delta": 0.05161427706480026,
      "fcm_dpo/margin": 33.40372848510742,
      "fcm_dpo/q_t": 0.42124661803245544,
      "grad_norm": 13.902162551879883,
      "learning_rate": 3.3301533956555885e-07,
      "logits/chosen": 0.7343845367431641,
      "logits/rejected": 0.7064188718795776,
      "logps/chosen": -140.72705078125,
      "logps/ref_chosen": -52.85089111328125,
      "logps/ref_rejected": -69.97584533691406,
      "logps/rejected": -191.2557373046875,
      "loss": 1.1701,
      "margin_dpo/margin_mean": 33.40372848510742,
      "margin_dpo/margin_std": 63.270591735839844,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -91.56837463378906,
      "KL/mean": -104.143310546875,
      "KL/rejected_KL_mean": -116.71824645996094,
      "KL/std": 54.15486526489258,
      "epoch": 0.4565381708238851,
      "fcm_dpo/beta": 0.010741431266069412,
      "fcm_dpo/delta": 0.13358688354492188,
      "fcm_dpo/margin": 25.149850845336914,
      "fcm_dpo/q_t": 0.4388381242752075,
      "grad_norm": 18.761884689331055,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": 0.5893893241882324,
      "logits/rejected": 0.5324574708938599,
      "logps/chosen": -158.53488159179688,
      "logps/ref_chosen": -66.96650695800781,
      "logps/ref_rejected": -88.09510803222656,
      "logps/rejected": -204.8133544921875,
      "loss": 1.233,
      "margin_dpo/margin_mean": 25.149852752685547,
      "margin_dpo/margin_std": 60.384178161621094,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -81.19476318359375,
      "KL/mean": -104.3687744140625,
      "KL/rejected_KL_mean": -127.54279327392578,
      "KL/std": 55.12614440917969,
      "epoch": 0.4580498866213152,
      "fcm_dpo/beta": 0.01067281048744917,
      "fcm_dpo/delta": -0.09954620897769928,
      "fcm_dpo/margin": 46.34803009033203,
      "fcm_dpo/q_t": 0.3895590305328369,
      "grad_norm": 12.493925094604492,
      "learning_rate": 3.3051635489464793e-07,
      "logits/chosen": 0.6610472202301025,
      "logits/rejected": 0.5935695171356201,
      "logps/chosen": -143.3162841796875,
      "logps/ref_chosen": -62.12152862548828,
      "logps/ref_rejected": -90.31204223632812,
      "logps/rejected": -217.85482788085938,
      "loss": 1.0754,
      "margin_dpo/margin_mean": 46.34803009033203,
      "margin_dpo/margin_std": 68.50209045410156,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -70.5691909790039,
      "KL/mean": -94.00971984863281,
      "KL/rejected_KL_mean": -117.45024871826172,
      "KL/std": 57.87809753417969,
      "epoch": 0.4595616024187453,
      "fcm_dpo/beta": 0.010455337353050709,
      "fcm_dpo/delta": -0.09490203857421875,
      "fcm_dpo/margin": 46.881046295166016,
      "fcm_dpo/q_t": 0.3863632082939148,
      "grad_norm": 13.3855619430542,
      "learning_rate": 3.292634667444117e-07,
      "logits/chosen": 0.6622233390808105,
      "logits/rejected": 0.6075294017791748,
      "logps/chosen": -131.2642822265625,
      "logps/ref_chosen": -60.695091247558594,
      "logps/ref_rejected": -78.2525405883789,
      "logps/rejected": -195.70278930664062,
      "loss": 1.0257,
      "margin_dpo/margin_mean": 46.88105010986328,
      "margin_dpo/margin_std": 52.37290573120117,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -82.67062377929688,
      "KL/mean": -99.52821350097656,
      "KL/rejected_KL_mean": -116.38580322265625,
      "KL/std": 54.650360107421875,
      "epoch": 0.46107331821617537,
      "fcm_dpo/beta": 0.010445987805724144,
      "fcm_dpo/delta": 0.04939526319503784,
      "fcm_dpo/margin": 33.71518325805664,
      "fcm_dpo/q_t": 0.42067134380340576,
      "grad_norm": 13.54196834564209,
      "learning_rate": 3.280083614246217e-07,
      "logits/chosen": 0.6038622260093689,
      "logits/rejected": 0.634021520614624,
      "logps/chosen": -155.3697509765625,
      "logps/ref_chosen": -72.69914245605469,
      "logps/ref_rejected": -65.65670776367188,
      "logps/rejected": -182.04251098632812,
      "loss": 1.1771,
      "margin_dpo/margin_mean": 33.715187072753906,
      "margin_dpo/margin_std": 65.36822509765625,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -76.23724365234375,
      "KL/mean": -94.49736022949219,
      "KL/rejected_KL_mean": -112.75747680664062,
      "KL/std": 52.728240966796875,
      "epoch": 0.46258503401360546,
      "fcm_dpo/beta": 0.01048213616013527,
      "fcm_dpo/delta": 0.0172632597386837,
      "fcm_dpo/margin": 36.520233154296875,
      "fcm_dpo/q_t": 0.4122518301010132,
      "grad_norm": 12.87689208984375,
      "learning_rate": 3.267510740432719e-07,
      "logits/chosen": 0.7085878849029541,
      "logits/rejected": 0.5961357355117798,
      "logps/chosen": -130.20777893066406,
      "logps/ref_chosen": -53.97052764892578,
      "logps/ref_rejected": -71.02423095703125,
      "logps/rejected": -183.78170776367188,
      "loss": 1.1092,
      "margin_dpo/margin_mean": 36.52022933959961,
      "margin_dpo/margin_std": 50.67652893066406,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -71.5609359741211,
      "KL/mean": -80.18887329101562,
      "KL/rejected_KL_mean": -88.81682586669922,
      "KL/std": 51.007423400878906,
      "epoch": 0.46409674981103555,
      "fcm_dpo/beta": 0.010636195540428162,
      "fcm_dpo/delta": 0.07790957391262054,
      "fcm_dpo/margin": 17.255882263183594,
      "fcm_dpo/q_t": 0.46016865968704224,
      "grad_norm": 17.592483520507812,
      "learning_rate": 3.2549163976939285e-07,
      "logits/chosen": 0.729952335357666,
      "logits/rejected": 0.6798655986785889,
      "logps/chosen": -128.9740447998047,
      "logps/ref_chosen": -57.413108825683594,
      "logps/ref_rejected": -68.68010711669922,
      "logps/rejected": -157.49693298339844,
      "loss": 1.3259,
      "margin_dpo/margin_mean": 17.255882263183594,
      "margin_dpo/margin_std": 65.2297134399414,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -69.87811279296875,
      "KL/mean": -87.25666809082031,
      "KL/rejected_KL_mean": -104.63520812988281,
      "KL/std": 52.88311767578125,
      "epoch": 0.4656084656084656,
      "fcm_dpo/beta": 0.010723689571022987,
      "fcm_dpo/delta": 0.02814718894660473,
      "fcm_dpo/margin": 34.75708770751953,
      "fcm_dpo/q_t": 0.41550326347351074,
      "grad_norm": 11.834020614624023,
      "learning_rate": 3.2423009383206874e-07,
      "logits/chosen": 0.6749851703643799,
      "logits/rejected": 0.6642191410064697,
      "logps/chosen": -136.47689819335938,
      "logps/ref_chosen": -66.59879302978516,
      "logps/ref_rejected": -74.337158203125,
      "logps/rejected": -178.9723663330078,
      "loss": 1.1479,
      "margin_dpo/margin_mean": 34.7570915222168,
      "margin_dpo/margin_std": 60.48528289794922,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -81.1751708984375,
      "KL/mean": -98.79702758789062,
      "KL/rejected_KL_mean": -116.41889190673828,
      "KL/std": 50.54866027832031,
      "epoch": 0.4671201814058957,
      "fcm_dpo/beta": 0.010767925530672073,
      "fcm_dpo/delta": 0.021030962467193604,
      "fcm_dpo/margin": 35.24372100830078,
      "fcm_dpo/q_t": 0.41353365778923035,
      "grad_norm": 11.959304809570312,
      "learning_rate": 3.229664715194511e-07,
      "logits/chosen": 0.7317670583724976,
      "logits/rejected": 0.6719903945922852,
      "logps/chosen": -146.56991577148438,
      "logps/ref_chosen": -65.39474487304688,
      "logps/ref_rejected": -75.70930480957031,
      "logps/rejected": -192.12820434570312,
      "loss": 1.1245,
      "margin_dpo/margin_mean": 35.24372100830078,
      "margin_dpo/margin_std": 54.025550842285156,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -82.6306381225586,
      "KL/mean": -93.36459350585938,
      "KL/rejected_KL_mean": -104.09854125976562,
      "KL/std": 46.84593200683594,
      "epoch": 0.46863189720332576,
      "fcm_dpo/beta": 0.010971201583743095,
      "fcm_dpo/delta": 0.06707384437322617,
      "fcm_dpo/margin": 21.467905044555664,
      "fcm_dpo/q_t": 0.44845932722091675,
      "grad_norm": 14.013663291931152,
      "learning_rate": 3.2170080817777257e-07,
      "logits/chosen": 0.6825644373893738,
      "logits/rejected": 0.6685233116149902,
      "logps/chosen": -157.29891967773438,
      "logps/ref_chosen": -74.66827392578125,
      "logps/ref_rejected": -80.5689697265625,
      "logps/rejected": -184.66751098632812,
      "loss": 1.256,
      "margin_dpo/margin_mean": 21.467906951904297,
      "margin_dpo/margin_std": 56.56273651123047,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -66.05873107910156,
      "KL/mean": -84.16709899902344,
      "KL/rejected_KL_mean": -102.27548217773438,
      "KL/std": 56.333656311035156,
      "epoch": 0.47014361300075586,
      "fcm_dpo/beta": 0.010918300598859787,
      "fcm_dpo/delta": 0.004174619913101196,
      "fcm_dpo/margin": 36.216758728027344,
      "fcm_dpo/q_t": 0.4116850197315216,
      "grad_norm": 13.10424518585205,
      "learning_rate": 3.204331392103574e-07,
      "logits/chosen": 0.6124294996261597,
      "logits/rejected": 0.46503138542175293,
      "logps/chosen": -125.79676055908203,
      "logps/ref_chosen": -59.738033294677734,
      "logps/ref_rejected": -93.60757446289062,
      "logps/rejected": -195.883056640625,
      "loss": 1.1203,
      "margin_dpo/margin_mean": 36.216758728027344,
      "margin_dpo/margin_std": 56.744300842285156,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -68.74053192138672,
      "KL/mean": -91.16142272949219,
      "KL/rejected_KL_mean": -113.58231353759766,
      "KL/std": 52.23027038574219,
      "epoch": 0.47165532879818595,
      "fcm_dpo/beta": 0.01086367480456829,
      "fcm_dpo/delta": -0.09151628613471985,
      "fcm_dpo/margin": 44.84178161621094,
      "fcm_dpo/q_t": 0.3876720070838928,
      "grad_norm": 12.957348823547363,
      "learning_rate": 3.1916350007663176e-07,
      "logits/chosen": 0.694495677947998,
      "logits/rejected": 0.5962769985198975,
      "logps/chosen": -122.55696868896484,
      "logps/ref_chosen": -53.816436767578125,
      "logps/ref_rejected": -68.6575698852539,
      "logps/rejected": -182.23988342285156,
      "loss": 1.0309,
      "margin_dpo/margin_mean": 44.84178161621094,
      "margin_dpo/margin_std": 51.59107208251953,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -70.51898956298828,
      "KL/mean": -82.34886169433594,
      "KL/rejected_KL_mean": -94.17871856689453,
      "KL/std": 50.00667953491211,
      "epoch": 0.47316704459561604,
      "fcm_dpo/beta": 0.011014842428267002,
      "fcm_dpo/delta": 0.1431160867214203,
      "fcm_dpo/margin": 23.659717559814453,
      "fcm_dpo/q_t": 0.4426537752151489,
      "grad_norm": 12.38918685913086,
      "learning_rate": 3.178919262911314e-07,
      "logits/chosen": 0.7352012991905212,
      "logits/rejected": 0.7140610814094543,
      "logps/chosen": -130.47634887695312,
      "logps/ref_chosen": -59.957359313964844,
      "logps/ref_rejected": -69.31729888916016,
      "logps/rejected": -163.4960174560547,
      "loss": 1.2298,
      "margin_dpo/margin_mean": 23.659717559814453,
      "margin_dpo/margin_std": 55.20978546142578,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -66.67250061035156,
      "KL/mean": -90.28575897216797,
      "KL/rejected_KL_mean": -113.8990249633789,
      "KL/std": 52.87154769897461,
      "epoch": 0.47467876039304613,
      "fcm_dpo/beta": 0.010864382609724998,
      "fcm_dpo/delta": -0.11978011578321457,
      "fcm_dpo/margin": 47.22651672363281,
      "fcm_dpo/q_t": 0.3849368691444397,
      "grad_norm": 12.260848045349121,
      "learning_rate": 3.166184534225087e-07,
      "logits/chosen": 0.6412978172302246,
      "logits/rejected": 0.6751775145530701,
      "logps/chosen": -136.94065856933594,
      "logps/ref_chosen": -70.26815795898438,
      "logps/ref_rejected": -69.23971557617188,
      "logps/rejected": -183.13873291015625,
      "loss": 1.031,
      "margin_dpo/margin_mean": 47.22651672363281,
      "margin_dpo/margin_std": 58.29710388183594,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -73.16780090332031,
      "KL/mean": -91.0605239868164,
      "KL/rejected_KL_mean": -108.9532470703125,
      "KL/std": 52.2242431640625,
      "epoch": 0.47619047619047616,
      "fcm_dpo/beta": 0.010901417583227158,
      "fcm_dpo/delta": 0.009739186614751816,
      "fcm_dpo/margin": 35.78544616699219,
      "fcm_dpo/q_t": 0.4104297459125519,
      "grad_norm": 12.950126647949219,
      "learning_rate": 3.1534311709253723e-07,
      "logits/chosen": 0.6009180545806885,
      "logits/rejected": 0.564073383808136,
      "logps/chosen": -140.96249389648438,
      "logps/ref_chosen": -67.79469299316406,
      "logps/ref_rejected": -74.55148315429688,
      "logps/rejected": -183.50473022460938,
      "loss": 1.1153,
      "margin_dpo/margin_mean": 35.78544616699219,
      "margin_dpo/margin_std": 53.13254928588867,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -67.7064437866211,
      "KL/mean": -91.29676818847656,
      "KL/rejected_KL_mean": -114.8870849609375,
      "KL/std": 54.27487564086914,
      "epoch": 0.47770219198790626,
      "fcm_dpo/beta": 0.010776463896036148,
      "fcm_dpo/delta": -0.11513285338878632,
      "fcm_dpo/margin": 47.18065643310547,
      "fcm_dpo/q_t": 0.3842179477214813,
      "grad_norm": 13.766993522644043,
      "learning_rate": 3.1406595297511564e-07,
      "logits/chosen": 0.5479520559310913,
      "logits/rejected": 0.41542547941207886,
      "logps/chosen": -122.99492645263672,
      "logps/ref_chosen": -55.288482666015625,
      "logps/ref_rejected": -96.15723419189453,
      "logps/rejected": -211.04432678222656,
      "loss": 1.0239,
      "margin_dpo/margin_mean": 47.1806526184082,
      "margin_dpo/margin_std": 52.00682067871094,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -64.59381103515625,
      "KL/mean": -89.12858581542969,
      "KL/rejected_KL_mean": -113.66336059570312,
      "KL/std": 51.89478302001953,
      "epoch": 0.47921390778533635,
      "fcm_dpo/beta": 0.010388361290097237,
      "fcm_dpo/delta": -0.11628536880016327,
      "fcm_dpo/margin": 49.06956481933594,
      "fcm_dpo/q_t": 0.3830963969230652,
      "grad_norm": 16.768312454223633,
      "learning_rate": 3.1278699679526975e-07,
      "logits/chosen": 0.681576669216156,
      "logits/rejected": 0.6368537545204163,
      "logps/chosen": -119.17518615722656,
      "logps/ref_chosen": -54.58137512207031,
      "logps/ref_rejected": -72.77232360839844,
      "logps/rejected": -186.43568420410156,
      "loss": 1.0224,
      "margin_dpo/margin_mean": 49.06956481933594,
      "margin_dpo/margin_std": 57.30916976928711,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -73.56890869140625,
      "KL/mean": -90.65696716308594,
      "KL/rejected_KL_mean": -107.74502563476562,
      "KL/std": 55.31390380859375,
      "epoch": 0.48072562358276644,
      "fcm_dpo/beta": 0.010412232019007206,
      "fcm_dpo/delta": 0.04578985273838043,
      "fcm_dpo/margin": 34.176109313964844,
      "fcm_dpo/q_t": 0.42275407910346985,
      "grad_norm": 12.690337181091309,
      "learning_rate": 3.1150628432815336e-07,
      "logits/chosen": 0.6963962316513062,
      "logits/rejected": 0.625290036201477,
      "logps/chosen": -126.45714569091797,
      "logps/ref_chosen": -52.88822937011719,
      "logps/ref_rejected": -80.63988494873047,
      "logps/rejected": -188.38491821289062,
      "loss": 1.1911,
      "margin_dpo/margin_mean": 34.176109313964844,
      "margin_dpo/margin_std": 71.72990417480469,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -69.83882141113281,
      "KL/mean": -93.78386688232422,
      "KL/rejected_KL_mean": -117.72889709472656,
      "KL/std": 56.57563018798828,
      "epoch": 0.48223733938019653,
      "fcm_dpo/beta": 0.010303584858775139,
      "fcm_dpo/delta": -0.09830920398235321,
      "fcm_dpo/margin": 47.89008331298828,
      "fcm_dpo/q_t": 0.39008021354675293,
      "grad_norm": 13.183405876159668,
      "learning_rate": 3.1022385139804707e-07,
      "logits/chosen": 0.642350971698761,
      "logits/rejected": 0.6263134479522705,
      "logps/chosen": -134.2021484375,
      "logps/ref_chosen": -64.36333465576172,
      "logps/ref_rejected": -79.47296142578125,
      "logps/rejected": -197.2018585205078,
      "loss": 1.057,
      "margin_dpo/margin_mean": 47.89008331298828,
      "margin_dpo/margin_std": 65.74710083007812,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -66.4017105102539,
      "KL/mean": -86.16297912597656,
      "KL/rejected_KL_mean": -105.92425537109375,
      "KL/std": 57.253265380859375,
      "epoch": 0.4837490551776266,
      "fcm_dpo/beta": 0.010127190500497818,
      "fcm_dpo/delta": -0.12487079203128815,
      "fcm_dpo/margin": 39.52253723144531,
      "fcm_dpo/q_t": 0.41120392084121704,
      "grad_norm": 14.036691665649414,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 0.5755819082260132,
      "logits/rejected": 0.534381091594696,
      "logps/chosen": -115.96045684814453,
      "logps/ref_chosen": -49.558746337890625,
      "logps/ref_rejected": -71.23444366455078,
      "logps/rejected": -177.15869140625,
      "loss": 1.131,
      "margin_dpo/margin_mean": 39.52253723144531,
      "margin_dpo/margin_std": 62.14351272583008,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -71.81083679199219,
      "KL/mean": -93.93961334228516,
      "KL/rejected_KL_mean": -116.06838989257812,
      "KL/std": 54.39446258544922,
      "epoch": 0.4852607709750567,
      "fcm_dpo/beta": 0.00986267440021038,
      "fcm_dpo/delta": -0.0397893451154232,
      "fcm_dpo/margin": 44.2575569152832,
      "fcm_dpo/q_t": 0.40008509159088135,
      "grad_norm": 19.625513076782227,
      "learning_rate": 3.0765396768561004e-07,
      "logits/chosen": 0.6763529777526855,
      "logits/rejected": 0.6586930751800537,
      "logps/chosen": -123.89610290527344,
      "logps/ref_chosen": -52.08526611328125,
      "logps/ref_rejected": -55.58674621582031,
      "logps/rejected": -171.65513610839844,
      "loss": 1.0957,
      "margin_dpo/margin_mean": 44.2575569152832,
      "margin_dpo/margin_std": 63.376220703125,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -83.38032531738281,
      "KL/mean": -109.8524169921875,
      "KL/rejected_KL_mean": -136.32449340820312,
      "KL/std": 60.07176208496094,
      "epoch": 0.48677248677248675,
      "fcm_dpo/beta": 0.009765025228261948,
      "fcm_dpo/delta": -0.12320294976234436,
      "fcm_dpo/margin": 52.944183349609375,
      "fcm_dpo/q_t": 0.3813475966453552,
      "grad_norm": 12.46308422088623,
      "learning_rate": 3.063665887884511e-07,
      "logits/chosen": 0.7335154414176941,
      "logits/rejected": 0.6464250087738037,
      "logps/chosen": -130.78443908691406,
      "logps/ref_chosen": -47.404109954833984,
      "logps/ref_rejected": -73.4260025024414,
      "logps/rejected": -209.75048828125,
      "loss": 1.0162,
      "margin_dpo/margin_mean": 52.944183349609375,
      "margin_dpo/margin_std": 60.424591064453125,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -85.19161987304688,
      "KL/mean": -102.52122497558594,
      "KL/rejected_KL_mean": -119.850830078125,
      "KL/std": 58.33759689331055,
      "epoch": 0.48828420256991684,
      "fcm_dpo/beta": 0.009794240817427635,
      "fcm_dpo/delta": 0.062395162880420685,
      "fcm_dpo/margin": 34.65920639038086,
      "fcm_dpo/q_t": 0.42608678340911865,
      "grad_norm": 13.784662246704102,
      "learning_rate": 3.0507763319663517e-07,
      "logits/chosen": 0.5997161269187927,
      "logits/rejected": 0.5194276571273804,
      "logps/chosen": -155.1979217529297,
      "logps/ref_chosen": -70.00630187988281,
      "logps/ref_rejected": -86.96690368652344,
      "logps/rejected": -206.81773376464844,
      "loss": 1.1954,
      "margin_dpo/margin_mean": 34.65920639038086,
      "margin_dpo/margin_std": 73.38899230957031,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -73.76261901855469,
      "KL/mean": -98.49959564208984,
      "KL/rejected_KL_mean": -123.23657989501953,
      "KL/std": 63.17657470703125,
      "epoch": 0.4897959183673469,
      "fcm_dpo/beta": 0.00964970514178276,
      "fcm_dpo/delta": -0.08181394636631012,
      "fcm_dpo/margin": 49.47395324707031,
      "fcm_dpo/q_t": 0.39091211557388306,
      "grad_norm": 18.15755844116211,
      "learning_rate": 3.0378713696502097e-07,
      "logits/chosen": 0.6848981380462646,
      "logits/rejected": 0.628462553024292,
      "logps/chosen": -129.6514434814453,
      "logps/ref_chosen": -55.88882064819336,
      "logps/ref_rejected": -75.23088073730469,
      "logps/rejected": -198.46746826171875,
      "loss": 1.0421,
      "margin_dpo/margin_mean": 49.47395324707031,
      "margin_dpo/margin_std": 59.73385238647461,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -92.95695495605469,
      "KL/mean": -115.26091003417969,
      "KL/rejected_KL_mean": -137.56484985351562,
      "KL/std": 59.86162185668945,
      "epoch": 0.491307634164777,
      "fcm_dpo/beta": 0.00955934077501297,
      "fcm_dpo/delta": -0.02813401073217392,
      "fcm_dpo/margin": 44.60791015625,
      "fcm_dpo/q_t": 0.4026256203651428,
      "grad_norm": 14.239675521850586,
      "learning_rate": 3.0249513619156206e-07,
      "logits/chosen": 0.6517459154129028,
      "logits/rejected": 0.5855910778045654,
      "logps/chosen": -157.10397338867188,
      "logps/ref_chosen": -64.14701843261719,
      "logps/ref_rejected": -79.91143798828125,
      "logps/rejected": -217.47628784179688,
      "loss": 1.1092,
      "margin_dpo/margin_mean": 44.60791015625,
      "margin_dpo/margin_std": 69.75248718261719,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -108.92955780029297,
      "KL/mean": -118.86935424804688,
      "KL/rejected_KL_mean": -128.80917358398438,
      "KL/std": 61.31150436401367,
      "epoch": 0.4928193499622071,
      "fcm_dpo/beta": 0.009740164503455162,
      "fcm_dpo/delta": 0.07853961735963821,
      "fcm_dpo/margin": 19.87961769104004,
      "fcm_dpo/q_t": 0.45686638355255127,
      "grad_norm": 14.16883373260498,
      "learning_rate": 3.012016670162977e-07,
      "logits/chosen": 0.6108545660972595,
      "logits/rejected": 0.6177682876586914,
      "logps/chosen": -184.46087646484375,
      "logps/ref_chosen": -75.53131103515625,
      "logps/ref_rejected": -76.5898666381836,
      "logps/rejected": -205.39903259277344,
      "loss": 1.3066,
      "margin_dpo/margin_mean": 19.879615783691406,
      "margin_dpo/margin_std": 67.34158325195312,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -100.75117492675781,
      "KL/mean": -118.57402801513672,
      "KL/rejected_KL_mean": -136.39688110351562,
      "KL/std": 63.51454162597656,
      "epoch": 0.4943310657596372,
      "fcm_dpo/beta": 0.009847394190728664,
      "fcm_dpo/delta": 0.05035046860575676,
      "fcm_dpo/margin": 35.64568328857422,
      "fcm_dpo/q_t": 0.42222487926483154,
      "grad_norm": 16.052671432495117,
      "learning_rate": 2.99906765620341e-07,
      "logits/chosen": 0.5630265474319458,
      "logits/rejected": 0.5328375101089478,
      "logps/chosen": -170.08834838867188,
      "logps/ref_chosen": -69.33717346191406,
      "logps/ref_rejected": -73.37751770019531,
      "logps/rejected": -209.77438354492188,
      "loss": 1.1821,
      "margin_dpo/margin_mean": 35.645687103271484,
      "margin_dpo/margin_std": 71.13593292236328,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -87.39762878417969,
      "KL/mean": -109.10784912109375,
      "KL/rejected_KL_mean": -130.81805419921875,
      "KL/std": 63.99862289428711,
      "epoch": 0.4958427815570673,
      "fcm_dpo/beta": 0.00981416366994381,
      "fcm_dpo/delta": -0.027305733412504196,
      "fcm_dpo/margin": 43.420433044433594,
      "fcm_dpo/q_t": 0.4043551981449127,
      "grad_norm": 13.175795555114746,
      "learning_rate": 2.9861046822486766e-07,
      "logits/chosen": 0.5762934684753418,
      "logits/rejected": 0.5433114171028137,
      "logps/chosen": -149.10385131835938,
      "logps/ref_chosen": -61.70623016357422,
      "logps/ref_rejected": -83.73808288574219,
      "logps/rejected": -214.55615234375,
      "loss": 1.0962,
      "margin_dpo/margin_mean": 43.42043685913086,
      "margin_dpo/margin_std": 63.37994384765625,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -96.39016723632812,
      "KL/mean": -118.81510925292969,
      "KL/rejected_KL_mean": -141.24005126953125,
      "KL/std": 64.04859161376953,
      "epoch": 0.4973544973544973,
      "fcm_dpo/beta": 0.009732028469443321,
      "fcm_dpo/delta": -0.038137733936309814,
      "fcm_dpo/margin": 44.84989929199219,
      "fcm_dpo/q_t": 0.4027097821235657,
      "grad_norm": 15.691971778869629,
      "learning_rate": 2.9731281109010253e-07,
      "logits/chosen": 0.701872706413269,
      "logits/rejected": 0.6433833837509155,
      "logps/chosen": -160.88858032226562,
      "logps/ref_chosen": -64.4984130859375,
      "logps/ref_rejected": -83.6591796875,
      "logps/rejected": -224.89923095703125,
      "loss": 1.0883,
      "margin_dpo/margin_mean": 44.84989929199219,
      "margin_dpo/margin_std": 64.86563873291016,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -82.26272583007812,
      "KL/mean": -106.7244644165039,
      "KL/rejected_KL_mean": -131.18618774414062,
      "KL/std": 62.682167053222656,
      "epoch": 0.4988662131519274,
      "fcm_dpo/beta": 0.009643211960792542,
      "fcm_dpo/delta": -0.07537820935249329,
      "fcm_dpo/margin": 48.92347717285156,
      "fcm_dpo/q_t": 0.3949437737464905,
      "grad_norm": 14.849321365356445,
      "learning_rate": 2.9601383051430505e-07,
      "logits/chosen": 0.6764267086982727,
      "logits/rejected": 0.6068094968795776,
      "logps/chosen": -137.06736755371094,
      "logps/ref_chosen": -54.80464172363281,
      "logps/ref_rejected": -75.3194351196289,
      "logps/rejected": -206.50563049316406,
      "loss": 1.1047,
      "margin_dpo/margin_mean": 48.92347717285156,
      "margin_dpo/margin_std": 78.65251922607422,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -90.41842651367188,
      "KL/mean": -120.35186004638672,
      "KL/rejected_KL_mean": -150.28529357910156,
      "KL/std": 66.08181762695312,
      "epoch": 0.5003779289493575,
      "fcm_dpo/beta": 0.0093461312353611,
      "fcm_dpo/delta": -0.16873988509178162,
      "fcm_dpo/margin": 59.86686706542969,
      "fcm_dpo/q_t": 0.37292051315307617,
      "grad_norm": 12.564268112182617,
      "learning_rate": 2.947135628327544e-07,
      "logits/chosen": 0.7615466713905334,
      "logits/rejected": 0.7352020740509033,
      "logps/chosen": -149.6610107421875,
      "logps/ref_chosen": -59.242584228515625,
      "logps/ref_rejected": -69.87483215332031,
      "logps/rejected": -220.16012573242188,
      "loss": 1.0103,
      "margin_dpo/margin_mean": 59.86686706542969,
      "margin_dpo/margin_std": 72.77942657470703,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -91.10173034667969,
      "KL/mean": -115.93171691894531,
      "KL/rejected_KL_mean": -140.76168823242188,
      "KL/std": 63.23088073730469,
      "epoch": 0.5018896447467877,
      "fcm_dpo/beta": 0.009242605417966843,
      "fcm_dpo/delta": -0.06282474100589752,
      "fcm_dpo/margin": 49.65996551513672,
      "fcm_dpo/q_t": 0.3964909017086029,
      "grad_norm": 13.50660228729248,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": 0.5807977318763733,
      "logits/rejected": 0.5386539101600647,
      "logps/chosen": -158.21148681640625,
      "logps/ref_chosen": -67.10975646972656,
      "logps/ref_rejected": -77.11839294433594,
      "logps/rejected": -217.8800811767578,
      "loss": 1.0674,
      "margin_dpo/margin_mean": 49.65996170043945,
      "margin_dpo/margin_std": 64.06481170654297,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -98.18170928955078,
      "KL/mean": -124.52760314941406,
      "KL/rejected_KL_mean": -150.8734893798828,
      "KL/std": 62.95512771606445,
      "epoch": 0.5034013605442177,
      "fcm_dpo/beta": 0.00904078409075737,
      "fcm_dpo/delta": -0.08011743426322937,
      "fcm_dpo/margin": 52.691776275634766,
      "fcm_dpo/q_t": 0.39210766553878784,
      "grad_norm": 12.745790481567383,
      "learning_rate": 2.921093116725076e-07,
      "logits/chosen": 0.6305129528045654,
      "logits/rejected": 0.5573608875274658,
      "logps/chosen": -156.56283569335938,
      "logps/ref_chosen": -58.381134033203125,
      "logps/ref_rejected": -85.02839660644531,
      "logps/rejected": -235.90188598632812,
      "loss": 1.0488,
      "margin_dpo/margin_mean": 52.691776275634766,
      "margin_dpo/margin_std": 66.21095275878906,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -93.12535095214844,
      "KL/mean": -112.17302703857422,
      "KL/rejected_KL_mean": -131.22067260742188,
      "KL/std": 65.46946716308594,
      "epoch": 0.5049130763416477,
      "fcm_dpo/beta": 0.009079881943762302,
      "fcm_dpo/delta": 0.05585712566971779,
      "fcm_dpo/margin": 38.0953254699707,
      "fcm_dpo/q_t": 0.423047810792923,
      "grad_norm": 13.126421928405762,
      "learning_rate": 2.9080540104031484e-07,
      "logits/chosen": 0.6646161675453186,
      "logits/rejected": 0.6201997399330139,
      "logps/chosen": -160.01736450195312,
      "logps/ref_chosen": -66.89199829101562,
      "logps/ref_rejected": -91.83695220947266,
      "logps/rejected": -223.05763244628906,
      "loss": 1.1835,
      "margin_dpo/margin_mean": 38.0953254699707,
      "margin_dpo/margin_std": 76.72137451171875,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -93.09357452392578,
      "KL/mean": -115.08125305175781,
      "KL/rejected_KL_mean": -137.06893920898438,
      "KL/std": 64.84080505371094,
      "epoch": 0.5064247921390779,
      "fcm_dpo/beta": 0.009129097685217857,
      "fcm_dpo/delta": -0.0019676052033901215,
      "fcm_dpo/margin": 43.975379943847656,
      "fcm_dpo/q_t": 0.41044336557388306,
      "grad_norm": 18.434982299804688,
      "learning_rate": 2.895003489933375e-07,
      "logits/chosen": 0.6574596762657166,
      "logits/rejected": 0.6203071475028992,
      "logps/chosen": -154.6080322265625,
      "logps/ref_chosen": -61.51445770263672,
      "logps/ref_rejected": -75.68916320800781,
      "logps/rejected": -212.75811767578125,
      "loss": 1.1278,
      "margin_dpo/margin_mean": 43.97538757324219,
      "margin_dpo/margin_std": 72.31240844726562,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -103.81600952148438,
      "KL/mean": -126.09706115722656,
      "KL/rejected_KL_mean": -148.37811279296875,
      "KL/std": 66.46051025390625,
      "epoch": 0.5079365079365079,
      "fcm_dpo/beta": 0.009012982249259949,
      "fcm_dpo/delta": -0.002887345850467682,
      "fcm_dpo/margin": 44.562095642089844,
      "fcm_dpo/q_t": 0.4118959605693817,
      "grad_norm": 12.43103313446045,
      "learning_rate": 2.8819419203668675e-07,
      "logits/chosen": 0.5812788605690002,
      "logits/rejected": 0.5591377019882202,
      "logps/chosen": -172.66607666015625,
      "logps/ref_chosen": -68.85006713867188,
      "logps/ref_rejected": -92.99603271484375,
      "logps/rejected": -241.37413024902344,
      "loss": 1.1288,
      "margin_dpo/margin_mean": 44.562095642089844,
      "margin_dpo/margin_std": 72.99846649169922,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -107.84507751464844,
      "KL/mean": -125.63278198242188,
      "KL/rejected_KL_mean": -143.4204864501953,
      "KL/std": 64.94393157958984,
      "epoch": 0.509448223733938,
      "fcm_dpo/beta": 0.009188439697027206,
      "fcm_dpo/delta": 0.07553110271692276,
      "fcm_dpo/margin": 35.575401306152344,
      "fcm_dpo/q_t": 0.42577266693115234,
      "grad_norm": 12.857539176940918,
      "learning_rate": 2.8688696670638053e-07,
      "logits/chosen": 0.5172953605651855,
      "logits/rejected": 0.48620158433914185,
      "logps/chosen": -181.0329132080078,
      "logps/ref_chosen": -73.18783569335938,
      "logps/ref_rejected": -86.89118957519531,
      "logps/rejected": -230.31167602539062,
      "loss": 1.1753,
      "margin_dpo/margin_mean": 35.575401306152344,
      "margin_dpo/margin_std": 67.20249938964844,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -103.88814544677734,
      "KL/mean": -122.87848663330078,
      "KL/rejected_KL_mean": -141.8688201904297,
      "KL/std": 63.28398895263672,
      "epoch": 0.5109599395313681,
      "fcm_dpo/beta": 0.00926903635263443,
      "fcm_dpo/delta": 0.04972708970308304,
      "fcm_dpo/margin": 37.98067855834961,
      "fcm_dpo/q_t": 0.42106711864471436,
      "grad_norm": 12.044840812683105,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": 0.625525951385498,
      "logits/rejected": 0.5995627641677856,
      "logps/chosen": -167.8277587890625,
      "logps/ref_chosen": -63.939613342285156,
      "logps/ref_rejected": -75.34243774414062,
      "logps/rejected": -217.21127319335938,
      "loss": 1.1662,
      "margin_dpo/margin_mean": 37.980674743652344,
      "margin_dpo/margin_std": 70.91877746582031,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -84.7950439453125,
      "KL/mean": -105.71109008789062,
      "KL/rejected_KL_mean": -126.62712097167969,
      "KL/std": 61.910316467285156,
      "epoch": 0.5124716553287982,
      "fcm_dpo/beta": 0.009312020614743233,
      "fcm_dpo/delta": 0.010858274064958096,
      "fcm_dpo/margin": 41.83207702636719,
      "fcm_dpo/q_t": 0.41156482696533203,
      "grad_norm": 13.738910675048828,
      "learning_rate": 2.842694572172736e-07,
      "logits/chosen": 0.8080116510391235,
      "logits/rejected": 0.7182115316390991,
      "logps/chosen": -130.34417724609375,
      "logps/ref_chosen": -45.54913330078125,
      "logps/ref_rejected": -67.0482177734375,
      "logps/rejected": -193.67535400390625,
      "loss": 1.1235,
      "margin_dpo/margin_mean": 41.83207702636719,
      "margin_dpo/margin_std": 65.60867309570312,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -96.89959716796875,
      "KL/mean": -118.34950256347656,
      "KL/rejected_KL_mean": -139.79940795898438,
      "KL/std": 68.1254653930664,
      "epoch": 0.5139833711262283,
      "fcm_dpo/beta": 0.009367447346448898,
      "fcm_dpo/delta": -0.0023114457726478577,
      "fcm_dpo/margin": 42.899810791015625,
      "fcm_dpo/q_t": 0.4121158719062805,
      "grad_norm": 12.531046867370605,
      "learning_rate": 2.8295924627584004e-07,
      "logits/chosen": 0.67206871509552,
      "logits/rejected": 0.6508908271789551,
      "logps/chosen": -150.90524291992188,
      "logps/ref_chosen": -54.00564956665039,
      "logps/ref_rejected": -61.314430236816406,
      "logps/rejected": -201.1138458251953,
      "loss": 1.1529,
      "margin_dpo/margin_mean": 42.899810791015625,
      "margin_dpo/margin_std": 79.07963562011719,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -94.7508773803711,
      "KL/mean": -119.87841033935547,
      "KL/rejected_KL_mean": -145.0059356689453,
      "KL/std": 66.08937072753906,
      "epoch": 0.5154950869236583,
      "fcm_dpo/beta": 0.009038900956511497,
      "fcm_dpo/delta": -0.1562565118074417,
      "fcm_dpo/margin": 50.25506591796875,
      "fcm_dpo/q_t": 0.3968254327774048,
      "grad_norm": 13.2722806930542,
      "learning_rate": 2.816481133934373e-07,
      "logits/chosen": 0.690357506275177,
      "logits/rejected": 0.6406093835830688,
      "logps/chosen": -158.14596557617188,
      "logps/ref_chosen": -63.39509582519531,
      "logps/ref_rejected": -76.20973205566406,
      "logps/rejected": -221.21566772460938,
      "loss": 1.0876,
      "margin_dpo/margin_mean": 50.255062103271484,
      "margin_dpo/margin_std": 69.64410400390625,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -94.90184020996094,
      "KL/mean": -119.85208129882812,
      "KL/rejected_KL_mean": -144.80233764648438,
      "KL/std": 69.12544250488281,
      "epoch": 0.5170068027210885,
      "fcm_dpo/beta": 0.008942769840359688,
      "fcm_dpo/delta": -0.04881645366549492,
      "fcm_dpo/margin": 49.9005126953125,
      "fcm_dpo/q_t": 0.4006722569465637,
      "grad_norm": 12.530938148498535,
      "learning_rate": 2.8033609524527046e-07,
      "logits/chosen": 0.757080078125,
      "logits/rejected": 0.7138710021972656,
      "logps/chosen": -147.94964599609375,
      "logps/ref_chosen": -53.047813415527344,
      "logps/ref_rejected": -68.2854232788086,
      "logps/rejected": -213.0877685546875,
      "loss": 1.0874,
      "margin_dpo/margin_mean": 49.900508880615234,
      "margin_dpo/margin_std": 72.3186264038086,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -88.19702911376953,
      "KL/mean": -105.32231140136719,
      "KL/rejected_KL_mean": -122.44758605957031,
      "KL/std": 64.40070343017578,
      "epoch": 0.5185185185185185,
      "fcm_dpo/beta": 0.008904541842639446,
      "fcm_dpo/delta": -0.02606440708041191,
      "fcm_dpo/margin": 34.25056838989258,
      "fcm_dpo/q_t": 0.4294404983520508,
      "grad_norm": 11.548450469970703,
      "learning_rate": 2.7902322853130753e-07,
      "logits/chosen": 0.5559418797492981,
      "logits/rejected": 0.5485849976539612,
      "logps/chosen": -158.7755584716797,
      "logps/ref_chosen": -70.57852935791016,
      "logps/ref_rejected": -84.73873901367188,
      "logps/rejected": -207.1863250732422,
      "loss": 1.1887,
      "margin_dpo/margin_mean": 34.25056838989258,
      "margin_dpo/margin_std": 65.59944152832031,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -97.81423950195312,
      "KL/mean": -123.5518798828125,
      "KL/rejected_KL_mean": -149.28952026367188,
      "KL/std": 65.39834594726562,
      "epoch": 0.5200302343159486,
      "fcm_dpo/beta": 0.008810698986053467,
      "fcm_dpo/delta": -0.056301526725292206,
      "fcm_dpo/margin": 51.47527313232422,
      "fcm_dpo/q_t": 0.3976425528526306,
      "grad_norm": 14.105023384094238,
      "learning_rate": 2.7770954997525274e-07,
      "logits/chosen": 0.6968499422073364,
      "logits/rejected": 0.627306342124939,
      "logps/chosen": -153.625244140625,
      "logps/ref_chosen": -55.811004638671875,
      "logps/ref_rejected": -84.77637481689453,
      "logps/rejected": -234.06588745117188,
      "loss": 1.072,
      "margin_dpo/margin_mean": 51.47527313232422,
      "margin_dpo/margin_std": 70.21475219726562,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -78.0170669555664,
      "KL/mean": -99.81597900390625,
      "KL/rejected_KL_mean": -121.61490631103516,
      "KL/std": 58.8808708190918,
      "epoch": 0.5215419501133787,
      "fcm_dpo/beta": 0.008850732818245888,
      "fcm_dpo/delta": 0.0144614577293396,
      "fcm_dpo/margin": 43.597835540771484,
      "fcm_dpo/q_t": 0.4125995635986328,
      "grad_norm": 13.581534385681152,
      "learning_rate": 2.7639509632351927e-07,
      "logits/chosen": 0.7043867111206055,
      "logits/rejected": 0.6588037014007568,
      "logps/chosen": -135.80316162109375,
      "logps/ref_chosen": -57.78609848022461,
      "logps/ref_rejected": -78.91847229003906,
      "logps/rejected": -200.53338623046875,
      "loss": 1.1266,
      "margin_dpo/margin_mean": 43.59783172607422,
      "margin_dpo/margin_std": 69.64117431640625,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -85.94567108154297,
      "KL/mean": -111.72139739990234,
      "KL/rejected_KL_mean": -137.49713134765625,
      "KL/std": 67.8180160522461,
      "epoch": 0.5230536659108088,
      "fcm_dpo/beta": 0.008791204541921616,
      "fcm_dpo/delta": -0.05581257864832878,
      "fcm_dpo/margin": 51.55143737792969,
      "fcm_dpo/q_t": 0.3968457877635956,
      "grad_norm": 14.196526527404785,
      "learning_rate": 2.7507990434420123e-07,
      "logits/chosen": 0.7042652368545532,
      "logits/rejected": 0.6169871687889099,
      "logps/chosen": -142.23080444335938,
      "logps/ref_chosen": -56.285125732421875,
      "logps/ref_rejected": -91.15303039550781,
      "logps/rejected": -228.650146484375,
      "loss": 1.0798,
      "margin_dpo/margin_mean": 51.55143737792969,
      "margin_dpo/margin_std": 71.25596618652344,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -94.4802474975586,
      "KL/mean": -114.50068664550781,
      "KL/rejected_KL_mean": -134.5211181640625,
      "KL/std": 65.73883819580078,
      "epoch": 0.5245653817082389,
      "fcm_dpo/beta": 0.008795950561761856,
      "fcm_dpo/delta": 0.049553703516721725,
      "fcm_dpo/margin": 40.040870666503906,
      "fcm_dpo/q_t": 0.421281099319458,
      "grad_norm": 15.615790367126465,
      "learning_rate": 2.737640108260456e-07,
      "logits/chosen": 0.7799099683761597,
      "logits/rejected": 0.728537380695343,
      "logps/chosen": -147.97979736328125,
      "logps/ref_chosen": -53.499542236328125,
      "logps/ref_rejected": -72.52565002441406,
      "logps/rejected": -207.04676818847656,
      "loss": 1.1524,
      "margin_dpo/margin_mean": 40.040870666503906,
      "margin_dpo/margin_std": 69.70988464355469,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -84.69274139404297,
      "KL/mean": -108.81976318359375,
      "KL/rejected_KL_mean": -132.94677734375,
      "KL/std": 63.638397216796875,
      "epoch": 0.5260770975056689,
      "fcm_dpo/beta": 0.008763780817389488,
      "fcm_dpo/delta": -0.024115797132253647,
      "fcm_dpo/margin": 48.25403594970703,
      "fcm_dpo/q_t": 0.4073898196220398,
      "grad_norm": 13.05552864074707,
      "learning_rate": 2.724474525774229e-07,
      "logits/chosen": 0.761476993560791,
      "logits/rejected": 0.7315517663955688,
      "logps/chosen": -135.47959899902344,
      "logps/ref_chosen": -50.78684997558594,
      "logps/ref_rejected": -68.63732147216797,
      "logps/rejected": -201.58409118652344,
      "loss": 1.115,
      "margin_dpo/margin_mean": 48.25403594970703,
      "margin_dpo/margin_std": 78.05335235595703,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -84.82550048828125,
      "KL/mean": -109.17646789550781,
      "KL/rejected_KL_mean": -133.52743530273438,
      "KL/std": 66.30206298828125,
      "epoch": 0.527588813303099,
      "fcm_dpo/beta": 0.008777445182204247,
      "fcm_dpo/delta": -0.028968583792448044,
      "fcm_dpo/margin": 48.701942443847656,
      "fcm_dpo/q_t": 0.40451472997665405,
      "grad_norm": 13.449342727661133,
      "learning_rate": 2.711302664252973e-07,
      "logits/chosen": 0.7194141745567322,
      "logits/rejected": 0.624089777469635,
      "logps/chosen": -138.1505126953125,
      "logps/ref_chosen": -53.325008392333984,
      "logps/ref_rejected": -83.21236419677734,
      "logps/rejected": -216.7397918701172,
      "loss": 1.0978,
      "margin_dpo/margin_mean": 48.70194625854492,
      "margin_dpo/margin_std": 71.7383804321289,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -90.23080444335938,
      "KL/mean": -121.05146026611328,
      "KL/rejected_KL_mean": -151.87213134765625,
      "KL/std": 71.03602600097656,
      "epoch": 0.5291005291005291,
      "fcm_dpo/beta": 0.008537888526916504,
      "fcm_dpo/delta": -0.13364244997501373,
      "fcm_dpo/margin": 61.64132308959961,
      "fcm_dpo/q_t": 0.3797275424003601,
      "grad_norm": 15.501083374023438,
      "learning_rate": 2.698124892141971e-07,
      "logits/chosen": 0.6883540153503418,
      "logits/rejected": 0.6040031909942627,
      "logps/chosen": -151.8565673828125,
      "logps/ref_chosen": -61.625770568847656,
      "logps/ref_rejected": -87.63627624511719,
      "logps/rejected": -239.50839233398438,
      "loss": 1.012,
      "margin_dpo/margin_mean": 61.641326904296875,
      "margin_dpo/margin_std": 70.89884185791016,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -86.40487670898438,
      "KL/mean": -109.91165161132812,
      "KL/rejected_KL_mean": -133.41842651367188,
      "KL/std": 63.214752197265625,
      "epoch": 0.5306122448979592,
      "fcm_dpo/beta": 0.008473677560687065,
      "fcm_dpo/delta": 0.0015265997499227524,
      "fcm_dpo/margin": 47.013553619384766,
      "fcm_dpo/q_t": 0.40819916129112244,
      "grad_norm": 13.260918617248535,
      "learning_rate": 2.6849415780518357e-07,
      "logits/chosen": 0.631534218788147,
      "logits/rejected": 0.5537710189819336,
      "logps/chosen": -142.66122436523438,
      "logps/ref_chosen": -56.2563362121582,
      "logps/ref_rejected": -79.11589813232422,
      "logps/rejected": -212.53433227539062,
      "loss": 1.1328,
      "margin_dpo/margin_mean": 47.013553619384766,
      "margin_dpo/margin_std": 78.18861389160156,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -86.28941345214844,
      "KL/mean": -112.20276641845703,
      "KL/rejected_KL_mean": -138.11611938476562,
      "KL/std": 67.36019897460938,
      "epoch": 0.5321239606953893,
      "fcm_dpo/beta": 0.008432027883827686,
      "fcm_dpo/delta": -0.038864314556121826,
      "fcm_dpo/margin": 51.82670593261719,
      "fcm_dpo/q_t": 0.4009664058685303,
      "grad_norm": 12.21044921875,
      "learning_rate": 2.6717530907482024e-07,
      "logits/chosen": 0.7267534136772156,
      "logits/rejected": 0.6707276701927185,
      "logps/chosen": -149.34136962890625,
      "logps/ref_chosen": -63.05195236206055,
      "logps/ref_rejected": -85.52035522460938,
      "logps/rejected": -223.636474609375,
      "loss": 1.0834,
      "margin_dpo/margin_mean": 51.82670593261719,
      "margin_dpo/margin_std": 72.85710906982422,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -84.61532592773438,
      "KL/mean": -109.72518920898438,
      "KL/rejected_KL_mean": -134.83505249023438,
      "KL/std": 65.03328704833984,
      "epoch": 0.5336356764928194,
      "fcm_dpo/beta": 0.008424321189522743,
      "fcm_dpo/delta": -0.024160068482160568,
      "fcm_dpo/margin": 50.21971130371094,
      "fcm_dpo/q_t": 0.4032408595085144,
      "grad_norm": 11.878581047058105,
      "learning_rate": 2.658559799141411e-07,
      "logits/chosen": 0.6597447395324707,
      "logits/rejected": 0.6647744178771973,
      "logps/chosen": -153.62451171875,
      "logps/ref_chosen": -69.00918579101562,
      "logps/ref_rejected": -72.65840148925781,
      "logps/rejected": -207.49343872070312,
      "loss": 1.0909,
      "margin_dpo/margin_mean": 50.21971130371094,
      "margin_dpo/margin_std": 70.21359252929688,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -86.30844116210938,
      "KL/mean": -114.49634552001953,
      "KL/rejected_KL_mean": -142.68423461914062,
      "KL/std": 63.66696548461914,
      "epoch": 0.5351473922902494,
      "fcm_dpo/beta": 0.008266786113381386,
      "fcm_dpo/delta": -0.07003847509622574,
      "fcm_dpo/margin": 56.37580871582031,
      "fcm_dpo/q_t": 0.3940245509147644,
      "grad_norm": 13.222548484802246,
      "learning_rate": 2.6453620722761895e-07,
      "logits/chosen": 0.7399217486381531,
      "logits/rejected": 0.6065776348114014,
      "logps/chosen": -126.0967788696289,
      "logps/ref_chosen": -39.78833770751953,
      "logps/ref_rejected": -69.56885528564453,
      "logps/rejected": -212.25311279296875,
      "loss": 1.0692,
      "margin_dpo/margin_mean": 56.37581253051758,
      "margin_dpo/margin_std": 76.49386596679688,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -91.62583923339844,
      "KL/mean": -120.8445053100586,
      "KL/rejected_KL_mean": -150.06314086914062,
      "KL/std": 70.24827575683594,
      "epoch": 0.5366591080876795,
      "fcm_dpo/beta": 0.008196991868317127,
      "fcm_dpo/delta": -0.08291341364383698,
      "fcm_dpo/margin": 58.43730926513672,
      "fcm_dpo/q_t": 0.39158326387405396,
      "grad_norm": 15.241929054260254,
      "learning_rate": 2.632160279321328e-07,
      "logits/chosen": 0.7336651086807251,
      "logits/rejected": 0.5956906080245972,
      "logps/chosen": -137.8812255859375,
      "logps/ref_chosen": -46.25537872314453,
      "logps/ref_rejected": -78.20236206054688,
      "logps/rejected": -228.2655029296875,
      "loss": 1.069,
      "margin_dpo/margin_mean": 58.43730926513672,
      "margin_dpo/margin_std": 81.20545959472656,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -87.25131225585938,
      "KL/mean": -110.87397003173828,
      "KL/rejected_KL_mean": -134.49661254882812,
      "KL/std": 67.73360443115234,
      "epoch": 0.5381708238851096,
      "fcm_dpo/beta": 0.00812261551618576,
      "fcm_dpo/delta": 0.016548369079828262,
      "fcm_dpo/margin": 47.24530792236328,
      "fcm_dpo/q_t": 0.41417133808135986,
      "grad_norm": 12.109288215637207,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": 0.7372743487358093,
      "logits/rejected": 0.6521209478378296,
      "logps/chosen": -135.157470703125,
      "logps/ref_chosen": -47.906158447265625,
      "logps/ref_rejected": -74.29397583007812,
      "logps/rejected": -208.79058837890625,
      "loss": 1.1629,
      "margin_dpo/margin_mean": 47.24530792236328,
      "margin_dpo/margin_std": 88.80447387695312,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -104.42169189453125,
      "KL/mean": -125.28157043457031,
      "KL/rejected_KL_mean": -146.14144897460938,
      "KL/std": 66.77864074707031,
      "epoch": 0.5396825396825397,
      "fcm_dpo/beta": 0.008050942793488503,
      "fcm_dpo/delta": -0.07113456726074219,
      "fcm_dpo/margin": 41.71974182128906,
      "fcm_dpo/q_t": 0.4217901825904846,
      "grad_norm": 12.633148193359375,
      "learning_rate": 2.6057459723762076e-07,
      "logits/chosen": 0.6925714015960693,
      "logits/rejected": 0.6672199368476868,
      "logps/chosen": -167.05670166015625,
      "logps/ref_chosen": -62.63500213623047,
      "logps/ref_rejected": -65.11399841308594,
      "logps/rejected": -211.25543212890625,
      "loss": 1.1656,
      "margin_dpo/margin_mean": 41.71974563598633,
      "margin_dpo/margin_std": 71.41889953613281,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -99.59010314941406,
      "KL/mean": -128.130859375,
      "KL/rejected_KL_mean": -156.671630859375,
      "KL/std": 68.63123321533203,
      "epoch": 0.5411942554799698,
      "fcm_dpo/beta": 0.008035003207623959,
      "fcm_dpo/delta": -0.06208521127700806,
      "fcm_dpo/margin": 57.08154296875,
      "fcm_dpo/q_t": 0.3969094753265381,
      "grad_norm": 15.602532386779785,
      "learning_rate": 2.5925341972508954e-07,
      "logits/chosen": 0.6635780334472656,
      "logits/rejected": 0.6794674396514893,
      "logps/chosen": -166.79971313476562,
      "logps/ref_chosen": -67.20960998535156,
      "logps/ref_rejected": -69.34715270996094,
      "logps/rejected": -226.018798828125,
      "loss": 1.0743,
      "margin_dpo/margin_mean": 57.08154296875,
      "margin_dpo/margin_std": 77.6881103515625,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -111.93756103515625,
      "KL/mean": -126.5499267578125,
      "KL/rejected_KL_mean": -141.16229248046875,
      "KL/std": 67.31485748291016,
      "epoch": 0.5427059712773998,
      "fcm_dpo/beta": 0.007966868579387665,
      "fcm_dpo/delta": 0.01917518675327301,
      "fcm_dpo/margin": 29.224727630615234,
      "fcm_dpo/q_t": 0.44687217473983765,
      "grad_norm": 14.170949935913086,
      "learning_rate": 2.579319833745169e-07,
      "logits/chosen": 0.6356140971183777,
      "logits/rejected": 0.6054831743240356,
      "logps/chosen": -174.46334838867188,
      "logps/ref_chosen": -62.52578353881836,
      "logps/ref_rejected": -76.63114929199219,
      "logps/rejected": -217.79342651367188,
      "loss": 1.2444,
      "margin_dpo/margin_mean": 29.224727630615234,
      "margin_dpo/margin_std": 69.22132873535156,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -109.06979370117188,
      "KL/mean": -133.2952880859375,
      "KL/rejected_KL_mean": -157.52076721191406,
      "KL/std": 71.7374267578125,
      "epoch": 0.54421768707483,
      "fcm_dpo/beta": 0.007983379997313023,
      "fcm_dpo/delta": 0.01368173211812973,
      "fcm_dpo/margin": 48.45096206665039,
      "fcm_dpo/q_t": 0.41253405809402466,
      "grad_norm": 11.498431205749512,
      "learning_rate": 2.5661032514931834e-07,
      "logits/chosen": 0.5817567706108093,
      "logits/rejected": 0.48873624205589294,
      "logps/chosen": -172.55752563476562,
      "logps/ref_chosen": -63.48772048950195,
      "logps/ref_rejected": -90.6891098022461,
      "logps/rejected": -248.20986938476562,
      "loss": 1.1163,
      "margin_dpo/margin_mean": 48.45096206665039,
      "margin_dpo/margin_std": 72.77732849121094,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -105.44109344482422,
      "KL/mean": -134.56918334960938,
      "KL/rejected_KL_mean": -163.69729614257812,
      "KL/std": 71.06137084960938,
      "epoch": 0.54572940287226,
      "fcm_dpo/beta": 0.007947279140353203,
      "fcm_dpo/delta": -0.06600625813007355,
      "fcm_dpo/margin": 58.25619888305664,
      "fcm_dpo/q_t": 0.39347726106643677,
      "grad_norm": 11.478053092956543,
      "learning_rate": 2.552884820191154e-07,
      "logits/chosen": 0.761210560798645,
      "logits/rejected": 0.7106046676635742,
      "logps/chosen": -163.35824584960938,
      "logps/ref_chosen": -57.917144775390625,
      "logps/ref_rejected": -72.39089965820312,
      "logps/rejected": -236.08819580078125,
      "loss": 1.0523,
      "margin_dpo/margin_mean": 58.25619888305664,
      "margin_dpo/margin_std": 71.41291809082031,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -107.20101928710938,
      "KL/mean": -136.78494262695312,
      "KL/rejected_KL_mean": -166.3688507080078,
      "KL/std": 74.79295349121094,
      "epoch": 0.54724111866969,
      "fcm_dpo/beta": 0.00786098837852478,
      "fcm_dpo/delta": -0.06851021200418472,
      "fcm_dpo/margin": 59.16782760620117,
      "fcm_dpo/q_t": 0.3969106078147888,
      "grad_norm": 13.232868194580078,
      "learning_rate": 2.53966490958702e-07,
      "logits/chosen": 0.7699177265167236,
      "logits/rejected": 0.6532548666000366,
      "logps/chosen": -170.64450073242188,
      "logps/ref_chosen": -63.4434700012207,
      "logps/ref_rejected": -103.45516967773438,
      "logps/rejected": -269.8240051269531,
      "loss": 1.0851,
      "margin_dpo/margin_mean": 59.16782760620117,
      "margin_dpo/margin_std": 87.77825927734375,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -112.96141815185547,
      "KL/mean": -141.4286651611328,
      "KL/rejected_KL_mean": -169.89590454101562,
      "KL/std": 68.03427124023438,
      "epoch": 0.5487528344671202,
      "fcm_dpo/beta": 0.007731410674750805,
      "fcm_dpo/delta": -0.0421409048140049,
      "fcm_dpo/margin": 56.934486389160156,
      "fcm_dpo/q_t": 0.3995745778083801,
      "grad_norm": 14.695847511291504,
      "learning_rate": 2.526443889470099e-07,
      "logits/chosen": 0.7318873405456543,
      "logits/rejected": 0.5941910743713379,
      "logps/chosen": -161.61325073242188,
      "logps/ref_chosen": -48.65182876586914,
      "logps/ref_rejected": -88.65904235839844,
      "logps/rejected": -258.5549621582031,
      "loss": 1.0773,
      "margin_dpo/margin_mean": 56.934486389160156,
      "margin_dpo/margin_std": 77.69886779785156,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -100.91127014160156,
      "KL/mean": -132.60020446777344,
      "KL/rejected_KL_mean": -164.28912353515625,
      "KL/std": 75.34201049804688,
      "epoch": 0.5502645502645502,
      "fcm_dpo/beta": 0.007619412615895271,
      "fcm_dpo/delta": -0.08720940351486206,
      "fcm_dpo/margin": 63.37786865234375,
      "fcm_dpo/q_t": 0.392307311296463,
      "grad_norm": 11.564719200134277,
      "learning_rate": 2.513222129660744e-07,
      "logits/chosen": 0.5989806652069092,
      "logits/rejected": 0.508395791053772,
      "logps/chosen": -158.7823486328125,
      "logps/ref_chosen": -57.87107467651367,
      "logps/ref_rejected": -80.95503234863281,
      "logps/rejected": -245.24417114257812,
      "loss": 1.0774,
      "margin_dpo/margin_mean": 63.377872467041016,
      "margin_dpo/margin_std": 93.51548767089844,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -91.82067108154297,
      "KL/mean": -121.90553283691406,
      "KL/rejected_KL_mean": -151.99041748046875,
      "KL/std": 78.06063079833984,
      "epoch": 0.5517762660619804,
      "fcm_dpo/beta": 0.007519586943089962,
      "fcm_dpo/delta": -0.055061712861061096,
      "fcm_dpo/margin": 60.169734954833984,
      "fcm_dpo/q_t": 0.3952232897281647,
      "grad_norm": 10.55562973022461,
      "learning_rate": 2.5e-07,
      "logits/chosen": 0.7038200497627258,
      "logits/rejected": 0.6985729336738586,
      "logps/chosen": -156.76284790039062,
      "logps/ref_chosen": -64.94217681884766,
      "logps/ref_rejected": -74.8599853515625,
      "logps/rejected": -226.8503875732422,
      "loss": 1.0467,
      "margin_dpo/margin_mean": 60.16973876953125,
      "margin_dpo/margin_std": 68.7835693359375,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -92.98997497558594,
      "KL/mean": -117.69391632080078,
      "KL/rejected_KL_mean": -142.39785766601562,
      "KL/std": 69.93231201171875,
      "epoch": 0.5532879818594104,
      "fcm_dpo/beta": 0.00756697915494442,
      "fcm_dpo/delta": 0.02677847445011139,
      "fcm_dpo/margin": 49.40788269042969,
      "fcm_dpo/q_t": 0.41649293899536133,
      "grad_norm": 14.271859169006348,
      "learning_rate": 2.486777870339255e-07,
      "logits/chosen": 0.6564346551895142,
      "logits/rejected": 0.6428935527801514,
      "logps/chosen": -148.1559600830078,
      "logps/ref_chosen": -55.16598129272461,
      "logps/ref_rejected": -65.26121520996094,
      "logps/rejected": -207.6590576171875,
      "loss": 1.1583,
      "margin_dpo/margin_mean": 49.40788269042969,
      "margin_dpo/margin_std": 90.67138671875,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -104.67523193359375,
      "KL/mean": -129.92697143554688,
      "KL/rejected_KL_mean": -155.17872619628906,
      "KL/std": 71.09664916992188,
      "epoch": 0.5547996976568406,
      "fcm_dpo/beta": 0.007551530376076698,
      "fcm_dpo/delta": 0.019268203526735306,
      "fcm_dpo/margin": 50.503501892089844,
      "fcm_dpo/q_t": 0.4110247492790222,
      "grad_norm": 12.524863243103027,
      "learning_rate": 2.4735561105299014e-07,
      "logits/chosen": 0.6887466907501221,
      "logits/rejected": 0.5802679061889648,
      "logps/chosen": -160.68569946289062,
      "logps/ref_chosen": -56.01046371459961,
      "logps/ref_rejected": -77.31010437011719,
      "logps/rejected": -232.48883056640625,
      "loss": 1.1223,
      "margin_dpo/margin_mean": 50.503501892089844,
      "margin_dpo/margin_std": 77.30694580078125,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -115.24920654296875,
      "KL/mean": -138.61520385742188,
      "KL/rejected_KL_mean": -161.981201171875,
      "KL/std": 70.51954650878906,
      "epoch": 0.5563114134542706,
      "fcm_dpo/beta": 0.007625661790370941,
      "fcm_dpo/delta": 0.04526998847723007,
      "fcm_dpo/margin": 46.73200225830078,
      "fcm_dpo/q_t": 0.4176589548587799,
      "grad_norm": 13.562691688537598,
      "learning_rate": 2.46033509041298e-07,
      "logits/chosen": 0.5015436410903931,
      "logits/rejected": 0.5021830797195435,
      "logps/chosen": -190.0784912109375,
      "logps/ref_chosen": -74.82927703857422,
      "logps/ref_rejected": -76.11680603027344,
      "logps/rejected": -238.09800720214844,
      "loss": 1.1396,
      "margin_dpo/margin_mean": 46.731998443603516,
      "margin_dpo/margin_std": 75.205810546875,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -108.24713134765625,
      "KL/mean": -128.3904266357422,
      "KL/rejected_KL_mean": -148.53372192382812,
      "KL/std": 71.04008483886719,
      "epoch": 0.5578231292517006,
      "fcm_dpo/beta": 0.007783809676766396,
      "fcm_dpo/delta": 0.08842340856790543,
      "fcm_dpo/margin": 40.286590576171875,
      "fcm_dpo/q_t": 0.42879199981689453,
      "grad_norm": 12.779073715209961,
      "learning_rate": 2.447115179808846e-07,
      "logits/chosen": 0.6825852990150452,
      "logits/rejected": 0.6281259059906006,
      "logps/chosen": -166.57334899902344,
      "logps/ref_chosen": -58.32621765136719,
      "logps/ref_rejected": -80.92183685302734,
      "logps/rejected": -229.45556640625,
      "loss": 1.1802,
      "margin_dpo/margin_mean": 40.286590576171875,
      "margin_dpo/margin_std": 75.56928253173828,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -103.44606018066406,
      "KL/mean": -132.55752563476562,
      "KL/rejected_KL_mean": -161.66897583007812,
      "KL/std": 74.00811004638672,
      "epoch": 0.5593348450491308,
      "fcm_dpo/beta": 0.007711863610893488,
      "fcm_dpo/delta": -0.051456257700920105,
      "fcm_dpo/margin": 58.222930908203125,
      "fcm_dpo/q_t": 0.3976071774959564,
      "grad_norm": 13.002893447875977,
      "learning_rate": 2.4338967485068164e-07,
      "logits/chosen": 0.7718208432197571,
      "logits/rejected": 0.7014021873474121,
      "logps/chosen": -156.32977294921875,
      "logps/ref_chosen": -52.88372039794922,
      "logps/ref_rejected": -79.43692016601562,
      "logps/rejected": -241.10589599609375,
      "loss": 1.0894,
      "margin_dpo/margin_mean": 58.22292709350586,
      "margin_dpo/margin_std": 86.16522216796875,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -103.91604614257812,
      "KL/mean": -129.42440795898438,
      "KL/rejected_KL_mean": -154.93276977539062,
      "KL/std": 72.055419921875,
      "epoch": 0.5608465608465608,
      "fcm_dpo/beta": 0.0077507393434643745,
      "fcm_dpo/delta": 0.004167079925537109,
      "fcm_dpo/margin": 51.016727447509766,
      "fcm_dpo/q_t": 0.4105232357978821,
      "grad_norm": 15.76284122467041,
      "learning_rate": 2.420680166254831e-07,
      "logits/chosen": 0.8419981002807617,
      "logits/rejected": 0.8073742389678955,
      "logps/chosen": -153.1402587890625,
      "logps/ref_chosen": -49.224212646484375,
      "logps/ref_rejected": -63.348472595214844,
      "logps/rejected": -218.28125,
      "loss": 1.1187,
      "margin_dpo/margin_mean": 51.016727447509766,
      "margin_dpo/margin_std": 78.32734680175781,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -111.38978576660156,
      "KL/mean": -126.72420501708984,
      "KL/rejected_KL_mean": -142.05862426757812,
      "KL/std": 71.40890502929688,
      "epoch": 0.562358276643991,
      "fcm_dpo/beta": 0.007726870942860842,
      "fcm_dpo/delta": 0.02053908072412014,
      "fcm_dpo/margin": 30.6688289642334,
      "fcm_dpo/q_t": 0.4469439387321472,
      "grad_norm": 16.74570083618164,
      "learning_rate": 2.4074658027491044e-07,
      "logits/chosen": 0.754707932472229,
      "logits/rejected": 0.6516159772872925,
      "logps/chosen": -163.65933227539062,
      "logps/ref_chosen": -52.269554138183594,
      "logps/ref_rejected": -72.99522399902344,
      "logps/rejected": -215.0538330078125,
      "loss": 1.2877,
      "margin_dpo/margin_mean": 30.66883087158203,
      "margin_dpo/margin_std": 92.51069641113281,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -123.06751251220703,
      "KL/mean": -144.330078125,
      "KL/rejected_KL_mean": -165.5926513671875,
      "KL/std": 71.5724105834961,
      "epoch": 0.563869992441421,
      "fcm_dpo/beta": 0.007832320407032967,
      "fcm_dpo/delta": 0.0691244974732399,
      "fcm_dpo/margin": 42.5251350402832,
      "fcm_dpo/q_t": 0.42559584975242615,
      "grad_norm": 14.197022438049316,
      "learning_rate": 2.394254027623792e-07,
      "logits/chosen": 0.7258500456809998,
      "logits/rejected": 0.6516068577766418,
      "logps/chosen": -184.18051147460938,
      "logps/ref_chosen": -61.112998962402344,
      "logps/ref_rejected": -76.24851989746094,
      "logps/rejected": -241.84115600585938,
      "loss": 1.2035,
      "margin_dpo/margin_mean": 42.5251350402832,
      "margin_dpo/margin_std": 92.18357849121094,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -102.48390197753906,
      "KL/mean": -137.93734741210938,
      "KL/rejected_KL_mean": -173.3907928466797,
      "KL/std": 74.25117492675781,
      "epoch": 0.5653817082388511,
      "fcm_dpo/beta": 0.007663751021027565,
      "fcm_dpo/delta": -0.15193237364292145,
      "fcm_dpo/margin": 70.90689086914062,
      "fcm_dpo/q_t": 0.376120924949646,
      "grad_norm": 13.917801856994629,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 0.620780348777771,
      "logits/rejected": 0.6218676567077637,
      "logps/chosen": -175.15310668945312,
      "logps/ref_chosen": -72.66920471191406,
      "logps/ref_rejected": -76.83158874511719,
      "logps/rejected": -250.22238159179688,
      "loss": 1.013,
      "margin_dpo/margin_mean": 70.90689086914062,
      "margin_dpo/margin_std": 85.2769775390625,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -101.1163330078125,
      "KL/mean": -126.18803405761719,
      "KL/rejected_KL_mean": -151.25973510742188,
      "KL/std": 77.74549865722656,
      "epoch": 0.5668934240362812,
      "fcm_dpo/beta": 0.007613973692059517,
      "fcm_dpo/delta": 0.018803158774971962,
      "fcm_dpo/margin": 50.143394470214844,
      "fcm_dpo/q_t": 0.4139704704284668,
      "grad_norm": 15.40609073638916,
      "learning_rate": 2.3678397206786715e-07,
      "logits/chosen": 0.7243174314498901,
      "logits/rejected": 0.6623414754867554,
      "logps/chosen": -158.79965209960938,
      "logps/ref_chosen": -57.68330383300781,
      "logps/ref_rejected": -79.34097290039062,
      "logps/rejected": -230.6007080078125,
      "loss": 1.1434,
      "margin_dpo/margin_mean": 50.14339065551758,
      "margin_dpo/margin_std": 86.62193298339844,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -110.031982421875,
      "KL/mean": -141.00003051757812,
      "KL/rejected_KL_mean": -171.96807861328125,
      "KL/std": 77.46763610839844,
      "epoch": 0.5684051398337112,
      "fcm_dpo/beta": 0.00755238626152277,
      "fcm_dpo/delta": -0.07124269008636475,
      "fcm_dpo/margin": 61.936100006103516,
      "fcm_dpo/q_t": 0.395659863948822,
      "grad_norm": 13.033273696899414,
      "learning_rate": 2.3546379277238103e-07,
      "logits/chosen": 0.7544640898704529,
      "logits/rejected": 0.6791675090789795,
      "logps/chosen": -161.7060546875,
      "logps/ref_chosen": -51.674072265625,
      "logps/ref_rejected": -75.69713592529297,
      "logps/rejected": -247.66522216796875,
      "loss": 1.0809,
      "margin_dpo/margin_mean": 61.936100006103516,
      "margin_dpo/margin_std": 90.93395233154297,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -114.1806640625,
      "KL/mean": -136.430908203125,
      "KL/rejected_KL_mean": -158.68118286132812,
      "KL/std": 70.613525390625,
      "epoch": 0.5699168556311414,
      "fcm_dpo/beta": 0.007640031632035971,
      "fcm_dpo/delta": 0.061430174857378006,
      "fcm_dpo/margin": 44.500518798828125,
      "fcm_dpo/q_t": 0.4220554828643799,
      "grad_norm": 12.877668380737305,
      "learning_rate": 2.3414402008585886e-07,
      "logits/chosen": 0.6904243230819702,
      "logits/rejected": 0.667314887046814,
      "logps/chosen": -160.35919189453125,
      "logps/ref_chosen": -46.17853546142578,
      "logps/ref_rejected": -57.756500244140625,
      "logps/rejected": -216.43768310546875,
      "loss": 1.1664,
      "margin_dpo/margin_mean": 44.500518798828125,
      "margin_dpo/margin_std": 80.63041687011719,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -111.80108642578125,
      "KL/mean": -132.49267578125,
      "KL/rejected_KL_mean": -153.18423461914062,
      "KL/std": 75.15191650390625,
      "epoch": 0.5714285714285714,
      "fcm_dpo/beta": 0.007755584083497524,
      "fcm_dpo/delta": 0.08093470335006714,
      "fcm_dpo/margin": 41.38316345214844,
      "fcm_dpo/q_t": 0.4269237220287323,
      "grad_norm": 12.859025001525879,
      "learning_rate": 2.3282469092517977e-07,
      "logits/chosen": 0.7610163688659668,
      "logits/rejected": 0.7104548215866089,
      "logps/chosen": -171.01995849609375,
      "logps/ref_chosen": -59.21887969970703,
      "logps/ref_rejected": -71.24818420410156,
      "logps/rejected": -224.43243408203125,
      "loss": 1.1796,
      "margin_dpo/margin_mean": 41.38316345214844,
      "margin_dpo/margin_std": 77.51055908203125,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -107.71281433105469,
      "KL/mean": -136.4612579345703,
      "KL/rejected_KL_mean": -165.20968627929688,
      "KL/std": 75.73796081542969,
      "epoch": 0.5729402872260015,
      "fcm_dpo/beta": 0.007706031668931246,
      "fcm_dpo/delta": -0.04505161941051483,
      "fcm_dpo/margin": 57.49687957763672,
      "fcm_dpo/q_t": 0.40112510323524475,
      "grad_norm": 14.837937355041504,
      "learning_rate": 2.3150584219481643e-07,
      "logits/chosen": 0.6838923692703247,
      "logits/rejected": 0.6072291731834412,
      "logps/chosen": -184.02940368652344,
      "logps/ref_chosen": -76.31658935546875,
      "logps/ref_rejected": -104.26200103759766,
      "logps/rejected": -269.4716796875,
      "loss": 1.0898,
      "margin_dpo/margin_mean": 57.49687957763672,
      "margin_dpo/margin_std": 85.10267639160156,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -94.17698669433594,
      "KL/mean": -129.25741577148438,
      "KL/rejected_KL_mean": -164.3378448486328,
      "KL/std": 71.46331787109375,
      "epoch": 0.5744520030234316,
      "fcm_dpo/beta": 0.007537417113780975,
      "fcm_dpo/delta": -0.135920912027359,
      "fcm_dpo/margin": 70.16085815429688,
      "fcm_dpo/q_t": 0.37863287329673767,
      "grad_norm": 12.04366683959961,
      "learning_rate": 2.3018751078580283e-07,
      "logits/chosen": 0.7140184044837952,
      "logits/rejected": 0.6739776730537415,
      "logps/chosen": -155.46014404296875,
      "logps/ref_chosen": -61.283164978027344,
      "logps/ref_rejected": -72.38892364501953,
      "logps/rejected": -236.72677612304688,
      "loss": 1.025,
      "margin_dpo/margin_mean": 70.16085815429688,
      "margin_dpo/margin_std": 86.23661041259766,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -115.3492202758789,
      "KL/mean": -128.97865295410156,
      "KL/rejected_KL_mean": -142.60809326171875,
      "KL/std": 72.53305053710938,
      "epoch": 0.5759637188208617,
      "fcm_dpo/beta": 0.0075180139392614365,
      "fcm_dpo/delta": 0.05303092673420906,
      "fcm_dpo/margin": 27.258872985839844,
      "fcm_dpo/q_t": 0.4539121389389038,
      "grad_norm": 13.296960830688477,
      "learning_rate": 2.288697335747027e-07,
      "logits/chosen": 0.6931901574134827,
      "logits/rejected": 0.6708425879478455,
      "logps/chosen": -173.56320190429688,
      "logps/ref_chosen": -58.2139892578125,
      "logps/ref_rejected": -60.78669357299805,
      "logps/rejected": -203.394775390625,
      "loss": 1.2916,
      "margin_dpo/margin_mean": 27.258869171142578,
      "margin_dpo/margin_std": 85.79790496826172,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -116.8239517211914,
      "KL/mean": -140.7882080078125,
      "KL/rejected_KL_mean": -164.75244140625,
      "KL/std": 73.31473541259766,
      "epoch": 0.5774754346182918,
      "fcm_dpo/beta": 0.007631244137883186,
      "fcm_dpo/delta": 0.0349888876080513,
      "fcm_dpo/margin": 47.92848587036133,
      "fcm_dpo/q_t": 0.41606825590133667,
      "grad_norm": 13.151206016540527,
      "learning_rate": 2.2755254742257706e-07,
      "logits/chosen": 0.7141730785369873,
      "logits/rejected": 0.6572399139404297,
      "logps/chosen": -178.64927673339844,
      "logps/ref_chosen": -61.82532501220703,
      "logps/ref_rejected": -83.0452880859375,
      "logps/rejected": -247.7977294921875,
      "loss": 1.1268,
      "margin_dpo/margin_mean": 47.92848587036133,
      "margin_dpo/margin_std": 72.02082061767578,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -114.61416625976562,
      "KL/mean": -138.47982788085938,
      "KL/rejected_KL_mean": -162.3455047607422,
      "KL/std": 73.81539916992188,
      "epoch": 0.5789871504157218,
      "fcm_dpo/beta": 0.007621276192367077,
      "fcm_dpo/delta": 0.03738650679588318,
      "fcm_dpo/margin": 47.7313232421875,
      "fcm_dpo/q_t": 0.419203519821167,
      "grad_norm": 14.164161682128906,
      "learning_rate": 2.2623598917395436e-07,
      "logits/chosen": 0.5664623975753784,
      "logits/rejected": 0.5980826616287231,
      "logps/chosen": -195.17742919921875,
      "logps/ref_chosen": -80.56326293945312,
      "logps/ref_rejected": -74.62922668457031,
      "logps/rejected": -236.9747314453125,
      "loss": 1.1728,
      "margin_dpo/margin_mean": 47.7313232421875,
      "margin_dpo/margin_std": 92.29366302490234,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -113.17518615722656,
      "KL/mean": -137.73904418945312,
      "KL/rejected_KL_mean": -162.30288696289062,
      "KL/std": 73.91085052490234,
      "epoch": 0.5804988662131519,
      "fcm_dpo/beta": 0.007690755650401115,
      "fcm_dpo/delta": 0.02304769679903984,
      "fcm_dpo/margin": 49.127716064453125,
      "fcm_dpo/q_t": 0.4123014807701111,
      "grad_norm": 14.989981651306152,
      "learning_rate": 2.2492009565579875e-07,
      "logits/chosen": 0.7450392246246338,
      "logits/rejected": 0.697953999042511,
      "logps/chosen": -178.65032958984375,
      "logps/ref_chosen": -65.47514343261719,
      "logps/ref_rejected": -79.67378234863281,
      "logps/rejected": -241.9766845703125,
      "loss": 1.1301,
      "margin_dpo/margin_mean": 49.127716064453125,
      "margin_dpo/margin_std": 79.05022430419922,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -109.96649169921875,
      "KL/mean": -142.77560424804688,
      "KL/rejected_KL_mean": -175.58473205566406,
      "KL/std": 73.74166870117188,
      "epoch": 0.582010582010582,
      "fcm_dpo/beta": 0.00761133898049593,
      "fcm_dpo/delta": -0.10455459356307983,
      "fcm_dpo/margin": 65.61822509765625,
      "fcm_dpo/q_t": 0.38668984174728394,
      "grad_norm": 13.53128719329834,
      "learning_rate": 2.2360490367648084e-07,
      "logits/chosen": 0.6406357884407043,
      "logits/rejected": 0.5983352661132812,
      "logps/chosen": -176.02301025390625,
      "logps/ref_chosen": -66.0565185546875,
      "logps/ref_rejected": -86.68023681640625,
      "logps/rejected": -262.26495361328125,
      "loss": 1.0326,
      "margin_dpo/margin_mean": 65.61822509765625,
      "margin_dpo/margin_std": 78.6586685180664,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -128.26878356933594,
      "KL/mean": -149.02406311035156,
      "KL/rejected_KL_mean": -169.77932739257812,
      "KL/std": 75.39730834960938,
      "epoch": 0.5835222978080121,
      "fcm_dpo/beta": 0.007630414329469204,
      "fcm_dpo/delta": 0.08606353402137756,
      "fcm_dpo/margin": 41.51054763793945,
      "fcm_dpo/q_t": 0.42653924226760864,
      "grad_norm": 13.79835033416748,
      "learning_rate": 2.2229045002474724e-07,
      "logits/chosen": 0.5970888733863831,
      "logits/rejected": 0.5378561019897461,
      "logps/chosen": -203.89244079589844,
      "logps/ref_chosen": -75.6236572265625,
      "logps/ref_rejected": -92.62330627441406,
      "logps/rejected": -262.40264892578125,
      "loss": 1.1789,
      "margin_dpo/margin_mean": 41.51054763793945,
      "margin_dpo/margin_std": 78.0374526977539,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -113.36140441894531,
      "KL/mean": -144.81375122070312,
      "KL/rejected_KL_mean": -176.2660675048828,
      "KL/std": 72.10386657714844,
      "epoch": 0.5850340136054422,
      "fcm_dpo/beta": 0.007580885663628578,
      "fcm_dpo/delta": -0.08075231313705444,
      "fcm_dpo/margin": 62.904659271240234,
      "fcm_dpo/q_t": 0.3911029100418091,
      "grad_norm": 13.386743545532227,
      "learning_rate": 2.209767714686924e-07,
      "logits/chosen": 0.713404655456543,
      "logits/rejected": 0.6014559864997864,
      "logps/chosen": -160.58311462402344,
      "logps/ref_chosen": -47.22170639038086,
      "logps/ref_rejected": -87.338134765625,
      "logps/rejected": -263.60418701171875,
      "loss": 1.0382,
      "margin_dpo/margin_mean": 62.90465545654297,
      "margin_dpo/margin_std": 74.2324447631836,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -114.71498107910156,
      "KL/mean": -134.3380126953125,
      "KL/rejected_KL_mean": -153.96102905273438,
      "KL/std": 75.28630828857422,
      "epoch": 0.5865457294028723,
      "fcm_dpo/beta": 0.007554663810878992,
      "fcm_dpo/delta": 0.0016909594414755702,
      "fcm_dpo/margin": 39.246063232421875,
      "fcm_dpo/q_t": 0.4340188503265381,
      "grad_norm": 13.501871109008789,
      "learning_rate": 2.1966390475472954e-07,
      "logits/chosen": 0.706336498260498,
      "logits/rejected": 0.7000705003738403,
      "logps/chosen": -189.29446411132812,
      "logps/ref_chosen": -74.5794677734375,
      "logps/ref_rejected": -79.92558288574219,
      "logps/rejected": -233.88662719726562,
      "loss": 1.2208,
      "margin_dpo/margin_mean": 39.246063232421875,
      "margin_dpo/margin_std": 89.60990142822266,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -109.41908264160156,
      "KL/mean": -141.17974853515625,
      "KL/rejected_KL_mean": -172.9404296875,
      "KL/std": 74.57106018066406,
      "epoch": 0.5880574452003023,
      "fcm_dpo/beta": 0.0074761672876775265,
      "fcm_dpo/delta": -0.07856467366218567,
      "fcm_dpo/margin": 63.52134704589844,
      "fcm_dpo/q_t": 0.3921729326248169,
      "grad_norm": 26.400636672973633,
      "learning_rate": 2.1835188660656265e-07,
      "logits/chosen": 0.7010380029678345,
      "logits/rejected": 0.6631730794906616,
      "logps/chosen": -171.04345703125,
      "logps/ref_chosen": -61.624366760253906,
      "logps/ref_rejected": -76.50978088378906,
      "logps/rejected": -249.45021057128906,
      "loss": 1.0539,
      "margin_dpo/margin_mean": 63.52134704589844,
      "margin_dpo/margin_std": 82.0999755859375,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -100.01317596435547,
      "KL/mean": -124.32142639160156,
      "KL/rejected_KL_mean": -148.62966918945312,
      "KL/std": 71.7291488647461,
      "epoch": 0.5895691609977324,
      "fcm_dpo/beta": 0.007465273607522249,
      "fcm_dpo/delta": 0.0384586863219738,
      "fcm_dpo/margin": 48.61649703979492,
      "fcm_dpo/q_t": 0.4167312681674957,
      "grad_norm": 11.193785667419434,
      "learning_rate": 2.170407537241599e-07,
      "logits/chosen": 0.7603079080581665,
      "logits/rejected": 0.6859769225120544,
      "logps/chosen": -145.88504028320312,
      "logps/ref_chosen": -45.871864318847656,
      "logps/ref_rejected": -61.305999755859375,
      "logps/rejected": -209.9356689453125,
      "loss": 1.1315,
      "margin_dpo/margin_mean": 48.616493225097656,
      "margin_dpo/margin_std": 75.53978729248047,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -110.56118774414062,
      "KL/mean": -139.5545654296875,
      "KL/rejected_KL_mean": -168.54794311523438,
      "KL/std": 72.66812133789062,
      "epoch": 0.5910808767951625,
      "fcm_dpo/beta": 0.007431542966514826,
      "fcm_dpo/delta": -0.03269674628973007,
      "fcm_dpo/margin": 57.98676681518555,
      "fcm_dpo/q_t": 0.40178489685058594,
      "grad_norm": 12.458271026611328,
      "learning_rate": 2.1573054278272636e-07,
      "logits/chosen": 0.7125017046928406,
      "logits/rejected": 0.6410657167434692,
      "logps/chosen": -168.74819946289062,
      "logps/ref_chosen": -58.18701171875,
      "logps/ref_rejected": -83.63442993164062,
      "logps/rejected": -252.18238830566406,
      "loss": 1.1064,
      "margin_dpo/margin_mean": 57.986759185791016,
      "margin_dpo/margin_std": 89.69422912597656,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -97.53421020507812,
      "KL/mean": -128.83755493164062,
      "KL/rejected_KL_mean": -160.14089965820312,
      "KL/std": 76.88148498535156,
      "epoch": 0.5925925925925926,
      "fcm_dpo/beta": 0.0074156527407467365,
      "fcm_dpo/delta": -0.06773370504379272,
      "fcm_dpo/margin": 62.606693267822266,
      "fcm_dpo/q_t": 0.39553213119506836,
      "grad_norm": 11.342584609985352,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": 0.7609713673591614,
      "logits/rejected": 0.698552131652832,
      "logps/chosen": -167.27874755859375,
      "logps/ref_chosen": -69.7445297241211,
      "logps/ref_rejected": -94.05877685546875,
      "logps/rejected": -254.19967651367188,
      "loss": 1.082,
      "margin_dpo/margin_mean": 62.606693267822266,
      "margin_dpo/margin_std": 90.55340576171875,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -111.23446655273438,
      "KL/mean": -144.1527099609375,
      "KL/rejected_KL_mean": -177.07095336914062,
      "KL/std": 75.27520751953125,
      "epoch": 0.5941043083900227,
      "fcm_dpo/beta": 0.007229278329759836,
      "fcm_dpo/delta": -0.08038505166769028,
      "fcm_dpo/margin": 65.83646392822266,
      "fcm_dpo/q_t": 0.39052367210388184,
      "grad_norm": 11.659123420715332,
      "learning_rate": 2.131130332936195e-07,
      "logits/chosen": 0.7413580417633057,
      "logits/rejected": 0.6996890902519226,
      "logps/chosen": -163.56936645507812,
      "logps/ref_chosen": -52.33489990234375,
      "logps/ref_rejected": -74.33809661865234,
      "logps/rejected": -251.40904235839844,
      "loss": 1.0394,
      "margin_dpo/margin_mean": 65.83646392822266,
      "margin_dpo/margin_std": 77.09921264648438,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -106.2999267578125,
      "KL/mean": -134.94711303710938,
      "KL/rejected_KL_mean": -163.59429931640625,
      "KL/std": 69.1613540649414,
      "epoch": 0.5956160241874527,
      "fcm_dpo/beta": 0.00723269023001194,
      "fcm_dpo/delta": -0.015140345320105553,
      "fcm_dpo/margin": 57.294368743896484,
      "fcm_dpo/q_t": 0.4030148386955261,
      "grad_norm": 11.63623046875,
      "learning_rate": 2.1180580796331323e-07,
      "logits/chosen": 0.7115650177001953,
      "logits/rejected": 0.680920422077179,
      "logps/chosen": -166.97605895996094,
      "logps/ref_chosen": -60.6761360168457,
      "logps/ref_rejected": -71.36074829101562,
      "logps/rejected": -234.95504760742188,
      "loss": 1.0765,
      "margin_dpo/margin_mean": 57.29436492919922,
      "margin_dpo/margin_std": 69.48764038085938,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -111.16607666015625,
      "KL/mean": -135.60264587402344,
      "KL/rejected_KL_mean": -160.03921508789062,
      "KL/std": 71.98497009277344,
      "epoch": 0.5971277399848829,
      "fcm_dpo/beta": 0.007283855229616165,
      "fcm_dpo/delta": 0.045138321816921234,
      "fcm_dpo/margin": 48.873130798339844,
      "fcm_dpo/q_t": 0.4198164939880371,
      "grad_norm": 14.008892059326172,
      "learning_rate": 2.104996510066625e-07,
      "logits/chosen": 0.7485306262969971,
      "logits/rejected": 0.6434615850448608,
      "logps/chosen": -161.77040100097656,
      "logps/ref_chosen": -50.60432434082031,
      "logps/ref_rejected": -77.08731079101562,
      "logps/rejected": -237.12652587890625,
      "loss": 1.1366,
      "margin_dpo/margin_mean": 48.87313461303711,
      "margin_dpo/margin_std": 76.98291778564453,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -104.07820129394531,
      "KL/mean": -131.68917846679688,
      "KL/rejected_KL_mean": -159.30015563964844,
      "KL/std": 80.30957794189453,
      "epoch": 0.5986394557823129,
      "fcm_dpo/beta": 0.007214938756078482,
      "fcm_dpo/delta": 0.00036709755659103394,
      "fcm_dpo/margin": 55.221954345703125,
      "fcm_dpo/q_t": 0.4084014892578125,
      "grad_norm": 11.316884994506836,
      "learning_rate": 2.0919459895968517e-07,
      "logits/chosen": 0.7228500247001648,
      "logits/rejected": 0.6213551163673401,
      "logps/chosen": -155.43780517578125,
      "logps/ref_chosen": -51.35961151123047,
      "logps/ref_rejected": -79.89360046386719,
      "logps/rejected": -239.19375610351562,
      "loss": 1.0943,
      "margin_dpo/margin_mean": 55.221946716308594,
      "margin_dpo/margin_std": 70.75540161132812,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -120.00655364990234,
      "KL/mean": -135.69989013671875,
      "KL/rejected_KL_mean": -151.39320373535156,
      "KL/std": 73.55288696289062,
      "epoch": 0.600151171579743,
      "fcm_dpo/beta": 0.007465363945811987,
      "fcm_dpo/delta": 0.16950058937072754,
      "fcm_dpo/margin": 31.38665008544922,
      "fcm_dpo/q_t": 0.4467281103134155,
      "grad_norm": 12.68991756439209,
      "learning_rate": 2.078906883274924e-07,
      "logits/chosen": 0.6106295585632324,
      "logits/rejected": 0.5607829689979553,
      "logps/chosen": -186.46278381347656,
      "logps/ref_chosen": -66.45622253417969,
      "logps/ref_rejected": -85.74736785888672,
      "logps/rejected": -237.14056396484375,
      "loss": 1.2719,
      "margin_dpo/margin_mean": 31.38665008544922,
      "margin_dpo/margin_std": 89.06100463867188,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -103.11650085449219,
      "KL/mean": -137.64886474609375,
      "KL/rejected_KL_mean": -172.18124389648438,
      "KL/std": 76.49958801269531,
      "epoch": 0.6016628873771731,
      "fcm_dpo/beta": 0.007364482153207064,
      "fcm_dpo/delta": -0.11527767032384872,
      "fcm_dpo/margin": 69.06472778320312,
      "fcm_dpo/q_t": 0.3848886489868164,
      "grad_norm": 10.936336517333984,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": 0.6943444013595581,
      "logits/rejected": 0.6249934434890747,
      "logps/chosen": -152.36074829101562,
      "logps/ref_chosen": -49.244239807128906,
      "logps/ref_rejected": -75.18949127197266,
      "logps/rejected": -247.3707275390625,
      "loss": 1.0191,
      "margin_dpo/margin_mean": 69.06472778320312,
      "margin_dpo/margin_std": 79.41087341308594,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -119.81324768066406,
      "KL/mean": -158.02162170410156,
      "KL/rejected_KL_mean": -196.22998046875,
      "KL/std": 81.09989929199219,
      "epoch": 0.6031746031746031,
      "fcm_dpo/beta": 0.007153850048780441,
      "fcm_dpo/delta": -0.15612734854221344,
      "fcm_dpo/margin": 76.416748046875,
      "fcm_dpo/q_t": 0.3771836757659912,
      "grad_norm": 12.9329252243042,
      "learning_rate": 2.052864371672457e-07,
      "logits/chosen": 0.6183818578720093,
      "logits/rejected": 0.4700758457183838,
      "logps/chosen": -188.1200408935547,
      "logps/ref_chosen": -68.30679321289062,
      "logps/ref_rejected": -113.2708511352539,
      "logps/rejected": -309.5008544921875,
      "loss": 1.0049,
      "margin_dpo/margin_mean": 76.416748046875,
      "margin_dpo/margin_std": 89.60525512695312,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -133.1045684814453,
      "KL/mean": -155.88046264648438,
      "KL/rejected_KL_mean": -178.6563720703125,
      "KL/std": 77.34986877441406,
      "epoch": 0.6046863189720333,
      "fcm_dpo/beta": 0.007073037791997194,
      "fcm_dpo/delta": -0.03691471368074417,
      "fcm_dpo/margin": 45.55181884765625,
      "fcm_dpo/q_t": 0.42521122097969055,
      "grad_norm": 16.572580337524414,
      "learning_rate": 2.0398616948569493e-07,
      "logits/chosen": 0.6649228930473328,
      "logits/rejected": 0.6034343242645264,
      "logps/chosen": -204.73106384277344,
      "logps/ref_chosen": -71.62649536132812,
      "logps/ref_rejected": -90.98765563964844,
      "logps/rejected": -269.64404296875,
      "loss": 1.1637,
      "margin_dpo/margin_mean": 45.551815032958984,
      "margin_dpo/margin_std": 76.09416198730469,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -102.09986877441406,
      "KL/mean": -134.02391052246094,
      "KL/rejected_KL_mean": -165.94793701171875,
      "KL/std": 82.41902160644531,
      "epoch": 0.6061980347694633,
      "fcm_dpo/beta": 0.00699904840439558,
      "fcm_dpo/delta": -0.04942867532372475,
      "fcm_dpo/margin": 63.84806442260742,
      "fcm_dpo/q_t": 0.39786165952682495,
      "grad_norm": 10.089393615722656,
      "learning_rate": 2.0268718890989752e-07,
      "logits/chosen": 0.7165747284889221,
      "logits/rejected": 0.6181085109710693,
      "logps/chosen": -155.8248291015625,
      "logps/ref_chosen": -53.72495651245117,
      "logps/ref_rejected": -75.06304931640625,
      "logps/rejected": -241.010986328125,
      "loss": 1.0585,
      "margin_dpo/margin_mean": 63.84806442260742,
      "margin_dpo/margin_std": 78.08700561523438,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -111.72433471679688,
      "KL/mean": -137.66998291015625,
      "KL/rejected_KL_mean": -163.6156463623047,
      "KL/std": 72.5931396484375,
      "epoch": 0.6077097505668935,
      "fcm_dpo/beta": 0.006999198347330093,
      "fcm_dpo/delta": 0.03762829676270485,
      "fcm_dpo/margin": 51.89130783081055,
      "fcm_dpo/q_t": 0.417421817779541,
      "grad_norm": 13.307319641113281,
      "learning_rate": 2.013895317751323e-07,
      "logits/chosen": 0.6779258847236633,
      "logits/rejected": 0.651907205581665,
      "logps/chosen": -173.5982666015625,
      "logps/ref_chosen": -61.873931884765625,
      "logps/ref_rejected": -66.15198516845703,
      "logps/rejected": -229.76763916015625,
      "loss": 1.1489,
      "margin_dpo/margin_mean": 51.89130401611328,
      "margin_dpo/margin_std": 87.42867279052734,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -119.16712951660156,
      "KL/mean": -151.31494140625,
      "KL/rejected_KL_mean": -183.46275329589844,
      "KL/std": 81.462646484375,
      "epoch": 0.6092214663643235,
      "fcm_dpo/beta": 0.006999680772423744,
      "fcm_dpo/delta": -0.05242285132408142,
      "fcm_dpo/margin": 64.29563903808594,
      "fcm_dpo/q_t": 0.3983529806137085,
      "grad_norm": 11.000279426574707,
      "learning_rate": 2.0009323437965898e-07,
      "logits/chosen": 0.7842544317245483,
      "logits/rejected": 0.6958855390548706,
      "logps/chosen": -170.48861694335938,
      "logps/ref_chosen": -51.321502685546875,
      "logps/ref_rejected": -86.54010772705078,
      "logps/rejected": -270.00286865234375,
      "loss": 1.0756,
      "margin_dpo/margin_mean": 64.29563903808594,
      "margin_dpo/margin_std": 87.97834777832031,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -111.2451171875,
      "KL/mean": -144.17141723632812,
      "KL/rejected_KL_mean": -177.09771728515625,
      "KL/std": 82.94351196289062,
      "epoch": 0.6107331821617535,
      "fcm_dpo/beta": 0.006878808606415987,
      "fcm_dpo/delta": -0.056754522025585175,
      "fcm_dpo/margin": 65.85259246826172,
      "fcm_dpo/q_t": 0.39783400297164917,
      "grad_norm": 13.444967269897461,
      "learning_rate": 1.9879833298370237e-07,
      "logits/chosen": 0.6783360242843628,
      "logits/rejected": 0.577847421169281,
      "logps/chosen": -173.50799560546875,
      "logps/ref_chosen": -62.26288604736328,
      "logps/ref_rejected": -95.19029998779297,
      "logps/rejected": -272.28802490234375,
      "loss": 1.076,
      "margin_dpo/margin_mean": 65.85258483886719,
      "margin_dpo/margin_std": 89.37802124023438,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -115.50639343261719,
      "KL/mean": -141.56947326660156,
      "KL/rejected_KL_mean": -167.63255310058594,
      "KL/std": 74.9796142578125,
      "epoch": 0.6122448979591837,
      "fcm_dpo/beta": 0.006899132858961821,
      "fcm_dpo/delta": 0.04119940102100372,
      "fcm_dpo/margin": 52.126182556152344,
      "fcm_dpo/q_t": 0.4182543158531189,
      "grad_norm": 11.586745262145996,
      "learning_rate": 1.975048638084379e-07,
      "logits/chosen": 0.7487200498580933,
      "logits/rejected": 0.6993913054466248,
      "logps/chosen": -166.09072875976562,
      "logps/ref_chosen": -50.5843391418457,
      "logps/ref_rejected": -65.43156433105469,
      "logps/rejected": -233.06411743164062,
      "loss": 1.1334,
      "margin_dpo/margin_mean": 52.126182556152344,
      "margin_dpo/margin_std": 78.4576416015625,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -111.00540161132812,
      "KL/mean": -145.43777465820312,
      "KL/rejected_KL_mean": -179.87014770507812,
      "KL/std": 80.82884216308594,
      "epoch": 0.6137566137566137,
      "fcm_dpo/beta": 0.006891036406159401,
      "fcm_dpo/delta": -0.07819212973117828,
      "fcm_dpo/margin": 68.86474609375,
      "fcm_dpo/q_t": 0.3913407325744629,
      "grad_norm": 13.305275917053223,
      "learning_rate": 1.9621286303497914e-07,
      "logits/chosen": 0.7364928722381592,
      "logits/rejected": 0.5690401196479797,
      "logps/chosen": -160.00100708007812,
      "logps/ref_chosen": -48.99560546875,
      "logps/ref_rejected": -92.47774505615234,
      "logps/rejected": -272.347900390625,
      "loss": 1.0614,
      "margin_dpo/margin_mean": 68.86474609375,
      "margin_dpo/margin_std": 91.55941772460938,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -137.87237548828125,
      "KL/mean": -164.59429931640625,
      "KL/rejected_KL_mean": -191.3162384033203,
      "KL/std": 88.6528091430664,
      "epoch": 0.6152683295540439,
      "fcm_dpo/beta": 0.006907115690410137,
      "fcm_dpo/delta": 0.031741708517074585,
      "fcm_dpo/margin": 53.44386291503906,
      "fcm_dpo/q_t": 0.4162091016769409,
      "grad_norm": 14.182259559631348,
      "learning_rate": 1.9492236680336483e-07,
      "logits/chosen": 0.6167929172515869,
      "logits/rejected": 0.5412212610244751,
      "logps/chosen": -227.27294921875,
      "logps/ref_chosen": -89.40056610107422,
      "logps/ref_rejected": -99.28775024414062,
      "logps/rejected": -290.60400390625,
      "loss": 1.1461,
      "margin_dpo/margin_mean": 53.44386291503906,
      "margin_dpo/margin_std": 91.42645263671875,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -104.82077026367188,
      "KL/mean": -144.03866577148438,
      "KL/rejected_KL_mean": -183.25656127929688,
      "KL/std": 77.72358703613281,
      "epoch": 0.6167800453514739,
      "fcm_dpo/beta": 0.0067956093698740005,
      "fcm_dpo/delta": -0.14040729403495789,
      "fcm_dpo/margin": 78.43579864501953,
      "fcm_dpo/q_t": 0.3770345449447632,
      "grad_norm": 10.191902160644531,
      "learning_rate": 1.9363341121154895e-07,
      "logits/chosen": 0.6959325075149536,
      "logits/rejected": 0.6139761805534363,
      "logps/chosen": -159.52468872070312,
      "logps/ref_chosen": -54.70391845703125,
      "logps/ref_rejected": -73.98648834228516,
      "logps/rejected": -257.2430419921875,
      "loss": 1.001,
      "margin_dpo/margin_mean": 78.43579864501953,
      "margin_dpo/margin_std": 83.04154205322266,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -129.40618896484375,
      "KL/mean": -148.52774047851562,
      "KL/rejected_KL_mean": -167.64930725097656,
      "KL/std": 68.60551452636719,
      "epoch": 0.618291761148904,
      "fcm_dpo/beta": 0.006842237897217274,
      "fcm_dpo/delta": 0.14218175411224365,
      "fcm_dpo/margin": 38.24311828613281,
      "fcm_dpo/q_t": 0.4402683973312378,
      "grad_norm": 12.969134330749512,
      "learning_rate": 1.9234603231438994e-07,
      "logits/chosen": 0.6811122894287109,
      "logits/rejected": 0.6903325319290161,
      "logps/chosen": -191.5244140625,
      "logps/ref_chosen": -62.11822509765625,
      "logps/ref_rejected": -61.933509826660156,
      "logps/rejected": -229.58282470703125,
      "loss": 1.2126,
      "margin_dpo/margin_mean": 38.24311828613281,
      "margin_dpo/margin_std": 79.25856018066406,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -121.08537292480469,
      "KL/mean": -152.85206604003906,
      "KL/rejected_KL_mean": -184.6187744140625,
      "KL/std": 74.780029296875,
      "epoch": 0.6198034769463341,
      "fcm_dpo/beta": 0.006816249340772629,
      "fcm_dpo/delta": -0.03573864325881004,
      "fcm_dpo/margin": 63.53340148925781,
      "fcm_dpo/q_t": 0.39918336272239685,
      "grad_norm": 11.410712242126465,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": 0.7246212363243103,
      "logits/rejected": 0.6982280015945435,
      "logps/chosen": -182.88803100585938,
      "logps/ref_chosen": -61.80266189575195,
      "logps/ref_rejected": -76.60002136230469,
      "logps/rejected": -261.2187805175781,
      "loss": 1.0595,
      "margin_dpo/margin_mean": 63.53340148925781,
      "margin_dpo/margin_std": 71.64531707763672,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -126.13032531738281,
      "KL/mean": -156.75860595703125,
      "KL/rejected_KL_mean": -187.38687133789062,
      "KL/std": 82.61457824707031,
      "epoch": 0.6213151927437641,
      "fcm_dpo/beta": 0.006837380118668079,
      "fcm_dpo/delta": -0.01966019906103611,
      "fcm_dpo/margin": 61.25654602050781,
      "fcm_dpo/q_t": 0.404565691947937,
      "grad_norm": 10.397010803222656,
      "learning_rate": 1.8977614860195296e-07,
      "logits/chosen": 0.701043963432312,
      "logits/rejected": 0.6398018598556519,
      "logps/chosen": -180.57571411132812,
      "logps/ref_chosen": -54.44539260864258,
      "logps/ref_rejected": -74.5650863647461,
      "logps/rejected": -261.95196533203125,
      "loss": 1.0963,
      "margin_dpo/margin_mean": 61.25654983520508,
      "margin_dpo/margin_std": 88.56686401367188,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -131.27859497070312,
      "KL/mean": -159.94105529785156,
      "KL/rejected_KL_mean": -188.603515625,
      "KL/std": 72.79525756835938,
      "epoch": 0.6228269085411943,
      "fcm_dpo/beta": 0.006809461396187544,
      "fcm_dpo/delta": 0.009827276691794395,
      "fcm_dpo/margin": 57.324928283691406,
      "fcm_dpo/q_t": 0.4100860357284546,
      "grad_norm": 12.925461769104004,
      "learning_rate": 1.8849371567184662e-07,
      "logits/chosen": 0.708077073097229,
      "logits/rejected": 0.6398712396621704,
      "logps/chosen": -186.52667236328125,
      "logps/ref_chosen": -55.248085021972656,
      "logps/ref_rejected": -68.96623229980469,
      "logps/rejected": -257.5697326660156,
      "loss": 1.1016,
      "margin_dpo/margin_mean": 57.324928283691406,
      "margin_dpo/margin_std": 78.18580627441406,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -143.82650756835938,
      "KL/mean": -169.30419921875,
      "KL/rejected_KL_mean": -194.7818603515625,
      "KL/std": 79.58856201171875,
      "epoch": 0.6243386243386243,
      "fcm_dpo/beta": 0.00689761433750391,
      "fcm_dpo/delta": 0.05018645152449608,
      "fcm_dpo/margin": 50.955360412597656,
      "fcm_dpo/q_t": 0.4213051497936249,
      "grad_norm": 14.219574928283691,
      "learning_rate": 1.872130032047302e-07,
      "logits/chosen": 0.5499156713485718,
      "logits/rejected": 0.5132287740707397,
      "logps/chosen": -212.54725646972656,
      "logps/ref_chosen": -68.72074890136719,
      "logps/ref_rejected": -78.76539611816406,
      "logps/rejected": -273.5472717285156,
      "loss": 1.178,
      "margin_dpo/margin_mean": 50.955360412597656,
      "margin_dpo/margin_std": 99.61614990234375,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -126.31494140625,
      "KL/mean": -157.68609619140625,
      "KL/rejected_KL_mean": -189.0572509765625,
      "KL/std": 85.60701751708984,
      "epoch": 0.6258503401360545,
      "fcm_dpo/beta": 0.006879427004605532,
      "fcm_dpo/delta": -0.033046744763851166,
      "fcm_dpo/margin": 62.74230194091797,
      "fcm_dpo/q_t": 0.40025120973587036,
      "grad_norm": 11.876262664794922,
      "learning_rate": 1.8593404702488436e-07,
      "logits/chosen": 0.6935607194900513,
      "logits/rejected": 0.6302182674407959,
      "logps/chosen": -180.45315551757812,
      "logps/ref_chosen": -54.138214111328125,
      "logps/ref_rejected": -74.65741729736328,
      "logps/rejected": -263.71466064453125,
      "loss": 1.0761,
      "margin_dpo/margin_mean": 62.74230194091797,
      "margin_dpo/margin_std": 81.80207824707031,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -126.15766906738281,
      "KL/mean": -152.6885528564453,
      "KL/rejected_KL_mean": -179.21945190429688,
      "KL/std": 81.44625091552734,
      "epoch": 0.6273620559334845,
      "fcm_dpo/beta": 0.0068847062066197395,
      "fcm_dpo/delta": 0.03599990904331207,
      "fcm_dpo/margin": 53.06175994873047,
      "fcm_dpo/q_t": 0.41671812534332275,
      "grad_norm": 12.530339241027832,
      "learning_rate": 1.846568829074628e-07,
      "logits/chosen": 0.7355213165283203,
      "logits/rejected": 0.7176867127418518,
      "logps/chosen": -182.07623291015625,
      "logps/ref_chosen": -55.91856002807617,
      "logps/ref_rejected": -61.747703552246094,
      "logps/rejected": -240.96714782714844,
      "loss": 1.1409,
      "margin_dpo/margin_mean": 53.06175994873047,
      "margin_dpo/margin_std": 88.03173828125,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -133.60458374023438,
      "KL/mean": -156.7396240234375,
      "KL/rejected_KL_mean": -179.8746337890625,
      "KL/std": 84.14654541015625,
      "epoch": 0.6288737717309146,
      "fcm_dpo/beta": 0.006849354133009911,
      "fcm_dpo/delta": -0.08797140419483185,
      "fcm_dpo/margin": 46.27003860473633,
      "fcm_dpo/q_t": 0.42819273471832275,
      "grad_norm": 13.571969985961914,
      "learning_rate": 1.8338154657749128e-07,
      "logits/chosen": 0.6719874143600464,
      "logits/rejected": 0.6208308935165405,
      "logps/chosen": -188.32766723632812,
      "logps/ref_chosen": -54.72308349609375,
      "logps/ref_rejected": -69.17388916015625,
      "logps/rejected": -249.04852294921875,
      "loss": 1.1849,
      "margin_dpo/margin_mean": 46.27003479003906,
      "margin_dpo/margin_std": 82.4261474609375,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -137.70266723632812,
      "KL/mean": -168.36795043945312,
      "KL/rejected_KL_mean": -199.033203125,
      "KL/std": 79.7743148803711,
      "epoch": 0.6303854875283447,
      "fcm_dpo/beta": 0.006801956798881292,
      "fcm_dpo/delta": -0.0182628370821476,
      "fcm_dpo/margin": 61.330543518066406,
      "fcm_dpo/q_t": 0.40349721908569336,
      "grad_norm": 12.702634811401367,
      "learning_rate": 1.8210807370886849e-07,
      "logits/chosen": 0.7900456190109253,
      "logits/rejected": 0.7208957672119141,
      "logps/chosen": -194.49392700195312,
      "logps/ref_chosen": -56.791259765625,
      "logps/ref_rejected": -68.7791748046875,
      "logps/rejected": -267.8123779296875,
      "loss": 1.1149,
      "margin_dpo/margin_mean": 61.330543518066406,
      "margin_dpo/margin_std": 96.73316955566406,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -146.98593139648438,
      "KL/mean": -171.6457977294922,
      "KL/rejected_KL_mean": -196.3056640625,
      "KL/std": 86.17100524902344,
      "epoch": 0.6318972033257747,
      "fcm_dpo/beta": 0.0066888537257909775,
      "fcm_dpo/delta": -0.05611763894557953,
      "fcm_dpo/margin": 49.319732666015625,
      "fcm_dpo/q_t": 0.4252815842628479,
      "grad_norm": 13.449596405029297,
      "learning_rate": 1.8083649992336825e-07,
      "logits/chosen": 0.7279735207557678,
      "logits/rejected": 0.7330294251441956,
      "logps/chosen": -216.09390258789062,
      "logps/ref_chosen": -69.10798645019531,
      "logps/ref_rejected": -75.09132385253906,
      "logps/rejected": -271.39697265625,
      "loss": 1.1702,
      "margin_dpo/margin_mean": 49.31972885131836,
      "margin_dpo/margin_std": 87.53064727783203,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -118.92839050292969,
      "KL/mean": -153.7598876953125,
      "KL/rejected_KL_mean": -188.5913848876953,
      "KL/std": 82.14205932617188,
      "epoch": 0.6334089191232048,
      "fcm_dpo/beta": 0.006614279001951218,
      "fcm_dpo/delta": -0.06387455016374588,
      "fcm_dpo/margin": 69.66299438476562,
      "fcm_dpo/q_t": 0.3961649537086487,
      "grad_norm": 12.24950885772705,
      "learning_rate": 1.7956686078964255e-07,
      "logits/chosen": 0.6012529134750366,
      "logits/rejected": 0.5487751960754395,
      "logps/chosen": -177.10015869140625,
      "logps/ref_chosen": -58.1717643737793,
      "logps/ref_rejected": -71.67066955566406,
      "logps/rejected": -260.2620544433594,
      "loss": 1.065,
      "margin_dpo/margin_mean": 69.66299438476562,
      "margin_dpo/margin_std": 93.18635559082031,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -147.60433959960938,
      "KL/mean": -166.267578125,
      "KL/rejected_KL_mean": -184.9307861328125,
      "KL/std": 83.89628601074219,
      "epoch": 0.6349206349206349,
      "fcm_dpo/beta": 0.0066644903272390366,
      "fcm_dpo/delta": 0.045629166066646576,
      "fcm_dpo/margin": 37.32643127441406,
      "fcm_dpo/q_t": 0.44440752267837524,
      "grad_norm": 12.646784782409668,
      "learning_rate": 1.782991918222275e-07,
      "logits/chosen": 0.6889985203742981,
      "logits/rejected": 0.6424489617347717,
      "logps/chosen": -204.65786743164062,
      "logps/ref_chosen": -57.05351257324219,
      "logps/ref_rejected": -62.670982360839844,
      "logps/rejected": -247.60177612304688,
      "loss": 1.2515,
      "margin_dpo/margin_mean": 37.32643127441406,
      "margin_dpo/margin_std": 96.15448760986328,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -137.88760375976562,
      "KL/mean": -164.39105224609375,
      "KL/rejected_KL_mean": -190.89451599121094,
      "KL/std": 82.96305084228516,
      "epoch": 0.636432350718065,
      "fcm_dpo/beta": 0.006677803583443165,
      "fcm_dpo/delta": 0.047430604696273804,
      "fcm_dpo/margin": 53.006935119628906,
      "fcm_dpo/q_t": 0.42090481519699097,
      "grad_norm": 13.67684268951416,
      "learning_rate": 1.7703352848054887e-07,
      "logits/chosen": 0.6522685289382935,
      "logits/rejected": 0.5913703441619873,
      "logps/chosen": -195.21084594726562,
      "logps/ref_chosen": -57.32324981689453,
      "logps/ref_rejected": -75.33782958984375,
      "logps/rejected": -266.23236083984375,
      "loss": 1.1871,
      "margin_dpo/margin_mean": 53.006935119628906,
      "margin_dpo/margin_std": 107.34759521484375,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -119.95924377441406,
      "KL/mean": -155.01040649414062,
      "KL/rejected_KL_mean": -190.0615692138672,
      "KL/std": 83.26985168457031,
      "epoch": 0.6379440665154951,
      "fcm_dpo/beta": 0.006677722558379173,
      "fcm_dpo/delta": -0.07144533842802048,
      "fcm_dpo/margin": 70.10234069824219,
      "fcm_dpo/q_t": 0.3929086923599243,
      "grad_norm": 13.983145713806152,
      "learning_rate": 1.7576990616793137e-07,
      "logits/chosen": 0.7097414135932922,
      "logits/rejected": 0.6987332701683044,
      "logps/chosen": -187.01681518554688,
      "logps/ref_chosen": -67.05757141113281,
      "logps/ref_rejected": -72.12803649902344,
      "logps/rejected": -262.1896057128906,
      "loss": 1.0488,
      "margin_dpo/margin_mean": 70.10234069824219,
      "margin_dpo/margin_std": 85.27452850341797,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -123.76502990722656,
      "KL/mean": -159.84022521972656,
      "KL/rejected_KL_mean": -195.9154052734375,
      "KL/std": 84.98675537109375,
      "epoch": 0.6394557823129252,
      "fcm_dpo/beta": 0.006547610275447369,
      "fcm_dpo/delta": -0.07617159932851791,
      "fcm_dpo/margin": 72.15037536621094,
      "fcm_dpo/q_t": 0.3926679193973541,
      "grad_norm": 11.574021339416504,
      "learning_rate": 1.745083602306071e-07,
      "logits/chosen": 0.7345231175422668,
      "logits/rejected": 0.662026047706604,
      "logps/chosen": -177.8267059326172,
      "logps/ref_chosen": -54.06167221069336,
      "logps/ref_rejected": -76.64092254638672,
      "logps/rejected": -272.55633544921875,
      "loss": 1.0493,
      "margin_dpo/margin_mean": 72.1503677368164,
      "margin_dpo/margin_std": 90.34888458251953,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -133.77822875976562,
      "KL/mean": -168.0508270263672,
      "KL/rejected_KL_mean": -202.32342529296875,
      "KL/std": 80.9405288696289,
      "epoch": 0.6409674981103552,
      "fcm_dpo/beta": 0.00645102746784687,
      "fcm_dpo/delta": -0.045091331005096436,
      "fcm_dpo/margin": 68.5451889038086,
      "fcm_dpo/q_t": 0.39931702613830566,
      "grad_norm": 16.19998550415039,
      "learning_rate": 1.7324892595672804e-07,
      "logits/chosen": 0.6296772956848145,
      "logits/rejected": 0.585532546043396,
      "logps/chosen": -187.38710021972656,
      "logps/ref_chosen": -53.60887145996094,
      "logps/ref_rejected": -79.2139892578125,
      "logps/rejected": -281.53741455078125,
      "loss": 1.0765,
      "margin_dpo/margin_mean": 68.54518127441406,
      "margin_dpo/margin_std": 91.9103012084961,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -131.75701904296875,
      "KL/mean": -159.09242248535156,
      "KL/rejected_KL_mean": -186.42779541015625,
      "KL/std": 78.51920318603516,
      "epoch": 0.6424792139077853,
      "fcm_dpo/beta": 0.0065160347148776054,
      "fcm_dpo/delta": 0.04538961499929428,
      "fcm_dpo/margin": 54.67079162597656,
      "fcm_dpo/q_t": 0.4188900589942932,
      "grad_norm": 12.962249755859375,
      "learning_rate": 1.7199163857537824e-07,
      "logits/chosen": 0.7621163129806519,
      "logits/rejected": 0.7318211793899536,
      "logps/chosen": -190.17169189453125,
      "logps/ref_chosen": -58.41468048095703,
      "logps/ref_rejected": -66.59054565429688,
      "logps/rejected": -253.01834106445312,
      "loss": 1.1419,
      "margin_dpo/margin_mean": 54.67079162597656,
      "margin_dpo/margin_std": 89.48291015625,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -157.2525634765625,
      "KL/mean": -174.17962646484375,
      "KL/rejected_KL_mean": -191.106689453125,
      "KL/std": 82.11293029785156,
      "epoch": 0.6439909297052154,
      "fcm_dpo/beta": 0.00670973677188158,
      "fcm_dpo/delta": 0.17681291699409485,
      "fcm_dpo/margin": 33.8541259765625,
      "fcm_dpo/q_t": 0.4480590224266052,
      "grad_norm": 16.367176055908203,
      "learning_rate": 1.7073653325558828e-07,
      "logits/chosen": 0.6534860134124756,
      "logits/rejected": 0.6601561307907104,
      "logps/chosen": -228.96078491210938,
      "logps/ref_chosen": -71.70822143554688,
      "logps/ref_rejected": -73.57725524902344,
      "logps/rejected": -264.6839599609375,
      "loss": 1.2822,
      "margin_dpo/margin_mean": 33.8541259765625,
      "margin_dpo/margin_std": 102.28767395019531,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -147.63272094726562,
      "KL/mean": -175.30328369140625,
      "KL/rejected_KL_mean": -202.9738311767578,
      "KL/std": 88.10664367675781,
      "epoch": 0.6455026455026455,
      "fcm_dpo/beta": 0.006783302407711744,
      "fcm_dpo/delta": 0.025555633008480072,
      "fcm_dpo/margin": 55.34111022949219,
      "fcm_dpo/q_t": 0.4163426160812378,
      "grad_norm": 14.305885314941406,
      "learning_rate": 1.6948364510535218e-07,
      "logits/chosen": 0.7149187922477722,
      "logits/rejected": 0.648948073387146,
      "logps/chosen": -206.27548217773438,
      "logps/ref_chosen": -58.64276885986328,
      "logps/ref_rejected": -86.25437927246094,
      "logps/rejected": -289.22821044921875,
      "loss": 1.1496,
      "margin_dpo/margin_mean": 55.34111022949219,
      "margin_dpo/margin_std": 98.70128631591797,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -140.02597045898438,
      "KL/mean": -171.49301147460938,
      "KL/rejected_KL_mean": -202.9600830078125,
      "KL/std": 90.05294036865234,
      "epoch": 0.6470143613000756,
      "fcm_dpo/beta": 0.0068000624887645245,
      "fcm_dpo/delta": -0.029416140168905258,
      "fcm_dpo/margin": 62.93410110473633,
      "fcm_dpo/q_t": 0.4039532244205475,
      "grad_norm": 13.004261016845703,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": 0.6538349986076355,
      "logits/rejected": 0.6088840961456299,
      "logps/chosen": -206.62200927734375,
      "logps/ref_chosen": -66.5960464477539,
      "logps/ref_rejected": -82.3941650390625,
      "logps/rejected": -285.354248046875,
      "loss": 1.1041,
      "margin_dpo/margin_mean": 62.93410110473633,
      "margin_dpo/margin_std": 96.0467529296875,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -144.10736083984375,
      "KL/mean": -168.20201110839844,
      "KL/rejected_KL_mean": -192.29669189453125,
      "KL/std": 79.76614379882812,
      "epoch": 0.6485260770975056,
      "fcm_dpo/beta": 0.006865202449262142,
      "fcm_dpo/delta": 0.07080723345279694,
      "fcm_dpo/margin": 48.1893310546875,
      "fcm_dpo/q_t": 0.4240524172782898,
      "grad_norm": 14.727472305297852,
      "learning_rate": 1.669846604344412e-07,
      "logits/chosen": 0.6704204082489014,
      "logits/rejected": 0.6889761686325073,
      "logps/chosen": -201.11705017089844,
      "logps/ref_chosen": -57.00970458984375,
      "logps/ref_rejected": -59.86549377441406,
      "logps/rejected": -252.16217041015625,
      "loss": 1.1763,
      "margin_dpo/margin_mean": 48.1893310546875,
      "margin_dpo/margin_std": 89.95539855957031,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -128.78807067871094,
      "KL/mean": -168.096923828125,
      "KL/rejected_KL_mean": -207.40579223632812,
      "KL/std": 82.24606323242188,
      "epoch": 0.6500377928949358,
      "fcm_dpo/beta": 0.006718984805047512,
      "fcm_dpo/delta": -0.13545790314674377,
      "fcm_dpo/margin": 78.61772155761719,
      "fcm_dpo/q_t": 0.378243088722229,
      "grad_norm": 13.049544334411621,
      "learning_rate": 1.6573863381573954e-07,
      "logits/chosen": 0.584052562713623,
      "logits/rejected": 0.583921492099762,
      "logps/chosen": -188.35125732421875,
      "logps/ref_chosen": -59.563194274902344,
      "logps/ref_rejected": -70.52289581298828,
      "logps/rejected": -277.9286804199219,
      "loss": 1.0201,
      "margin_dpo/margin_mean": 78.61772155761719,
      "margin_dpo/margin_std": 93.9211654663086,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -127.179931640625,
      "KL/mean": -155.37991333007812,
      "KL/rejected_KL_mean": -183.57992553710938,
      "KL/std": 82.5494384765625,
      "epoch": 0.6515495086923658,
      "fcm_dpo/beta": 0.0066922870464622974,
      "fcm_dpo/delta": 0.02344253659248352,
      "fcm_dpo/margin": 56.399993896484375,
      "fcm_dpo/q_t": 0.4140698313713074,
      "grad_norm": 12.710555076599121,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": 0.6873359680175781,
      "logits/rejected": 0.6306154131889343,
      "logps/chosen": -177.3802490234375,
      "logps/ref_chosen": -50.20032501220703,
      "logps/ref_rejected": -77.81680297851562,
      "logps/rejected": -261.396728515625,
      "loss": 1.1307,
      "margin_dpo/margin_mean": 56.399986267089844,
      "margin_dpo/margin_std": 90.4119873046875,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -133.48675537109375,
      "KL/mean": -163.28750610351562,
      "KL/rejected_KL_mean": -193.0882568359375,
      "KL/std": 80.02529907226562,
      "epoch": 0.6530612244897959,
      "fcm_dpo/beta": 0.006705043837428093,
      "fcm_dpo/delta": 0.0003821754362434149,
      "fcm_dpo/margin": 59.60150146484375,
      "fcm_dpo/q_t": 0.40856361389160156,
      "grad_norm": 13.228846549987793,
      "learning_rate": 1.632536862810844e-07,
      "logits/chosen": 0.7468098402023315,
      "logits/rejected": 0.6909035444259644,
      "logps/chosen": -195.14950561523438,
      "logps/ref_chosen": -61.662757873535156,
      "logps/ref_rejected": -83.94496154785156,
      "logps/rejected": -277.033203125,
      "loss": 1.1181,
      "margin_dpo/margin_mean": 59.60150146484375,
      "margin_dpo/margin_std": 93.37167358398438,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -132.0479736328125,
      "KL/mean": -167.79006958007812,
      "KL/rejected_KL_mean": -203.53219604492188,
      "KL/std": 78.00283813476562,
      "epoch": 0.654572940287226,
      "fcm_dpo/beta": 0.006647471338510513,
      "fcm_dpo/delta": -0.07887715846300125,
      "fcm_dpo/margin": 71.48422241210938,
      "fcm_dpo/q_t": 0.3904153108596802,
      "grad_norm": 12.910982131958008,
      "learning_rate": 1.6201483487445515e-07,
      "logits/chosen": 0.7597838044166565,
      "logits/rejected": 0.7596007585525513,
      "logps/chosen": -195.77716064453125,
      "logps/ref_chosen": -63.72917938232422,
      "logps/ref_rejected": -65.8391342163086,
      "logps/rejected": -269.371337890625,
      "loss": 1.0521,
      "margin_dpo/margin_mean": 71.4842300415039,
      "margin_dpo/margin_std": 89.87313842773438,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -110.9608154296875,
      "KL/mean": -150.4136962890625,
      "KL/rejected_KL_mean": -189.86659240722656,
      "KL/std": 86.0859146118164,
      "epoch": 0.656084656084656,
      "fcm_dpo/beta": 0.006446614395827055,
      "fcm_dpo/delta": -0.11617424339056015,
      "fcm_dpo/margin": 78.90575408935547,
      "fcm_dpo/q_t": 0.38356611132621765,
      "grad_norm": 12.279605865478516,
      "learning_rate": 1.6077844460203204e-07,
      "logits/chosen": 0.8191932439804077,
      "logits/rejected": 0.7523195743560791,
      "logps/chosen": -158.93414306640625,
      "logps/ref_chosen": -47.97331619262695,
      "logps/ref_rejected": -72.51132202148438,
      "logps/rejected": -262.3779296875,
      "loss": 1.0486,
      "margin_dpo/margin_mean": 78.90576171875,
      "margin_dpo/margin_std": 104.06834411621094,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -136.116943359375,
      "KL/mean": -166.03744506835938,
      "KL/rejected_KL_mean": -195.95794677734375,
      "KL/std": 81.53556060791016,
      "epoch": 0.6575963718820862,
      "fcm_dpo/beta": 0.006492358632385731,
      "fcm_dpo/delta": 0.011454716324806213,
      "fcm_dpo/margin": 59.841007232666016,
      "fcm_dpo/q_t": 0.4118584394454956,
      "grad_norm": 13.53164005279541,
      "learning_rate": 1.5954455004830878e-07,
      "logits/chosen": 0.8111344575881958,
      "logits/rejected": 0.7702116966247559,
      "logps/chosen": -193.17718505859375,
      "logps/ref_chosen": -57.06024932861328,
      "logps/ref_rejected": -71.69146728515625,
      "logps/rejected": -267.6494140625,
      "loss": 1.1248,
      "margin_dpo/margin_mean": 59.84100341796875,
      "margin_dpo/margin_std": 94.02011108398438,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -134.49127197265625,
      "KL/mean": -158.77999877929688,
      "KL/rejected_KL_mean": -183.06871032714844,
      "KL/std": 80.46412658691406,
      "epoch": 0.6591080876795162,
      "fcm_dpo/beta": 0.006544335745275021,
      "fcm_dpo/delta": 0.08483142405748367,
      "fcm_dpo/margin": 48.57743453979492,
      "fcm_dpo/q_t": 0.42810964584350586,
      "grad_norm": 15.03208065032959,
      "learning_rate": 1.5831318572796847e-07,
      "logits/chosen": 0.7064374685287476,
      "logits/rejected": 0.6471656560897827,
      "logps/chosen": -190.64932250976562,
      "logps/ref_chosen": -56.158050537109375,
      "logps/ref_rejected": -67.63787841796875,
      "logps/rejected": -250.7065887451172,
      "loss": 1.1948,
      "margin_dpo/margin_mean": 48.577430725097656,
      "margin_dpo/margin_std": 99.97824096679688,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -141.047119140625,
      "KL/mean": -169.44078063964844,
      "KL/rejected_KL_mean": -197.83445739746094,
      "KL/std": 86.12922668457031,
      "epoch": 0.6606198034769464,
      "fcm_dpo/beta": 0.006479623261839151,
      "fcm_dpo/delta": -0.07412885129451752,
      "fcm_dpo/margin": 56.78731918334961,
      "fcm_dpo/q_t": 0.4163801670074463,
      "grad_norm": 16.293567657470703,
      "learning_rate": 1.5708438608491815e-07,
      "logits/chosen": 0.7232198715209961,
      "logits/rejected": 0.5892056226730347,
      "logps/chosen": -198.03289794921875,
      "logps/ref_chosen": -56.98578643798828,
      "logps/ref_rejected": -85.61524963378906,
      "logps/rejected": -283.44970703125,
      "loss": 1.1738,
      "margin_dpo/margin_mean": 56.78731918334961,
      "margin_dpo/margin_std": 108.36846923828125,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -122.09769439697266,
      "KL/mean": -160.79986572265625,
      "KL/rejected_KL_mean": -199.50201416015625,
      "KL/std": 89.67132568359375,
      "epoch": 0.6621315192743764,
      "fcm_dpo/beta": 0.006402880884706974,
      "fcm_dpo/delta": -0.10047941654920578,
      "fcm_dpo/margin": 77.40432739257812,
      "fcm_dpo/q_t": 0.38928499817848206,
      "grad_norm": 12.973529815673828,
      "learning_rate": 1.558581854913253e-07,
      "logits/chosen": 0.7634217143058777,
      "logits/rejected": 0.695213794708252,
      "logps/chosen": -163.37547302246094,
      "logps/ref_chosen": -41.27777862548828,
      "logps/ref_rejected": -65.33840942382812,
      "logps/rejected": -264.8404541015625,
      "loss": 1.0399,
      "margin_dpo/margin_mean": 77.40432739257812,
      "margin_dpo/margin_std": 97.42752838134766,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -137.4156036376953,
      "KL/mean": -170.46905517578125,
      "KL/rejected_KL_mean": -203.52252197265625,
      "KL/std": 91.36854553222656,
      "epoch": 0.6636432350718064,
      "fcm_dpo/beta": 0.00636872835457325,
      "fcm_dpo/delta": -0.022579334676265717,
      "fcm_dpo/margin": 66.10691833496094,
      "fcm_dpo/q_t": 0.4040879011154175,
      "grad_norm": 13.05951976776123,
      "learning_rate": 1.5463461824665658e-07,
      "logits/chosen": 0.6218644976615906,
      "logits/rejected": 0.5845237970352173,
      "logps/chosen": -218.833251953125,
      "logps/ref_chosen": -81.41764831542969,
      "logps/ref_rejected": -94.72309875488281,
      "logps/rejected": -298.24560546875,
      "loss": 1.094,
      "margin_dpo/margin_mean": 66.10691833496094,
      "margin_dpo/margin_std": 93.14751434326172,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -120.3977279663086,
      "KL/mean": -154.42950439453125,
      "KL/rejected_KL_mean": -188.4612579345703,
      "KL/std": 83.01461029052734,
      "epoch": 0.6651549508692366,
      "fcm_dpo/beta": 0.006322925444692373,
      "fcm_dpo/delta": -0.0320570133626461,
      "fcm_dpo/margin": 68.06352233886719,
      "fcm_dpo/q_t": 0.4021187722682953,
      "grad_norm": 18.716856002807617,
      "learning_rate": 1.534137185767178e-07,
      "logits/chosen": 0.697509765625,
      "logits/rejected": 0.5949869155883789,
      "logps/chosen": -162.9359130859375,
      "logps/ref_chosen": -42.538185119628906,
      "logps/ref_rejected": -69.78813934326172,
      "logps/rejected": -258.2493896484375,
      "loss": 1.0976,
      "margin_dpo/margin_mean": 68.06352233886719,
      "margin_dpo/margin_std": 99.37464904785156,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -119.99906158447266,
      "KL/mean": -157.342529296875,
      "KL/rejected_KL_mean": -194.6859893798828,
      "KL/std": 87.70115661621094,
      "epoch": 0.6666666666666666,
      "fcm_dpo/beta": 0.006194580812007189,
      "fcm_dpo/delta": -0.06630893051624298,
      "fcm_dpo/margin": 74.68692779541016,
      "fcm_dpo/q_t": 0.39194971323013306,
      "grad_norm": 14.74911880493164,
      "learning_rate": 1.521955206326976e-07,
      "logits/chosen": 0.6925072073936462,
      "logits/rejected": 0.5931464433670044,
      "logps/chosen": -177.59228515625,
      "logps/ref_chosen": -57.593223571777344,
      "logps/ref_rejected": -84.82878875732422,
      "logps/rejected": -279.5147705078125,
      "loss": 1.0343,
      "margin_dpo/margin_mean": 74.68692779541016,
      "margin_dpo/margin_std": 78.96488952636719,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -144.77291870117188,
      "KL/mean": -180.74819946289062,
      "KL/rejected_KL_mean": -216.72348022460938,
      "KL/std": 86.60952758789062,
      "epoch": 0.6681783824640968,
      "fcm_dpo/beta": 0.006144754588603973,
      "fcm_dpo/delta": -0.04426190257072449,
      "fcm_dpo/margin": 71.95057678222656,
      "fcm_dpo/q_t": 0.39794474840164185,
      "grad_norm": 14.505967140197754,
      "learning_rate": 1.5098005849021078e-07,
      "logits/chosen": 0.6797877550125122,
      "logits/rejected": 0.6273739337921143,
      "logps/chosen": -212.234130859375,
      "logps/ref_chosen": -67.46121978759766,
      "logps/ref_rejected": -89.0693588256836,
      "logps/rejected": -305.7928466796875,
      "loss": 1.0643,
      "margin_dpo/margin_mean": 71.95057678222656,
      "margin_dpo/margin_std": 89.86045837402344,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -119.54753112792969,
      "KL/mean": -164.90615844726562,
      "KL/rejected_KL_mean": -210.26478576660156,
      "KL/std": 92.70027160644531,
      "epoch": 0.6696900982615268,
      "fcm_dpo/beta": 0.006005392409861088,
      "fcm_dpo/delta": -0.15315671265125275,
      "fcm_dpo/margin": 90.71723937988281,
      "fcm_dpo/q_t": 0.37534695863723755,
      "grad_norm": 12.429472923278809,
      "learning_rate": 1.4976736614834662e-07,
      "logits/chosen": 0.7227067947387695,
      "logits/rejected": 0.6522905826568604,
      "logps/chosen": -174.3436279296875,
      "logps/ref_chosen": -54.79610061645508,
      "logps/ref_rejected": -77.80781555175781,
      "logps/rejected": -288.0726013183594,
      "loss": 1.0017,
      "margin_dpo/margin_mean": 90.71723937988281,
      "margin_dpo/margin_std": 103.40176391601562,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -148.859130859375,
      "KL/mean": -166.98910522460938,
      "KL/rejected_KL_mean": -185.1190948486328,
      "KL/std": 90.59291076660156,
      "epoch": 0.671201814058957,
      "fcm_dpo/beta": 0.006004684139043093,
      "fcm_dpo/delta": 0.03823119029402733,
      "fcm_dpo/margin": 36.25995635986328,
      "fcm_dpo/q_t": 0.450982004404068,
      "grad_norm": 16.19681167602539,
      "learning_rate": 1.4855747752871654e-07,
      "logits/chosen": 0.7225247621536255,
      "logits/rejected": 0.6247001886367798,
      "logps/chosen": -207.60818481445312,
      "logps/ref_chosen": -58.749061584472656,
      "logps/ref_rejected": -86.87396240234375,
      "logps/rejected": -271.9930419921875,
      "loss": 1.2708,
      "margin_dpo/margin_mean": 36.25995635986328,
      "margin_dpo/margin_std": 100.88501739501953,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -138.85382080078125,
      "KL/mean": -177.49459838867188,
      "KL/rejected_KL_mean": -216.1353759765625,
      "KL/std": 86.65279388427734,
      "epoch": 0.672713529856387,
      "fcm_dpo/beta": 0.005953449755907059,
      "fcm_dpo/delta": -0.06295306235551834,
      "fcm_dpo/margin": 77.28157043457031,
      "fcm_dpo/q_t": 0.39340299367904663,
      "grad_norm": 13.37073040008545,
      "learning_rate": 1.473504264745062e-07,
      "logits/chosen": 0.6785054206848145,
      "logits/rejected": 0.6677216291427612,
      "logps/chosen": -199.771240234375,
      "logps/ref_chosen": -60.91743850708008,
      "logps/ref_rejected": -71.5637435913086,
      "logps/rejected": -287.6990966796875,
      "loss": 1.0496,
      "margin_dpo/margin_mean": 77.28157043457031,
      "margin_dpo/margin_std": 92.39187622070312,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -127.17668914794922,
      "KL/mean": -165.67218017578125,
      "KL/rejected_KL_mean": -204.16769409179688,
      "KL/std": 88.34965515136719,
      "epoch": 0.674225245653817,
      "fcm_dpo/beta": 0.005932152271270752,
      "fcm_dpo/delta": -0.06067255139350891,
      "fcm_dpo/margin": 76.99099731445312,
      "fcm_dpo/q_t": 0.39435237646102905,
      "grad_norm": 11.772911071777344,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 0.7165286540985107,
      "logits/rejected": 0.6347259283065796,
      "logps/chosen": -175.97593688964844,
      "logps/ref_chosen": -48.79924774169922,
      "logps/ref_rejected": -71.8719482421875,
      "logps/rejected": -276.0396728515625,
      "loss": 1.0475,
      "margin_dpo/margin_mean": 76.99099731445312,
      "margin_dpo/margin_std": 83.39089965820312,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -123.81182861328125,
      "KL/mean": -168.63693237304688,
      "KL/rejected_KL_mean": -213.4620361328125,
      "KL/std": 87.6279296875,
      "epoch": 0.6757369614512472,
      "fcm_dpo/beta": 0.005731572862714529,
      "fcm_dpo/delta": -0.12077778577804565,
      "fcm_dpo/margin": 89.65020751953125,
      "fcm_dpo/q_t": 0.3801065683364868,
      "grad_norm": 15.402129173278809,
      "learning_rate": 1.4494497203727843e-07,
      "logits/chosen": 0.6389660239219666,
      "logits/rejected": 0.5335906147956848,
      "logps/chosen": -177.49453735351562,
      "logps/ref_chosen": -53.682716369628906,
      "logps/ref_rejected": -88.17315673828125,
      "logps/rejected": -301.63519287109375,
      "loss": 1.0189,
      "margin_dpo/margin_mean": 89.65020751953125,
      "margin_dpo/margin_std": 101.47111511230469,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -133.37855529785156,
      "KL/mean": -169.70867919921875,
      "KL/rejected_KL_mean": -206.038818359375,
      "KL/std": 88.88766479492188,
      "epoch": 0.6772486772486772,
      "fcm_dpo/beta": 0.0057051535695791245,
      "fcm_dpo/delta": -0.015167122706770897,
      "fcm_dpo/margin": 72.6602554321289,
      "fcm_dpo/q_t": 0.4037356972694397,
      "grad_norm": 10.136807441711426,
      "learning_rate": 1.4374663593999256e-07,
      "logits/chosen": 0.7135224342346191,
      "logits/rejected": 0.6586642861366272,
      "logps/chosen": -187.1298065185547,
      "logps/ref_chosen": -53.75125503540039,
      "logps/ref_rejected": -77.17623901367188,
      "logps/rejected": -283.2150573730469,
      "loss": 1.084,
      "margin_dpo/margin_mean": 72.6602554321289,
      "margin_dpo/margin_std": 95.54232788085938,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -157.59341430664062,
      "KL/mean": -175.5412139892578,
      "KL/rejected_KL_mean": -193.48904418945312,
      "KL/std": 91.26512908935547,
      "epoch": 0.6787603930461074,
      "fcm_dpo/beta": 0.005783860106021166,
      "fcm_dpo/delta": 0.07222787290811539,
      "fcm_dpo/margin": 35.895633697509766,
      "fcm_dpo/q_t": 0.452186644077301,
      "grad_norm": 18.99204444885254,
      "learning_rate": 1.4255127197770707e-07,
      "logits/chosen": 0.568490207195282,
      "logits/rejected": 0.5677164793014526,
      "logps/chosen": -233.4207763671875,
      "logps/ref_chosen": -75.82737731933594,
      "logps/ref_rejected": -82.20687866210938,
      "logps/rejected": -275.6959228515625,
      "loss": 1.2544,
      "margin_dpo/margin_mean": 35.89563751220703,
      "margin_dpo/margin_std": 89.28213500976562,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -132.21817016601562,
      "KL/mean": -160.84329223632812,
      "KL/rejected_KL_mean": -189.4684295654297,
      "KL/std": 92.98245239257812,
      "epoch": 0.6802721088435374,
      "fcm_dpo/beta": 0.0058286152780056,
      "fcm_dpo/delta": 0.06864205747842789,
      "fcm_dpo/margin": 57.250274658203125,
      "fcm_dpo/q_t": 0.42646682262420654,
      "grad_norm": 12.80123519897461,
      "learning_rate": 1.4135891358732205e-07,
      "logits/chosen": 0.8172680139541626,
      "logits/rejected": 0.6988204717636108,
      "logps/chosen": -179.33389282226562,
      "logps/ref_chosen": -47.11572265625,
      "logps/ref_rejected": -78.7546615600586,
      "logps/rejected": -268.22308349609375,
      "loss": 1.1721,
      "margin_dpo/margin_mean": 57.250274658203125,
      "margin_dpo/margin_std": 107.00942993164062,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -133.332275390625,
      "KL/mean": -159.4159698486328,
      "KL/rejected_KL_mean": -185.4996337890625,
      "KL/std": 88.37306213378906,
      "epoch": 0.6817838246409675,
      "fcm_dpo/beta": 0.00595608027651906,
      "fcm_dpo/delta": 0.09190287441015244,
      "fcm_dpo/margin": 52.16736602783203,
      "fcm_dpo/q_t": 0.4287068843841553,
      "grad_norm": 12.640124320983887,
      "learning_rate": 1.4016959412166437e-07,
      "logits/chosen": 0.6722688674926758,
      "logits/rejected": 0.618954062461853,
      "logps/chosen": -196.68272399902344,
      "logps/ref_chosen": -63.350440979003906,
      "logps/ref_rejected": -76.28530883789062,
      "logps/rejected": -261.7849426269531,
      "loss": 1.1737,
      "margin_dpo/margin_mean": 52.16736602783203,
      "margin_dpo/margin_std": 93.84223175048828,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -131.25428771972656,
      "KL/mean": -161.52554321289062,
      "KL/rejected_KL_mean": -191.79681396484375,
      "KL/std": 84.22395324707031,
      "epoch": 0.6832955404383976,
      "fcm_dpo/beta": 0.006008903495967388,
      "fcm_dpo/delta": 0.0375509187579155,
      "fcm_dpo/margin": 60.54252624511719,
      "fcm_dpo/q_t": 0.41706210374832153,
      "grad_norm": 14.21445369720459,
      "learning_rate": 1.3898334684855645e-07,
      "logits/chosen": 0.6502448916435242,
      "logits/rejected": 0.5653257369995117,
      "logps/chosen": -186.84011840820312,
      "logps/ref_chosen": -55.58583450317383,
      "logps/ref_rejected": -77.68738555908203,
      "logps/rejected": -269.48419189453125,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 60.542518615722656,
      "margin_dpo/margin_std": 104.67784118652344,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -130.24049377441406,
      "KL/mean": -160.99288940429688,
      "KL/rejected_KL_mean": -191.74526977539062,
      "KL/std": 87.90748596191406,
      "epoch": 0.6848072562358276,
      "fcm_dpo/beta": 0.0060373879969120026,
      "fcm_dpo/delta": 0.029784685000777245,
      "fcm_dpo/margin": 61.50476837158203,
      "fcm_dpo/q_t": 0.41590872406959534,
      "grad_norm": 14.12247085571289,
      "learning_rate": 1.3780020494988445e-07,
      "logits/chosen": 0.6704771518707275,
      "logits/rejected": 0.6431600451469421,
      "logps/chosen": -192.01870727539062,
      "logps/ref_chosen": -61.778202056884766,
      "logps/ref_rejected": -71.51403045654297,
      "logps/rejected": -263.2593078613281,
      "loss": 1.1375,
      "margin_dpo/margin_mean": 61.50476837158203,
      "margin_dpo/margin_std": 101.02845764160156,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -121.20672607421875,
      "KL/mean": -156.61785888671875,
      "KL/rejected_KL_mean": -192.02899169921875,
      "KL/std": 88.88433074951172,
      "epoch": 0.6863189720332578,
      "fcm_dpo/beta": 0.006020670756697655,
      "fcm_dpo/delta": -0.02764631249010563,
      "fcm_dpo/margin": 70.822265625,
      "fcm_dpo/q_t": 0.4012266993522644,
      "grad_norm": 12.498412132263184,
      "learning_rate": 1.366202015206706e-07,
      "logits/chosen": 0.7018548250198364,
      "logits/rejected": 0.6610535383224487,
      "logps/chosen": -172.8018798828125,
      "logps/ref_chosen": -51.59515380859375,
      "logps/ref_rejected": -63.96732711791992,
      "logps/rejected": -255.99632263183594,
      "loss": 1.0866,
      "margin_dpo/margin_mean": 70.82225799560547,
      "margin_dpo/margin_std": 98.56320190429688,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -140.73394775390625,
      "KL/mean": -173.51605224609375,
      "KL/rejected_KL_mean": -206.2981719970703,
      "KL/std": 88.3670654296875,
      "epoch": 0.6878306878306878,
      "fcm_dpo/beta": 0.006017541047185659,
      "fcm_dpo/delta": 0.00566272996366024,
      "fcm_dpo/margin": 65.564208984375,
      "fcm_dpo/q_t": 0.4108354151248932,
      "grad_norm": 12.868791580200195,
      "learning_rate": 1.354433695681474e-07,
      "logits/chosen": 0.5786020755767822,
      "logits/rejected": 0.546318769454956,
      "logps/chosen": -211.38565063476562,
      "logps/ref_chosen": -70.65170288085938,
      "logps/ref_rejected": -77.44276428222656,
      "logps/rejected": -283.7409362792969,
      "loss": 1.1077,
      "margin_dpo/margin_mean": 65.564208984375,
      "margin_dpo/margin_std": 96.02351379394531,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -138.61538696289062,
      "KL/mean": -168.2949676513672,
      "KL/rejected_KL_mean": -197.9745635986328,
      "KL/std": 87.20057678222656,
      "epoch": 0.6893424036281179,
      "fcm_dpo/beta": 0.0060555217787623405,
      "fcm_dpo/delta": 0.04207714647054672,
      "fcm_dpo/margin": 59.35917663574219,
      "fcm_dpo/q_t": 0.4192585051059723,
      "grad_norm": 16.15275764465332,
      "learning_rate": 1.3426974201083439e-07,
      "logits/chosen": 0.6308639049530029,
      "logits/rejected": 0.5621622800827026,
      "logps/chosen": -195.013671875,
      "logps/ref_chosen": -56.398284912109375,
      "logps/ref_rejected": -82.61642456054688,
      "logps/rejected": -280.59100341796875,
      "loss": 1.1459,
      "margin_dpo/margin_mean": 59.35917663574219,
      "margin_dpo/margin_std": 100.7418212890625,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -136.33636474609375,
      "KL/mean": -169.51856994628906,
      "KL/rejected_KL_mean": -202.7008056640625,
      "KL/std": 90.15093994140625,
      "epoch": 0.690854119425548,
      "fcm_dpo/beta": 0.006060744635760784,
      "fcm_dpo/delta": -0.0024417489767074585,
      "fcm_dpo/margin": 66.36444091796875,
      "fcm_dpo/q_t": 0.4067618250846863,
      "grad_norm": 12.463237762451172,
      "learning_rate": 1.3309935167761717e-07,
      "logits/chosen": 0.7749881744384766,
      "logits/rejected": 0.6943407654762268,
      "logps/chosen": -181.0569305419922,
      "logps/ref_chosen": -44.72057342529297,
      "logps/ref_rejected": -68.1158676147461,
      "logps/rejected": -270.816650390625,
      "loss": 1.0905,
      "margin_dpo/margin_mean": 66.36444091796875,
      "margin_dpo/margin_std": 87.23387145996094,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -131.62559509277344,
      "KL/mean": -165.6820068359375,
      "KL/rejected_KL_mean": -199.73841857910156,
      "KL/std": 92.67698669433594,
      "epoch": 0.6923658352229781,
      "fcm_dpo/beta": 0.006066558416932821,
      "fcm_dpo/delta": -0.013779795728623867,
      "fcm_dpo/margin": 68.11283874511719,
      "fcm_dpo/q_t": 0.4059259295463562,
      "grad_norm": 13.29777717590332,
      "learning_rate": 1.3193223130682936e-07,
      "logits/chosen": 0.7090173959732056,
      "logits/rejected": 0.586572527885437,
      "logps/chosen": -181.63128662109375,
      "logps/ref_chosen": -50.00569152832031,
      "logps/ref_rejected": -87.50015258789062,
      "logps/rejected": -287.23858642578125,
      "loss": 1.1093,
      "margin_dpo/margin_mean": 68.11283874511719,
      "margin_dpo/margin_std": 104.8509750366211,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -125.52388000488281,
      "KL/mean": -167.6014404296875,
      "KL/rejected_KL_mean": -209.6790008544922,
      "KL/std": 102.99993133544922,
      "epoch": 0.6938775510204082,
      "fcm_dpo/beta": 0.006006724201142788,
      "fcm_dpo/delta": -0.11152348667383194,
      "fcm_dpo/margin": 84.15510559082031,
      "fcm_dpo/q_t": 0.38333696126937866,
      "grad_norm": 11.724173545837402,
      "learning_rate": 1.3076841354533658e-07,
      "logits/chosen": 0.7097588181495667,
      "logits/rejected": 0.6748946309089661,
      "logps/chosen": -190.90182495117188,
      "logps/ref_chosen": -65.37794494628906,
      "logps/ref_rejected": -88.19244384765625,
      "logps/rejected": -297.8714599609375,
      "loss": 1.0331,
      "margin_dpo/margin_mean": 84.15511322021484,
      "margin_dpo/margin_std": 99.3807373046875,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -138.87803649902344,
      "KL/mean": -179.3299102783203,
      "KL/rejected_KL_mean": -219.78176879882812,
      "KL/std": 92.59164428710938,
      "epoch": 0.6953892668178382,
      "fcm_dpo/beta": 0.005839239340275526,
      "fcm_dpo/delta": -0.07644946128129959,
      "fcm_dpo/margin": 80.90373229980469,
      "fcm_dpo/q_t": 0.39336204528808594,
      "grad_norm": 12.743875503540039,
      "learning_rate": 1.2960793094762345e-07,
      "logits/chosen": 0.7132609486579895,
      "logits/rejected": 0.5876985788345337,
      "logps/chosen": -203.439697265625,
      "logps/ref_chosen": -64.5616683959961,
      "logps/ref_rejected": -88.67890167236328,
      "logps/rejected": -308.4606628417969,
      "loss": 1.0515,
      "margin_dpo/margin_mean": 80.90373229980469,
      "margin_dpo/margin_std": 102.73490905761719,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -114.52981567382812,
      "KL/mean": -155.5570068359375,
      "KL/rejected_KL_mean": -196.58416748046875,
      "KL/std": 85.63592529296875,
      "epoch": 0.6969009826152683,
      "fcm_dpo/beta": 0.005715795326977968,
      "fcm_dpo/delta": -0.07494309544563293,
      "fcm_dpo/margin": 82.05435180664062,
      "fcm_dpo/q_t": 0.3918275237083435,
      "grad_norm": 13.224369049072266,
      "learning_rate": 1.2845081597488286e-07,
      "logits/chosen": 0.8121793866157532,
      "logits/rejected": 0.7237043380737305,
      "logps/chosen": -164.0077362060547,
      "logps/ref_chosen": -49.4779167175293,
      "logps/ref_rejected": -72.65262603759766,
      "logps/rejected": -269.2367858886719,
      "loss": 1.0494,
      "margin_dpo/margin_mean": 82.05435180664062,
      "margin_dpo/margin_std": 95.64311981201172,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -127.10210418701172,
      "KL/mean": -168.91156005859375,
      "KL/rejected_KL_mean": -210.72100830078125,
      "KL/std": 86.50918579101562,
      "epoch": 0.6984126984126984,
      "fcm_dpo/beta": 0.005665352568030357,
      "fcm_dpo/delta": -0.07779423892498016,
      "fcm_dpo/margin": 83.61892700195312,
      "fcm_dpo/q_t": 0.3893454670906067,
      "grad_norm": 11.871650695800781,
      "learning_rate": 1.27297100994108e-07,
      "logits/chosen": 0.7033039331436157,
      "logits/rejected": 0.6476036310195923,
      "logps/chosen": -187.59722900390625,
      "logps/ref_chosen": -60.4951171875,
      "logps/ref_rejected": -74.82136535644531,
      "logps/rejected": -285.5423889160156,
      "loss": 1.039,
      "margin_dpo/margin_mean": 83.61892700195312,
      "margin_dpo/margin_std": 96.71485900878906,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -152.2330780029297,
      "KL/mean": -177.74392700195312,
      "KL/rejected_KL_mean": -203.25479125976562,
      "KL/std": 84.8050537109375,
      "epoch": 0.6999244142101285,
      "fcm_dpo/beta": 0.00566816283389926,
      "fcm_dpo/delta": 0.007925955578684807,
      "fcm_dpo/margin": 51.02170944213867,
      "fcm_dpo/q_t": 0.4322904050350189,
      "grad_norm": 17.04616928100586,
      "learning_rate": 1.2614681827718695e-07,
      "logits/chosen": 0.6792501211166382,
      "logits/rejected": 0.6787852644920349,
      "logps/chosen": -219.91818237304688,
      "logps/ref_chosen": -67.68511962890625,
      "logps/ref_rejected": -71.32196044921875,
      "logps/rejected": -274.57672119140625,
      "loss": 1.1851,
      "margin_dpo/margin_mean": 51.02171325683594,
      "margin_dpo/margin_std": 91.60702514648438,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -138.76654052734375,
      "KL/mean": -177.50930786132812,
      "KL/rejected_KL_mean": -216.2520751953125,
      "KL/std": 93.81965637207031,
      "epoch": 0.7014361300075586,
      "fcm_dpo/beta": 0.005656754598021507,
      "fcm_dpo/delta": -0.04023423045873642,
      "fcm_dpo/margin": 77.48552703857422,
      "fcm_dpo/q_t": 0.3993530869483948,
      "grad_norm": 11.432208061218262,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": 0.7230494022369385,
      "logits/rejected": 0.6913472414016724,
      "logps/chosen": -197.93218994140625,
      "logps/ref_chosen": -59.16564178466797,
      "logps/ref_rejected": -69.56146240234375,
      "logps/rejected": -285.81353759765625,
      "loss": 1.0868,
      "margin_dpo/margin_mean": 77.48553466796875,
      "margin_dpo/margin_std": 107.69242858886719,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -146.481201171875,
      "KL/mean": -179.44790649414062,
      "KL/rejected_KL_mean": -212.41461181640625,
      "KL/std": 90.61519622802734,
      "epoch": 0.7029478458049887,
      "fcm_dpo/beta": 0.0056340936571359634,
      "fcm_dpo/delta": 0.02957913652062416,
      "fcm_dpo/margin": 65.93341064453125,
      "fcm_dpo/q_t": 0.4151262640953064,
      "grad_norm": 14.615275382995605,
      "learning_rate": 1.238566782415197e-07,
      "logits/chosen": 0.7887932062149048,
      "logits/rejected": 0.7234373092651367,
      "logps/chosen": -204.994873046875,
      "logps/ref_chosen": -58.513671875,
      "logps/ref_rejected": -84.31745910644531,
      "logps/rejected": -296.7320556640625,
      "loss": 1.1311,
      "margin_dpo/margin_mean": 65.93341064453125,
      "margin_dpo/margin_std": 104.03491973876953,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -159.19192504882812,
      "KL/mean": -177.74716186523438,
      "KL/rejected_KL_mean": -196.30239868164062,
      "KL/std": 97.22972869873047,
      "epoch": 0.7044595616024187,
      "fcm_dpo/beta": 0.005733816884458065,
      "fcm_dpo/delta": 0.0682370513677597,
      "fcm_dpo/margin": 37.110450744628906,
      "fcm_dpo/q_t": 0.4506417512893677,
      "grad_norm": 19.955272674560547,
      "learning_rate": 1.2271688498291334e-07,
      "logits/chosen": 0.6822282075881958,
      "logits/rejected": 0.6884140968322754,
      "logps/chosen": -232.45774841308594,
      "logps/ref_chosen": -73.26580810546875,
      "logps/ref_rejected": -74.83621215820312,
      "logps/rejected": -271.13861083984375,
      "loss": 1.2601,
      "margin_dpo/margin_mean": 37.110450744628906,
      "margin_dpo/margin_std": 97.2080078125,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -140.95587158203125,
      "KL/mean": -170.89016723632812,
      "KL/rejected_KL_mean": -200.824462890625,
      "KL/std": 92.99038696289062,
      "epoch": 0.7059712773998488,
      "fcm_dpo/beta": 0.005779305938631296,
      "fcm_dpo/delta": 0.05595749616622925,
      "fcm_dpo/margin": 59.86858367919922,
      "fcm_dpo/q_t": 0.42141276597976685,
      "grad_norm": 11.946219444274902,
      "learning_rate": 1.2158065210664848e-07,
      "logits/chosen": 0.782062292098999,
      "logits/rejected": 0.6244519352912903,
      "logps/chosen": -188.5353546142578,
      "logps/ref_chosen": -47.57947540283203,
      "logps/ref_rejected": -78.68522644042969,
      "logps/rejected": -279.50970458984375,
      "loss": 1.1384,
      "margin_dpo/margin_mean": 59.86858367919922,
      "margin_dpo/margin_std": 94.36546325683594,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -133.36300659179688,
      "KL/mean": -173.8083953857422,
      "KL/rejected_KL_mean": -214.25381469726562,
      "KL/std": 92.20292663574219,
      "epoch": 0.7074829931972789,
      "fcm_dpo/beta": 0.0057451799511909485,
      "fcm_dpo/delta": -0.06784342974424362,
      "fcm_dpo/margin": 80.89079284667969,
      "fcm_dpo/q_t": 0.3924998939037323,
      "grad_norm": 15.625370025634766,
      "learning_rate": 1.204480113956011e-07,
      "logits/chosen": 0.6952544450759888,
      "logits/rejected": 0.6817104816436768,
      "logps/chosen": -197.29080200195312,
      "logps/ref_chosen": -63.92778778076172,
      "logps/ref_rejected": -76.51626586914062,
      "logps/rejected": -290.77008056640625,
      "loss": 1.0619,
      "margin_dpo/margin_mean": 80.89079284667969,
      "margin_dpo/margin_std": 106.03605651855469,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -135.75332641601562,
      "KL/mean": -173.4093017578125,
      "KL/rejected_KL_mean": -211.06529235839844,
      "KL/std": 90.90241241455078,
      "epoch": 0.708994708994709,
      "fcm_dpo/beta": 0.00566452369093895,
      "fcm_dpo/delta": -0.028576456010341644,
      "fcm_dpo/margin": 75.31197357177734,
      "fcm_dpo/q_t": 0.4005330204963684,
      "grad_norm": 12.453137397766113,
      "learning_rate": 1.1931899453216697e-07,
      "logits/chosen": 0.7379674911499023,
      "logits/rejected": 0.7244564294815063,
      "logps/chosen": -194.81149291992188,
      "logps/ref_chosen": -59.05818176269531,
      "logps/ref_rejected": -75.67672729492188,
      "logps/rejected": -286.74200439453125,
      "loss": 1.062,
      "margin_dpo/margin_mean": 75.31197357177734,
      "margin_dpo/margin_std": 85.01697540283203,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -129.7604522705078,
      "KL/mean": -165.704345703125,
      "KL/rejected_KL_mean": -201.648193359375,
      "KL/std": 89.09385681152344,
      "epoch": 0.7105064247921391,
      "fcm_dpo/beta": 0.00569544080644846,
      "fcm_dpo/delta": -0.009973295032978058,
      "fcm_dpo/margin": 71.88774108886719,
      "fcm_dpo/q_t": 0.4055355489253998,
      "grad_norm": 12.146196365356445,
      "learning_rate": 1.1819363309737438e-07,
      "logits/chosen": 0.7284529209136963,
      "logits/rejected": 0.6563238501548767,
      "logps/chosen": -177.62789916992188,
      "logps/ref_chosen": -47.86743927001953,
      "logps/ref_rejected": -65.96859741210938,
      "logps/rejected": -267.6167907714844,
      "loss": 1.0957,
      "margin_dpo/margin_mean": 71.88774108886719,
      "margin_dpo/margin_std": 99.98933410644531,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -125.92105102539062,
      "KL/mean": -166.09390258789062,
      "KL/rejected_KL_mean": -206.26675415039062,
      "KL/std": 89.02778625488281,
      "epoch": 0.7120181405895691,
      "fcm_dpo/beta": 0.005620558280497789,
      "fcm_dpo/delta": -0.05419111251831055,
      "fcm_dpo/margin": 80.345703125,
      "fcm_dpo/q_t": 0.39483213424682617,
      "grad_norm": 11.916303634643555,
      "learning_rate": 1.1707195857000215e-07,
      "logits/chosen": 0.7052686810493469,
      "logits/rejected": 0.6493145227432251,
      "logps/chosen": -183.69891357421875,
      "logps/ref_chosen": -57.777854919433594,
      "logps/ref_rejected": -73.81172180175781,
      "logps/rejected": -280.0784912109375,
      "loss": 1.0583,
      "margin_dpo/margin_mean": 80.34571075439453,
      "margin_dpo/margin_std": 98.62115478515625,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -128.2120361328125,
      "KL/mean": -159.3995819091797,
      "KL/rejected_KL_mean": -190.58712768554688,
      "KL/std": 91.06816101074219,
      "epoch": 0.7135298563869993,
      "fcm_dpo/beta": 0.005674063693732023,
      "fcm_dpo/delta": 0.04737677052617073,
      "fcm_dpo/margin": 62.37507629394531,
      "fcm_dpo/q_t": 0.42010074853897095,
      "grad_norm": 13.604077339172363,
      "learning_rate": 1.1595400232569768e-07,
      "logits/chosen": 0.7396783828735352,
      "logits/rejected": 0.6912394762039185,
      "logps/chosen": -184.12071228027344,
      "logps/ref_chosen": -55.908668518066406,
      "logps/ref_rejected": -74.70294189453125,
      "logps/rejected": -265.2900695800781,
      "loss": 1.1645,
      "margin_dpo/margin_mean": 62.37507629394531,
      "margin_dpo/margin_std": 115.64419555664062,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -130.78228759765625,
      "KL/mean": -166.56057739257812,
      "KL/rejected_KL_mean": -202.33888244628906,
      "KL/std": 100.03340911865234,
      "epoch": 0.7150415721844293,
      "fcm_dpo/beta": 0.005662200972437859,
      "fcm_dpo/delta": -0.005389830097556114,
      "fcm_dpo/margin": 71.55660247802734,
      "fcm_dpo/q_t": 0.4097879230976105,
      "grad_norm": 13.815281867980957,
      "learning_rate": 1.1483979563610069e-07,
      "logits/chosen": 0.8241503238677979,
      "logits/rejected": 0.7026021480560303,
      "logps/chosen": -184.9431610107422,
      "logps/ref_chosen": -54.16088104248047,
      "logps/ref_rejected": -92.76789855957031,
      "logps/rejected": -295.1067810058594,
      "loss": 1.132,
      "margin_dpo/margin_mean": 71.55659484863281,
      "margin_dpo/margin_std": 121.6928482055664,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -133.13714599609375,
      "KL/mean": -165.9842529296875,
      "KL/rejected_KL_mean": -198.83135986328125,
      "KL/std": 93.28158569335938,
      "epoch": 0.7165532879818595,
      "fcm_dpo/beta": 0.005689322017133236,
      "fcm_dpo/delta": 0.0272356066852808,
      "fcm_dpo/margin": 65.6942138671875,
      "fcm_dpo/q_t": 0.41614243388175964,
      "grad_norm": 16.53423500061035,
      "learning_rate": 1.1372936966796709e-07,
      "logits/chosen": 0.8011815547943115,
      "logits/rejected": 0.7208400368690491,
      "logps/chosen": -179.82284545898438,
      "logps/ref_chosen": -46.685707092285156,
      "logps/ref_rejected": -71.44731903076172,
      "logps/rejected": -270.2786865234375,
      "loss": 1.1446,
      "margin_dpo/margin_mean": 65.6942138671875,
      "margin_dpo/margin_std": 113.58468627929688,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -127.18001556396484,
      "KL/mean": -173.79864501953125,
      "KL/rejected_KL_mean": -220.417236328125,
      "KL/std": 95.05990600585938,
      "epoch": 0.7180650037792895,
      "fcm_dpo/beta": 0.005567646585404873,
      "fcm_dpo/delta": -0.12634103000164032,
      "fcm_dpo/margin": 93.23724365234375,
      "fcm_dpo/q_t": 0.3815461993217468,
      "grad_norm": 10.223531723022461,
      "learning_rate": 1.126227554822985e-07,
      "logits/chosen": 0.7260850667953491,
      "logits/rejected": 0.671942949295044,
      "logps/chosen": -185.66732788085938,
      "logps/ref_chosen": -58.4873046875,
      "logps/ref_rejected": -87.00187683105469,
      "logps/rejected": -307.41912841796875,
      "loss": 1.0112,
      "margin_dpo/margin_mean": 93.23725128173828,
      "margin_dpo/margin_std": 104.22055053710938,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -153.9020233154297,
      "KL/mean": -184.61090087890625,
      "KL/rejected_KL_mean": -215.31976318359375,
      "KL/std": 98.1943130493164,
      "epoch": 0.7195767195767195,
      "fcm_dpo/beta": 0.005622149910777807,
      "fcm_dpo/delta": 0.056033432483673096,
      "fcm_dpo/margin": 61.41775131225586,
      "fcm_dpo/q_t": 0.42167773842811584,
      "grad_norm": 13.574383735656738,
      "learning_rate": 1.1151998403347243e-07,
      "logits/chosen": 0.6377418637275696,
      "logits/rejected": 0.6381895542144775,
      "logps/chosen": -229.2836456298828,
      "logps/ref_chosen": -75.38162231445312,
      "logps/ref_rejected": -76.99822235107422,
      "logps/rejected": -292.3179931640625,
      "loss": 1.1554,
      "margin_dpo/margin_mean": 61.417755126953125,
      "margin_dpo/margin_std": 105.65559387207031,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -156.93634033203125,
      "KL/mean": -188.2996368408203,
      "KL/rejected_KL_mean": -219.6629180908203,
      "KL/std": 97.05319213867188,
      "epoch": 0.7210884353741497,
      "fcm_dpo/beta": 0.005650391336530447,
      "fcm_dpo/delta": 0.04726497828960419,
      "fcm_dpo/margin": 62.726585388183594,
      "fcm_dpo/q_t": 0.41989630460739136,
      "grad_norm": 14.775123596191406,
      "learning_rate": 1.1042108616837692e-07,
      "logits/chosen": 0.721420168876648,
      "logits/rejected": 0.6686294078826904,
      "logps/chosen": -218.00973510742188,
      "logps/ref_chosen": -61.073387145996094,
      "logps/ref_rejected": -81.34375,
      "logps/rejected": -301.00665283203125,
      "loss": 1.1832,
      "margin_dpo/margin_mean": 62.72658920288086,
      "margin_dpo/margin_std": 126.39774322509766,
      "step": 477
    },
    {
      "KL/chosen_KL_mean": -137.70639038085938,
      "KL/mean": -163.52902221679688,
      "KL/rejected_KL_mean": -189.35162353515625,
      "KL/std": 87.43692779541016,
      "epoch": 0.7226001511715797,
      "fcm_dpo/beta": 0.005741935223340988,
      "fcm_dpo/delta": 0.1067572608590126,
      "fcm_dpo/margin": 51.64522933959961,
      "fcm_dpo/q_t": 0.432145893573761,
      "grad_norm": 15.854500770568848,
      "learning_rate": 1.0932609262554746e-07,
      "logits/chosen": 0.6768746376037598,
      "logits/rejected": 0.6878693103790283,
      "logps/chosen": -194.87371826171875,
      "logps/ref_chosen": -57.16731643676758,
      "logps/ref_rejected": -53.30917739868164,
      "logps/rejected": -242.66079711914062,
      "loss": 1.2021,
      "margin_dpo/margin_mean": 51.645225524902344,
      "margin_dpo/margin_std": 106.51991271972656,
      "step": 478
    },
    {
      "KL/chosen_KL_mean": -142.2852020263672,
      "KL/mean": -165.98593139648438,
      "KL/rejected_KL_mean": -189.6866455078125,
      "KL/std": 86.13800048828125,
      "epoch": 0.7241118669690099,
      "fcm_dpo/beta": 0.005812506657093763,
      "fcm_dpo/delta": 0.019841192290186882,
      "fcm_dpo/margin": 47.40142059326172,
      "fcm_dpo/q_t": 0.436745822429657,
      "grad_norm": 15.112234115600586,
      "learning_rate": 1.0823503403430734e-07,
      "logits/chosen": 0.6774095296859741,
      "logits/rejected": 0.6323498487472534,
      "logps/chosen": -201.19851684570312,
      "logps/ref_chosen": -58.91331481933594,
      "logps/ref_rejected": -63.7403450012207,
      "logps/rejected": -253.42697143554688,
      "loss": 1.2139,
      "margin_dpo/margin_mean": 47.40142059326172,
      "margin_dpo/margin_std": 101.36029052734375,
      "step": 479
    },
    {
      "KL/chosen_KL_mean": -146.10574340820312,
      "KL/mean": -182.49282836914062,
      "KL/rejected_KL_mean": -218.87991333007812,
      "KL/std": 90.76347351074219,
      "epoch": 0.7256235827664399,
      "fcm_dpo/beta": 0.005832264199852943,
      "fcm_dpo/delta": -0.026415158063173294,
      "fcm_dpo/margin": 72.774169921875,
      "fcm_dpo/q_t": 0.4029375910758972,
      "grad_norm": 15.650308609008789,
      "learning_rate": 1.0714794091391072e-07,
      "logits/chosen": 0.6514978408813477,
      "logits/rejected": 0.6414633989334106,
      "logps/chosen": -208.90634155273438,
      "logps/ref_chosen": -62.80061340332031,
      "logps/ref_rejected": -67.58859252929688,
      "logps/rejected": -286.468505859375,
      "loss": 1.0986,
      "margin_dpo/margin_mean": 72.774169921875,
      "margin_dpo/margin_std": 104.67288208007812,
      "step": 480
    },
    {
      "KL/chosen_KL_mean": -143.08575439453125,
      "KL/mean": -175.5304718017578,
      "KL/rejected_KL_mean": -207.97520446777344,
      "KL/std": 92.16765594482422,
      "epoch": 0.72713529856387,
      "fcm_dpo/beta": 0.005794272758066654,
      "fcm_dpo/delta": 0.024939395487308502,
      "fcm_dpo/margin": 64.88945770263672,
      "fcm_dpo/q_t": 0.414761483669281,
      "grad_norm": 15.237427711486816,
      "learning_rate": 1.0606484367268906e-07,
      "logits/chosen": 0.6654689311981201,
      "logits/rejected": 0.664907693862915,
      "logps/chosen": -208.3722381591797,
      "logps/ref_chosen": -65.28649139404297,
      "logps/ref_rejected": -70.78668212890625,
      "logps/rejected": -278.76190185546875,
      "loss": 1.1325,
      "margin_dpo/margin_mean": 64.88946533203125,
      "margin_dpo/margin_std": 105.70449829101562,
      "step": 481
    },
    {
      "KL/chosen_KL_mean": -160.675537109375,
      "KL/mean": -193.75030517578125,
      "KL/rejected_KL_mean": -226.82504272460938,
      "KL/std": 96.69131469726562,
      "epoch": 0.7286470143613001,
      "fcm_dpo/beta": 0.0058363573625683784,
      "fcm_dpo/delta": 0.014351559802889824,
      "fcm_dpo/margin": 66.14949798583984,
      "fcm_dpo/q_t": 0.41502517461776733,
      "grad_norm": 15.326761245727539,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": 0.6180112361907959,
      "logits/rejected": 0.4631701707839966,
      "logps/chosen": -221.58172607421875,
      "logps/ref_chosen": -60.906185150146484,
      "logps/ref_rejected": -103.44656372070312,
      "logps/rejected": -330.2716064453125,
      "loss": 1.1566,
      "margin_dpo/margin_mean": 66.14949798583984,
      "margin_dpo/margin_std": 123.32733154296875,
      "step": 482
    },
    {
      "KL/chosen_KL_mean": -135.55545043945312,
      "KL/mean": -176.0252685546875,
      "KL/rejected_KL_mean": -216.49508666992188,
      "KL/std": 90.32886505126953,
      "epoch": 0.7301587301587301,
      "fcm_dpo/beta": 0.005776412319391966,
      "fcm_dpo/delta": -0.07079232484102249,
      "fcm_dpo/margin": 80.93965148925781,
      "fcm_dpo/q_t": 0.39278119802474976,
      "grad_norm": 12.39647102355957,
      "learning_rate": 1.0391075790138232e-07,
      "logits/chosen": 0.7870754599571228,
      "logits/rejected": 0.6720584630966187,
      "logps/chosen": -188.74746704101562,
      "logps/ref_chosen": -53.192012786865234,
      "logps/ref_rejected": -81.83927154541016,
      "logps/rejected": -298.3343505859375,
      "loss": 1.0541,
      "margin_dpo/margin_mean": 80.93964385986328,
      "margin_dpo/margin_std": 102.43343353271484,
      "step": 483
    },
    {
      "KL/chosen_KL_mean": -140.46139526367188,
      "KL/mean": -166.83505249023438,
      "KL/rejected_KL_mean": -193.20867919921875,
      "KL/std": 87.44966125488281,
      "epoch": 0.7316704459561603,
      "fcm_dpo/beta": 0.005851096473634243,
      "fcm_dpo/delta": 0.09351673722267151,
      "fcm_dpo/margin": 52.747291564941406,
      "fcm_dpo/q_t": 0.42782455682754517,
      "grad_norm": 18.800477981567383,
      "learning_rate": 1.0283982962570681e-07,
      "logits/chosen": 0.7991921305656433,
      "logits/rejected": 0.764002799987793,
      "logps/chosen": -198.23085021972656,
      "logps/ref_chosen": -57.76945877075195,
      "logps/ref_rejected": -71.6829833984375,
      "logps/rejected": -264.89166259765625,
      "loss": 1.152,
      "margin_dpo/margin_mean": 52.747291564941406,
      "margin_dpo/margin_std": 77.53668975830078,
      "step": 484
    },
    {
      "KL/chosen_KL_mean": -144.2364959716797,
      "KL/mean": -173.74766540527344,
      "KL/rejected_KL_mean": -203.2588348388672,
      "KL/std": 91.27928924560547,
      "epoch": 0.7331821617535903,
      "fcm_dpo/beta": 0.005794328637421131,
      "fcm_dpo/delta": -0.05126110464334488,
      "fcm_dpo/margin": 59.022315979003906,
      "fcm_dpo/q_t": 0.4208328425884247,
      "grad_norm": 13.687870025634766,
      "learning_rate": 1.0177301773633992e-07,
      "logits/chosen": 0.7543034553527832,
      "logits/rejected": 0.7301796078681946,
      "logps/chosen": -200.87234497070312,
      "logps/ref_chosen": -56.63584899902344,
      "logps/ref_rejected": -70.85614013671875,
      "logps/rejected": -274.1149597167969,
      "loss": 1.141,
      "margin_dpo/margin_mean": 59.02231216430664,
      "margin_dpo/margin_std": 88.91297912597656,
      "step": 485
    },
    {
      "KL/chosen_KL_mean": -165.4752197265625,
      "KL/mean": -193.02178955078125,
      "KL/rejected_KL_mean": -220.56832885742188,
      "KL/std": 101.73480224609375,
      "epoch": 0.7346938775510204,
      "fcm_dpo/beta": 0.005861001089215279,
      "fcm_dpo/delta": 0.07970429956912994,
      "fcm_dpo/margin": 55.09308624267578,
      "fcm_dpo/q_t": 0.4290149509906769,
      "grad_norm": 12.349756240844727,
      "learning_rate": 1.007103520743035e-07,
      "logits/chosen": 0.7243193984031677,
      "logits/rejected": 0.6010168790817261,
      "logps/chosen": -221.82225036621094,
      "logps/ref_chosen": -56.347023010253906,
      "logps/ref_rejected": -85.97221374511719,
      "logps/rejected": -306.54052734375,
      "loss": 1.1997,
      "margin_dpo/margin_mean": 55.09308624267578,
      "margin_dpo/margin_std": 118.03974914550781,
      "step": 486
    },
    {
      "KL/chosen_KL_mean": -148.17214965820312,
      "KL/mean": -182.45108032226562,
      "KL/rejected_KL_mean": -216.73001098632812,
      "KL/std": 92.99481201171875,
      "epoch": 0.7362055933484505,
      "fcm_dpo/beta": 0.005880633369088173,
      "fcm_dpo/delta": -0.0033075781539082527,
      "fcm_dpo/margin": 68.5578384399414,
      "fcm_dpo/q_t": 0.410717636346817,
      "grad_norm": 14.310342788696289,
      "learning_rate": 9.965186236464046e-08,
      "logits/chosen": 0.8407909274101257,
      "logits/rejected": 0.7757810354232788,
      "logps/chosen": -208.78936767578125,
      "logps/ref_chosen": -60.617218017578125,
      "logps/ref_rejected": -82.50975036621094,
      "logps/rejected": -299.23974609375,
      "loss": 1.1157,
      "margin_dpo/margin_mean": 68.55783081054688,
      "margin_dpo/margin_std": 109.45668029785156,
      "step": 487
    },
    {
      "KL/chosen_KL_mean": -140.10665893554688,
      "KL/mean": -176.00613403320312,
      "KL/rejected_KL_mean": -211.90560913085938,
      "KL/std": 90.33186340332031,
      "epoch": 0.7377173091458806,
      "fcm_dpo/beta": 0.005884402431547642,
      "fcm_dpo/delta": -0.023673301562666893,
      "fcm_dpo/margin": 71.79894256591797,
      "fcm_dpo/q_t": 0.4045790731906891,
      "grad_norm": 15.953469276428223,
      "learning_rate": 9.859757821558337e-08,
      "logits/chosen": 0.7529109716415405,
      "logits/rejected": 0.6886953115463257,
      "logps/chosen": -203.21571350097656,
      "logps/ref_chosen": -63.10905075073242,
      "logps/ref_rejected": -82.49348449707031,
      "logps/rejected": -294.39910888671875,
      "loss": 1.0914,
      "margin_dpo/margin_mean": 71.7989501953125,
      "margin_dpo/margin_std": 101.41202545166016,
      "step": 488
    },
    {
      "KL/chosen_KL_mean": -162.86598205566406,
      "KL/mean": -185.1213836669922,
      "KL/rejected_KL_mean": -207.3767852783203,
      "KL/std": 99.34828186035156,
      "epoch": 0.7392290249433107,
      "fcm_dpo/beta": 0.005974326282739639,
      "fcm_dpo/delta": 0.1378115862607956,
      "fcm_dpo/margin": 44.51079559326172,
      "fcm_dpo/q_t": 0.43994566798210144,
      "grad_norm": 13.567418098449707,
      "learning_rate": 9.754752911772615e-08,
      "logits/chosen": 0.7378537654876709,
      "logits/rejected": 0.6861571669578552,
      "logps/chosen": -227.85494995117188,
      "logps/ref_chosen": -64.98896026611328,
      "logps/ref_rejected": -84.39607238769531,
      "logps/rejected": -291.7728576660156,
      "loss": 1.2396,
      "margin_dpo/margin_mean": 44.51079559326172,
      "margin_dpo/margin_std": 109.54621124267578,
      "step": 489
    },
    {
      "KL/chosen_KL_mean": -146.1357879638672,
      "KL/mean": -173.29544067382812,
      "KL/rejected_KL_mean": -200.4551239013672,
      "KL/std": 98.16156005859375,
      "epoch": 0.7407407407407407,
      "fcm_dpo/beta": 0.006055292207747698,
      "fcm_dpo/delta": 0.07335179299116135,
      "fcm_dpo/margin": 54.3193359375,
      "fcm_dpo/q_t": 0.42709293961524963,
      "grad_norm": 12.564495086669922,
      "learning_rate": 9.650174444319956e-08,
      "logits/chosen": 0.7777169346809387,
      "logits/rejected": 0.7545493841171265,
      "logps/chosen": -208.04452514648438,
      "logps/ref_chosen": -61.90874481201172,
      "logps/ref_rejected": -70.58566284179688,
      "logps/rejected": -271.040771484375,
      "loss": 1.2209,
      "margin_dpo/margin_mean": 54.3193359375,
      "margin_dpo/margin_std": 125.59601593017578,
      "step": 490
    },
    {
      "KL/chosen_KL_mean": -142.18975830078125,
      "KL/mean": -171.84603881835938,
      "KL/rejected_KL_mean": -201.5023193359375,
      "KL/std": 91.03143310546875,
      "epoch": 0.7422524565381708,
      "fcm_dpo/beta": 0.0061044651083648205,
      "fcm_dpo/delta": 0.0387558713555336,
      "fcm_dpo/margin": 59.31254196166992,
      "fcm_dpo/q_t": 0.4179537296295166,
      "grad_norm": 13.25456428527832,
      "learning_rate": 9.546025344484868e-08,
      "logits/chosen": 0.6695621013641357,
      "logits/rejected": 0.6074869632720947,
      "logps/chosen": -197.66546630859375,
      "logps/ref_chosen": -55.47570037841797,
      "logps/ref_rejected": -78.70318603515625,
      "logps/rejected": -280.20550537109375,
      "loss": 1.139,
      "margin_dpo/margin_mean": 59.31254577636719,
      "margin_dpo/margin_std": 94.94379425048828,
      "step": 491
    },
    {
      "KL/chosen_KL_mean": -166.24014282226562,
      "KL/mean": -191.16517639160156,
      "KL/rejected_KL_mean": -216.0902099609375,
      "KL/std": 98.4825439453125,
      "epoch": 0.7437641723356009,
      "fcm_dpo/beta": 0.00615697493776679,
      "fcm_dpo/delta": 0.0017743089701980352,
      "fcm_dpo/margin": 49.85006332397461,
      "fcm_dpo/q_t": 0.43032699823379517,
      "grad_norm": 15.069820404052734,
      "learning_rate": 9.442308525541589e-08,
      "logits/chosen": 0.7003054618835449,
      "logits/rejected": 0.6262869834899902,
      "logps/chosen": -233.52651977539062,
      "logps/ref_chosen": -67.28638458251953,
      "logps/ref_rejected": -82.78628540039062,
      "logps/rejected": -298.8764953613281,
      "loss": 1.2179,
      "margin_dpo/margin_mean": 49.85006332397461,
      "margin_dpo/margin_std": 110.67848205566406,
      "step": 492
    },
    {
      "KL/chosen_KL_mean": -137.14723205566406,
      "KL/mean": -174.77703857421875,
      "KL/rejected_KL_mean": -212.40684509277344,
      "KL/std": 95.96461486816406,
      "epoch": 0.745275888133031,
      "fcm_dpo/beta": 0.006142089609056711,
      "fcm_dpo/delta": -0.06584354490041733,
      "fcm_dpo/margin": 75.25961303710938,
      "fcm_dpo/q_t": 0.39572659134864807,
      "grad_norm": 14.114909172058105,
      "learning_rate": 9.339026888672468e-08,
      "logits/chosen": 0.6745371222496033,
      "logits/rejected": 0.593506932258606,
      "logps/chosen": -193.07473754882812,
      "logps/ref_chosen": -55.92750549316406,
      "logps/ref_rejected": -79.12149810791016,
      "logps/rejected": -291.5283203125,
      "loss": 1.0784,
      "margin_dpo/margin_mean": 75.25961303710938,
      "margin_dpo/margin_std": 104.96218872070312,
      "step": 493
    },
    {
      "KL/chosen_KL_mean": -142.44944763183594,
      "KL/mean": -174.2778778076172,
      "KL/rejected_KL_mean": -206.1063232421875,
      "KL/std": 96.92861938476562,
      "epoch": 0.7467876039304611,
      "fcm_dpo/beta": 0.006077418103814125,
      "fcm_dpo/delta": 0.013601415790617466,
      "fcm_dpo/margin": 63.656883239746094,
      "fcm_dpo/q_t": 0.4133981764316559,
      "grad_norm": 15.124746322631836,
      "learning_rate": 9.236183322886945e-08,
      "logits/chosen": 0.6335718631744385,
      "logits/rejected": 0.5776142477989197,
      "logps/chosen": -210.40354919433594,
      "logps/ref_chosen": -67.95410919189453,
      "logps/ref_rejected": -90.50865173339844,
      "logps/rejected": -296.614990234375,
      "loss": 1.157,
      "margin_dpo/margin_mean": 63.656883239746094,
      "margin_dpo/margin_std": 118.95513916015625,
      "step": 494
    },
    {
      "KL/chosen_KL_mean": -140.1368408203125,
      "KL/mean": -166.14944458007812,
      "KL/rejected_KL_mean": -192.16204833984375,
      "KL/std": 95.51089477539062,
      "epoch": 0.7482993197278912,
      "fcm_dpo/beta": 0.00619255006313324,
      "fcm_dpo/delta": 0.07974462956190109,
      "fcm_dpo/margin": 52.02519989013672,
      "fcm_dpo/q_t": 0.4316937029361725,
      "grad_norm": 17.694583892822266,
      "learning_rate": 9.133780704940594e-08,
      "logits/chosen": 0.7808051109313965,
      "logits/rejected": 0.7126421928405762,
      "logps/chosen": -192.76229858398438,
      "logps/ref_chosen": -52.62546157836914,
      "logps/ref_rejected": -72.06781005859375,
      "logps/rejected": -264.2298583984375,
      "loss": 1.2097,
      "margin_dpo/margin_mean": 52.02519607543945,
      "margin_dpo/margin_std": 117.4638671875,
      "step": 495
    },
    {
      "KL/chosen_KL_mean": -155.33493041992188,
      "KL/mean": -188.03475952148438,
      "KL/rejected_KL_mean": -220.73458862304688,
      "KL/std": 100.55799865722656,
      "epoch": 0.7498110355253212,
      "fcm_dpo/beta": 0.006147061474621296,
      "fcm_dpo/delta": -0.0030800998210906982,
      "fcm_dpo/margin": 65.39965057373047,
      "fcm_dpo/q_t": 0.41627591848373413,
      "grad_norm": 13.957164764404297,
      "learning_rate": 9.031821899254797e-08,
      "logits/chosen": 0.7075143456459045,
      "logits/rejected": 0.5884382724761963,
      "logps/chosen": -212.9322509765625,
      "logps/ref_chosen": -57.597320556640625,
      "logps/ref_rejected": -94.36127471923828,
      "logps/rejected": -315.09588623046875,
      "loss": 1.1605,
      "margin_dpo/margin_mean": 65.39965057373047,
      "margin_dpo/margin_std": 127.57322692871094,
      "step": 496
    },
    {
      "KL/chosen_KL_mean": -154.7255859375,
      "KL/mean": -191.44236755371094,
      "KL/rejected_KL_mean": -228.15916442871094,
      "KL/std": 96.41221618652344,
      "epoch": 0.7513227513227513,
      "fcm_dpo/beta": 0.006130448542535305,
      "fcm_dpo/delta": -0.05267590284347534,
      "fcm_dpo/margin": 73.43357849121094,
      "fcm_dpo/q_t": 0.3965461850166321,
      "grad_norm": 12.54123592376709,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": 0.7057574987411499,
      "logits/rejected": 0.6712203621864319,
      "logps/chosen": -227.51553344726562,
      "logps/ref_chosen": -72.78994750976562,
      "logps/ref_rejected": -89.48483276367188,
      "logps/rejected": -317.64398193359375,
      "loss": 1.0827,
      "margin_dpo/margin_mean": 73.43357849121094,
      "margin_dpo/margin_std": 104.81527709960938,
      "step": 497
    },
    {
      "KL/chosen_KL_mean": -138.21682739257812,
      "KL/mean": -174.46559143066406,
      "KL/rejected_KL_mean": -210.71435546875,
      "KL/std": 92.01454162597656,
      "epoch": 0.7528344671201814,
      "fcm_dpo/beta": 0.006081203930079937,
      "fcm_dpo/delta": -0.042786382138729095,
      "fcm_dpo/margin": 72.49751281738281,
      "fcm_dpo/q_t": 0.39820361137390137,
      "grad_norm": 15.794185638427734,
      "learning_rate": 8.829247120198563e-08,
      "logits/chosen": 0.6905786991119385,
      "logits/rejected": 0.6626016497612,
      "logps/chosen": -206.58255004882812,
      "logps/ref_chosen": -68.36572265625,
      "logps/ref_rejected": -71.28846740722656,
      "logps/rejected": -282.0028076171875,
      "loss": 1.0764,
      "margin_dpo/margin_mean": 72.49751281738281,
      "margin_dpo/margin_std": 98.33193969726562,
      "step": 498
    },
    {
      "KL/chosen_KL_mean": -138.8592529296875,
      "KL/mean": -174.84307861328125,
      "KL/rejected_KL_mean": -210.82687377929688,
      "KL/std": 97.4631576538086,
      "epoch": 0.7543461829176115,
      "fcm_dpo/beta": 0.00604314636439085,
      "fcm_dpo/delta": -0.036482226103544235,
      "fcm_dpo/margin": 71.96763610839844,
      "fcm_dpo/q_t": 0.4038216769695282,
      "grad_norm": 15.60657024383545,
      "learning_rate": 8.728636813280163e-08,
      "logits/chosen": 0.7012407779693604,
      "logits/rejected": 0.6313886046409607,
      "logps/chosen": -200.76808166503906,
      "logps/ref_chosen": -61.90882873535156,
      "logps/ref_rejected": -91.9411392211914,
      "logps/rejected": -302.76800537109375,
      "loss": 1.1363,
      "margin_dpo/margin_mean": 71.96763610839844,
      "margin_dpo/margin_std": 127.46099853515625,
      "step": 499
    },
    {
      "KL/chosen_KL_mean": -144.73477172851562,
      "KL/mean": -176.3829803466797,
      "KL/rejected_KL_mean": -208.03115844726562,
      "KL/std": 89.32550048828125,
      "epoch": 0.7558578987150416,
      "fcm_dpo/beta": 0.006025433540344238,
      "fcm_dpo/delta": 0.019208911806344986,
      "fcm_dpo/margin": 63.2963981628418,
      "fcm_dpo/q_t": 0.41165584325790405,
      "grad_norm": 16.173084259033203,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.6578192710876465,
      "logits/rejected": 0.6583301424980164,
      "logps/chosen": -214.96060180664062,
      "logps/ref_chosen": -70.225830078125,
      "logps/ref_rejected": -71.72203063964844,
      "logps/rejected": -279.7532043457031,
      "loss": 1.1635,
      "margin_dpo/margin_mean": 63.29639434814453,
      "margin_dpo/margin_std": 119.43624877929688,
      "step": 500
    },
    {
      "KL/chosen_KL_mean": -146.03167724609375,
      "KL/mean": -174.68145751953125,
      "KL/rejected_KL_mean": -203.3312225341797,
      "KL/std": 95.38584899902344,
      "epoch": 0.7573696145124716,
      "fcm_dpo/beta": 0.006113841198384762,
      "fcm_dpo/delta": 0.051176298409700394,
      "fcm_dpo/margin": 57.29954147338867,
      "fcm_dpo/q_t": 0.41882115602493286,
      "grad_norm": 12.610764503479004,
      "learning_rate": 8.528784436016878e-08,
      "logits/chosen": 0.696144700050354,
      "logits/rejected": 0.6981015205383301,
      "logps/chosen": -210.6304931640625,
      "logps/ref_chosen": -64.59880828857422,
      "logps/ref_rejected": -70.59329223632812,
      "logps/rejected": -273.92449951171875,
      "loss": 1.1234,
      "margin_dpo/margin_mean": 57.29954528808594,
      "margin_dpo/margin_std": 79.26424407958984,
      "step": 501
    },
    {
      "KL/chosen_KL_mean": -144.56405639648438,
      "KL/mean": -175.36734008789062,
      "KL/rejected_KL_mean": -206.17062377929688,
      "KL/std": 99.50743103027344,
      "epoch": 0.7588813303099018,
      "fcm_dpo/beta": 0.006148617714643478,
      "fcm_dpo/delta": 0.021840302273631096,
      "fcm_dpo/margin": 61.60658264160156,
      "fcm_dpo/q_t": 0.4134736657142639,
      "grad_norm": 15.2469482421875,
      "learning_rate": 8.4295479559726e-08,
      "logits/chosen": 0.7266432642936707,
      "logits/rejected": 0.6753150224685669,
      "logps/chosen": -210.03067016601562,
      "logps/ref_chosen": -65.46662902832031,
      "logps/ref_rejected": -90.22233581542969,
      "logps/rejected": -296.3929443359375,
      "loss": 1.1241,
      "margin_dpo/margin_mean": 61.60658264160156,
      "margin_dpo/margin_std": 95.19436645507812,
      "step": 502
    },
    {
      "KL/chosen_KL_mean": -131.74024963378906,
      "KL/mean": -163.972900390625,
      "KL/rejected_KL_mean": -196.2055206298828,
      "KL/std": 89.93467712402344,
      "epoch": 0.7603930461073318,
      "fcm_dpo/beta": 0.006136808544397354,
      "fcm_dpo/delta": 0.004550879821181297,
      "fcm_dpo/margin": 64.46528625488281,
      "fcm_dpo/q_t": 0.4090750217437744,
      "grad_norm": 12.482107162475586,
      "learning_rate": 8.330774987092712e-08,
      "logits/chosen": 0.7085367441177368,
      "logits/rejected": 0.7109423875808716,
      "logps/chosen": -183.57501220703125,
      "logps/ref_chosen": -51.83476257324219,
      "logps/ref_rejected": -57.62522506713867,
      "logps/rejected": -253.83074951171875,
      "loss": 1.1257,
      "margin_dpo/margin_mean": 64.46528625488281,
      "margin_dpo/margin_std": 103.35479736328125,
      "step": 503
    },
    {
      "KL/chosen_KL_mean": -134.92105102539062,
      "KL/mean": -177.01422119140625,
      "KL/rejected_KL_mean": -219.10740661621094,
      "KL/std": 88.60205841064453,
      "epoch": 0.7619047619047619,
      "fcm_dpo/beta": 0.006074085831642151,
      "fcm_dpo/delta": -0.1173659935593605,
      "fcm_dpo/margin": 84.18635559082031,
      "fcm_dpo/q_t": 0.3816917836666107,
      "grad_norm": 14.414610862731934,
      "learning_rate": 8.232468292269479e-08,
      "logits/chosen": 0.6959263682365417,
      "logits/rejected": 0.6736807227134705,
      "logps/chosen": -203.572265625,
      "logps/ref_chosen": -68.65119934082031,
      "logps/ref_rejected": -77.91394805908203,
      "logps/rejected": -297.0213623046875,
      "loss": 1.0129,
      "margin_dpo/margin_mean": 84.18635559082031,
      "margin_dpo/margin_std": 90.47264099121094,
      "step": 504
    },
    {
      "KL/chosen_KL_mean": -142.89309692382812,
      "KL/mean": -170.55242919921875,
      "KL/rejected_KL_mean": -198.2117462158203,
      "KL/std": 100.20260620117188,
      "epoch": 0.763416477702192,
      "fcm_dpo/beta": 0.005962444934993982,
      "fcm_dpo/delta": -0.033300042152404785,
      "fcm_dpo/margin": 55.31865310668945,
      "fcm_dpo/q_t": 0.4262439012527466,
      "grad_norm": 13.964221000671387,
      "learning_rate": 8.134630621352483e-08,
      "logits/chosen": 0.7036569118499756,
      "logits/rejected": 0.6635636687278748,
      "logps/chosen": -202.89195251464844,
      "logps/ref_chosen": -59.99884796142578,
      "logps/ref_rejected": -76.88048553466797,
      "logps/rejected": -275.09222412109375,
      "loss": 1.1953,
      "margin_dpo/margin_mean": 55.31865692138672,
      "margin_dpo/margin_std": 113.81221008300781,
      "step": 505
    },
    {
      "KL/chosen_KL_mean": -140.13351440429688,
      "KL/mean": -170.43092346191406,
      "KL/rejected_KL_mean": -200.7283477783203,
      "KL/std": 89.93391418457031,
      "epoch": 0.764928193499622,
      "fcm_dpo/beta": 0.0059835035353899,
      "fcm_dpo/delta": 0.03882179781794548,
      "fcm_dpo/margin": 60.59484100341797,
      "fcm_dpo/q_t": 0.4168153405189514,
      "grad_norm": 14.753436088562012,
      "learning_rate": 8.037264711071698e-08,
      "logits/chosen": 0.6791187524795532,
      "logits/rejected": 0.6618653535842896,
      "logps/chosen": -210.20480346679688,
      "logps/ref_chosen": -70.07130432128906,
      "logps/ref_rejected": -82.03775024414062,
      "logps/rejected": -282.76611328125,
      "loss": 1.1654,
      "margin_dpo/margin_mean": 60.594844818115234,
      "margin_dpo/margin_std": 113.21796417236328,
      "step": 506
    },
    {
      "KL/chosen_KL_mean": -153.470947265625,
      "KL/mean": -186.1775360107422,
      "KL/rejected_KL_mean": -218.8841552734375,
      "KL/std": 99.80972290039062,
      "epoch": 0.7664399092970522,
      "fcm_dpo/beta": 0.005990843288600445,
      "fcm_dpo/delta": 0.008119482547044754,
      "fcm_dpo/margin": 65.41321563720703,
      "fcm_dpo/q_t": 0.4153268337249756,
      "grad_norm": 14.057876586914062,
      "learning_rate": 7.940373284960933e-08,
      "logits/chosen": 0.7142482995986938,
      "logits/rejected": 0.6605731248855591,
      "logps/chosen": -225.47796630859375,
      "logps/ref_chosen": -72.00703430175781,
      "logps/ref_rejected": -93.94987487792969,
      "logps/rejected": -312.83404541015625,
      "loss": 1.1517,
      "margin_dpo/margin_mean": 65.41321563720703,
      "margin_dpo/margin_std": 118.75868225097656,
      "step": 507
    },
    {
      "KL/chosen_KL_mean": -140.153564453125,
      "KL/mean": -177.7708740234375,
      "KL/rejected_KL_mean": -215.38819885253906,
      "KL/std": 102.30191802978516,
      "epoch": 0.7679516250944822,
      "fcm_dpo/beta": 0.006008810829371214,
      "fcm_dpo/delta": -0.055044736713171005,
      "fcm_dpo/margin": 75.23462677001953,
      "fcm_dpo/q_t": 0.3986842930316925,
      "grad_norm": 16.15645408630371,
      "learning_rate": 7.843959053281663e-08,
      "logits/chosen": 0.648708701133728,
      "logits/rejected": 0.5197543501853943,
      "logps/chosen": -200.3734893798828,
      "logps/ref_chosen": -60.21992492675781,
      "logps/ref_rejected": -95.9200668334961,
      "logps/rejected": -311.30828857421875,
      "loss": 1.0918,
      "margin_dpo/margin_mean": 75.23462677001953,
      "margin_dpo/margin_std": 110.77383422851562,
      "step": 508
    },
    {
      "KL/chosen_KL_mean": -148.20401000976562,
      "KL/mean": -178.2255859375,
      "KL/rejected_KL_mean": -208.2471923828125,
      "KL/std": 92.92705535888672,
      "epoch": 0.7694633408919124,
      "fcm_dpo/beta": 0.005979306995868683,
      "fcm_dpo/delta": 0.04251670092344284,
      "fcm_dpo/margin": 60.04317855834961,
      "fcm_dpo/q_t": 0.4180784225463867,
      "grad_norm": 16.732942581176758,
      "learning_rate": 7.748024712947204e-08,
      "logits/chosen": 0.6488137245178223,
      "logits/rejected": 0.6248580813407898,
      "logps/chosen": -214.4741668701172,
      "logps/ref_chosen": -66.27017211914062,
      "logps/ref_rejected": -71.73065185546875,
      "logps/rejected": -279.97784423828125,
      "loss": 1.1551,
      "margin_dpo/margin_mean": 60.043182373046875,
      "margin_dpo/margin_std": 106.17031860351562,
      "step": 509
    },
    {
      "KL/chosen_KL_mean": -146.8265838623047,
      "KL/mean": -184.74188232421875,
      "KL/rejected_KL_mean": -222.65719604492188,
      "KL/std": 100.29558563232422,
      "epoch": 0.7709750566893424,
      "fcm_dpo/beta": 0.005928627215325832,
      "fcm_dpo/delta": -0.052614498883485794,
      "fcm_dpo/margin": 75.83062744140625,
      "fcm_dpo/q_t": 0.4021064341068268,
      "grad_norm": 14.241165161132812,
      "learning_rate": 7.652572947447272e-08,
      "logits/chosen": 0.7916622161865234,
      "logits/rejected": 0.6884767413139343,
      "logps/chosen": -200.3714599609375,
      "logps/ref_chosen": -53.54487609863281,
      "logps/ref_rejected": -91.36648559570312,
      "logps/rejected": -314.023681640625,
      "loss": 1.1155,
      "margin_dpo/margin_mean": 75.83061981201172,
      "margin_dpo/margin_std": 126.63345336914062,
      "step": 510
    },
    {
      "KL/chosen_KL_mean": -139.45733642578125,
      "KL/mean": -182.91036987304688,
      "KL/rejected_KL_mean": -226.36341857910156,
      "KL/std": 92.72401428222656,
      "epoch": 0.7724867724867724,
      "fcm_dpo/beta": 0.005851203575730324,
      "fcm_dpo/delta": -0.11418096721172333,
      "fcm_dpo/margin": 86.90606689453125,
      "fcm_dpo/q_t": 0.383342444896698,
      "grad_norm": 18.150293350219727,
      "learning_rate": 7.557606426772961e-08,
      "logits/chosen": 0.6983736753463745,
      "logits/rejected": 0.6386054754257202,
      "logps/chosen": -195.30172729492188,
      "logps/ref_chosen": -55.844383239746094,
      "logps/ref_rejected": -86.49819946289062,
      "logps/rejected": -312.86163330078125,
      "loss": 1.0275,
      "margin_dpo/margin_mean": 86.90606689453125,
      "margin_dpo/margin_std": 102.56002807617188,
      "step": 511
    },
    {
      "KL/chosen_KL_mean": -142.7176055908203,
      "KL/mean": -170.5921630859375,
      "KL/rejected_KL_mean": -198.46673583984375,
      "KL/std": 87.77848815917969,
      "epoch": 0.7739984882842026,
      "fcm_dpo/beta": 0.005858670920133591,
      "fcm_dpo/delta": 0.07592638581991196,
      "fcm_dpo/margin": 55.7491340637207,
      "fcm_dpo/q_t": 0.4245069622993469,
      "grad_norm": 19.153793334960938,
      "learning_rate": 7.463127807341966e-08,
      "logits/chosen": 0.5833500623703003,
      "logits/rejected": 0.5787808895111084,
      "logps/chosen": -204.37063598632812,
      "logps/ref_chosen": -61.653038024902344,
      "logps/ref_rejected": -72.83148193359375,
      "logps/rejected": -271.2982177734375,
      "loss": 1.184,
      "margin_dpo/margin_mean": 55.74913024902344,
      "margin_dpo/margin_std": 108.57861328125,
      "step": 512
    },
    {
      "KL/chosen_KL_mean": -127.30201721191406,
      "KL/mean": -163.33502197265625,
      "KL/rejected_KL_mean": -199.3680419921875,
      "KL/std": 89.71525573730469,
      "epoch": 0.7755102040816326,
      "fcm_dpo/beta": 0.005858708638697863,
      "fcm_dpo/delta": -0.02341538667678833,
      "fcm_dpo/margin": 72.06602478027344,
      "fcm_dpo/q_t": 0.4035298228263855,
      "grad_norm": 11.490484237670898,
      "learning_rate": 7.369139731924401e-08,
      "logits/chosen": 0.8692583441734314,
      "logits/rejected": 0.8085012435913086,
      "logps/chosen": -178.15457153320312,
      "logps/ref_chosen": -50.85256576538086,
      "logps/ref_rejected": -69.21754455566406,
      "logps/rejected": -268.5855712890625,
      "loss": 1.08,
      "margin_dpo/margin_mean": 72.06602478027344,
      "margin_dpo/margin_std": 94.39229583740234,
      "step": 513
    },
    {
      "KL/chosen_KL_mean": -143.91668701171875,
      "KL/mean": -183.69168090820312,
      "KL/rejected_KL_mean": -223.4666748046875,
      "KL/std": 97.1811294555664,
      "epoch": 0.7770219198790628,
      "fcm_dpo/beta": 0.005832049064338207,
      "fcm_dpo/delta": -0.06709263473749161,
      "fcm_dpo/margin": 79.54997253417969,
      "fcm_dpo/q_t": 0.3942224979400635,
      "grad_norm": 14.720767974853516,
      "learning_rate": 7.275644829568747e-08,
      "logits/chosen": 0.7055551409721375,
      "logits/rejected": 0.6707027554512024,
      "logps/chosen": -213.3016357421875,
      "logps/ref_chosen": -69.38493347167969,
      "logps/ref_rejected": -83.32447814941406,
      "logps/rejected": -306.7911376953125,
      "loss": 1.074,
      "margin_dpo/margin_mean": 79.54997253417969,
      "margin_dpo/margin_std": 111.23652648925781,
      "step": 514
    },
    {
      "KL/chosen_KL_mean": -152.89161682128906,
      "KL/mean": -183.92550659179688,
      "KL/rejected_KL_mean": -214.95941162109375,
      "KL/std": 92.05535888671875,
      "epoch": 0.7785336356764928,
      "fcm_dpo/beta": 0.005820984952151775,
      "fcm_dpo/delta": 0.04017217084765434,
      "fcm_dpo/margin": 62.06776809692383,
      "fcm_dpo/q_t": 0.4169883728027344,
      "grad_norm": 16.62370491027832,
      "learning_rate": 7.182645715528435e-08,
      "logits/chosen": 0.7016680240631104,
      "logits/rejected": 0.6188766956329346,
      "logps/chosen": -206.57864379882812,
      "logps/ref_chosen": -53.687034606933594,
      "logps/ref_rejected": -83.59614562988281,
      "logps/rejected": -298.5555419921875,
      "loss": 1.1502,
      "margin_dpo/margin_mean": 62.067771911621094,
      "margin_dpo/margin_std": 107.68792724609375,
      "step": 515
    },
    {
      "KL/chosen_KL_mean": -130.25364685058594,
      "KL/mean": -160.8660430908203,
      "KL/rejected_KL_mean": -191.47842407226562,
      "KL/std": 91.21895599365234,
      "epoch": 0.780045351473923,
      "fcm_dpo/beta": 0.005886279046535492,
      "fcm_dpo/delta": 0.04087837040424347,
      "fcm_dpo/margin": 61.22477722167969,
      "fcm_dpo/q_t": 0.4164145886898041,
      "grad_norm": 17.26055145263672,
      "learning_rate": 7.090144991188568e-08,
      "logits/chosen": 0.6850186586380005,
      "logits/rejected": 0.646237313747406,
      "logps/chosen": -187.15536499023438,
      "logps/ref_chosen": -56.9017219543457,
      "logps/ref_rejected": -67.83477783203125,
      "logps/rejected": -259.3132019042969,
      "loss": 1.1533,
      "margin_dpo/margin_mean": 61.22477722167969,
      "margin_dpo/margin_std": 107.42112731933594,
      "step": 516
    },
    {
      "KL/chosen_KL_mean": -157.5782012939453,
      "KL/mean": -178.9698486328125,
      "KL/rejected_KL_mean": -200.36146545410156,
      "KL/std": 95.36566925048828,
      "epoch": 0.781557067271353,
      "fcm_dpo/beta": 0.0059048025868833065,
      "fcm_dpo/delta": 0.0346204899251461,
      "fcm_dpo/margin": 42.78327178955078,
      "fcm_dpo/q_t": 0.44304513931274414,
      "grad_norm": 15.950164794921875,
      "learning_rate": 6.998145243993284e-08,
      "logits/chosen": 0.7405321598052979,
      "logits/rejected": 0.7390405535697937,
      "logps/chosen": -219.3533477783203,
      "logps/ref_chosen": -61.775142669677734,
      "logps/ref_rejected": -62.88270950317383,
      "logps/rejected": -263.2441711425781,
      "loss": 1.2353,
      "margin_dpo/margin_mean": 42.78327178955078,
      "margin_dpo/margin_std": 101.09457397460938,
      "step": 517
    },
    {
      "KL/chosen_KL_mean": -132.57608032226562,
      "KL/mean": -164.98403930664062,
      "KL/rejected_KL_mean": -197.3920135498047,
      "KL/std": 93.18022918701172,
      "epoch": 0.783068783068783,
      "fcm_dpo/beta": 0.005937398411333561,
      "fcm_dpo/delta": 0.015771884471178055,
      "fcm_dpo/margin": 64.81591033935547,
      "fcm_dpo/q_t": 0.4144596457481384,
      "grad_norm": 13.72231388092041,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": 0.7070802450180054,
      "logits/rejected": 0.6594283580780029,
      "logps/chosen": -194.601318359375,
      "logps/ref_chosen": -62.02523422241211,
      "logps/ref_rejected": -79.06085205078125,
      "logps/rejected": -276.452880859375,
      "loss": 1.1289,
      "margin_dpo/margin_mean": 64.81591033935547,
      "margin_dpo/margin_std": 105.89201354980469,
      "step": 518
    },
    {
      "KL/chosen_KL_mean": -162.1259002685547,
      "KL/mean": -181.00131225585938,
      "KL/rejected_KL_mean": -199.87669372558594,
      "KL/std": 95.811279296875,
      "epoch": 0.7845804988662132,
      "fcm_dpo/beta": 0.005983233917504549,
      "fcm_dpo/delta": 0.06563226133584976,
      "fcm_dpo/margin": 37.75080108642578,
      "fcm_dpo/q_t": 0.44908711314201355,
      "grad_norm": 21.459136962890625,
      "learning_rate": 6.815658960673781e-08,
      "logits/chosen": 0.7152000069618225,
      "logits/rejected": 0.6627354025840759,
      "logps/chosen": -223.73226928710938,
      "logps/ref_chosen": -61.60636901855469,
      "logps/ref_rejected": -74.50727844238281,
      "logps/rejected": -274.38397216796875,
      "loss": 1.3149,
      "margin_dpo/margin_mean": 37.75080108642578,
      "margin_dpo/margin_std": 129.23397827148438,
      "step": 519
    },
    {
      "KL/chosen_KL_mean": -147.61134338378906,
      "KL/mean": -175.1790771484375,
      "KL/rejected_KL_mean": -202.746826171875,
      "KL/std": 95.35908508300781,
      "epoch": 0.7860922146636432,
      "fcm_dpo/beta": 0.005995592102408409,
      "fcm_dpo/delta": -0.022391589358448982,
      "fcm_dpo/margin": 55.13550567626953,
      "fcm_dpo/q_t": 0.4252380132675171,
      "grad_norm": 14.73218059539795,
      "learning_rate": 6.725177529083209e-08,
      "logits/chosen": 0.782904863357544,
      "logits/rejected": 0.7218393683433533,
      "logps/chosen": -210.48477172851562,
      "logps/ref_chosen": -62.87343215942383,
      "logps/ref_rejected": -76.505615234375,
      "logps/rejected": -279.25244140625,
      "loss": 1.1675,
      "margin_dpo/margin_mean": 55.135501861572266,
      "margin_dpo/margin_std": 99.03907775878906,
      "step": 520
    },
    {
      "KL/chosen_KL_mean": -144.95758056640625,
      "KL/mean": -186.0238494873047,
      "KL/rejected_KL_mean": -227.09011840820312,
      "KL/std": 93.88986206054688,
      "epoch": 0.7876039304610734,
      "fcm_dpo/beta": 0.005928085185587406,
      "fcm_dpo/delta": -0.09123433381319046,
      "fcm_dpo/margin": 82.13256072998047,
      "fcm_dpo/q_t": 0.3894064724445343,
      "grad_norm": 12.339912414550781,
      "learning_rate": 6.63520728356167e-08,
      "logits/chosen": 0.6129434108734131,
      "logits/rejected": 0.5301312208175659,
      "logps/chosen": -209.16424560546875,
      "logps/ref_chosen": -64.20668029785156,
      "logps/ref_rejected": -92.28083038330078,
      "logps/rejected": -319.3709716796875,
      "loss": 1.0459,
      "margin_dpo/margin_mean": 82.13256072998047,
      "margin_dpo/margin_std": 104.75482177734375,
      "step": 521
    },
    {
      "KL/chosen_KL_mean": -146.2152099609375,
      "KL/mean": -171.54830932617188,
      "KL/rejected_KL_mean": -196.8814239501953,
      "KL/std": 95.36944580078125,
      "epoch": 0.7891156462585034,
      "fcm_dpo/beta": 0.005961663089692593,
      "fcm_dpo/delta": 0.10112152993679047,
      "fcm_dpo/margin": 50.66620635986328,
      "fcm_dpo/q_t": 0.43209362030029297,
      "grad_norm": 15.637158393859863,
      "learning_rate": 6.545750740770336e-08,
      "logits/chosen": 0.6734673380851746,
      "logits/rejected": 0.6656965017318726,
      "logps/chosen": -204.58493041992188,
      "logps/ref_chosen": -58.369720458984375,
      "logps/ref_rejected": -68.79248046875,
      "logps/rejected": -265.67388916015625,
      "loss": 1.2344,
      "margin_dpo/margin_mean": 50.66620635986328,
      "margin_dpo/margin_std": 123.20829772949219,
      "step": 522
    },
    {
      "KL/chosen_KL_mean": -150.59461975097656,
      "KL/mean": -180.4583740234375,
      "KL/rejected_KL_mean": -210.32211303710938,
      "KL/std": 94.50711059570312,
      "epoch": 0.7906273620559335,
      "fcm_dpo/beta": 0.006038610823452473,
      "fcm_dpo/delta": 0.04082069545984268,
      "fcm_dpo/margin": 59.72750473022461,
      "fcm_dpo/q_t": 0.41616764664649963,
      "grad_norm": 17.926328659057617,
      "learning_rate": 6.456810403001012e-08,
      "logits/chosen": 0.6982331275939941,
      "logits/rejected": 0.5693163871765137,
      "logps/chosen": -216.307861328125,
      "logps/ref_chosen": -65.71324157714844,
      "logps/ref_rejected": -91.98896789550781,
      "logps/rejected": -302.31109619140625,
      "loss": 1.1651,
      "margin_dpo/margin_mean": 59.72750473022461,
      "margin_dpo/margin_std": 111.73890686035156,
      "step": 523
    },
    {
      "KL/chosen_KL_mean": -129.25332641601562,
      "KL/mean": -161.02879333496094,
      "KL/rejected_KL_mean": -192.80429077148438,
      "KL/std": 91.20438385009766,
      "epoch": 0.7921390778533636,
      "fcm_dpo/beta": 0.006090350449085236,
      "fcm_dpo/delta": 0.013166261836886406,
      "fcm_dpo/margin": 63.550968170166016,
      "fcm_dpo/q_t": 0.41099852323532104,
      "grad_norm": 14.604881286621094,
      "learning_rate": 6.368388758106134e-08,
      "logits/chosen": 0.6385599374771118,
      "logits/rejected": 0.612282395362854,
      "logps/chosen": -205.6045684814453,
      "logps/ref_chosen": -76.35124969482422,
      "logps/ref_rejected": -89.96072387695312,
      "logps/rejected": -282.7650146484375,
      "loss": 1.1181,
      "margin_dpo/margin_mean": 63.55097198486328,
      "margin_dpo/margin_std": 96.34831237792969,
      "step": 524
    },
    {
      "KL/chosen_KL_mean": -145.90765380859375,
      "KL/mean": -172.04031372070312,
      "KL/rejected_KL_mean": -198.17300415039062,
      "KL/std": 92.273193359375,
      "epoch": 0.7936507936507936,
      "fcm_dpo/beta": 0.006146572530269623,
      "fcm_dpo/delta": 0.08136504143476486,
      "fcm_dpo/margin": 52.26536560058594,
      "fcm_dpo/q_t": 0.42859983444213867,
      "grad_norm": 18.74329376220703,
      "learning_rate": 6.280488279429185e-08,
      "logits/chosen": 0.5296494960784912,
      "logits/rejected": 0.5250794887542725,
      "logps/chosen": -221.4034423828125,
      "logps/ref_chosen": -75.49578857421875,
      "logps/ref_rejected": -84.04852294921875,
      "logps/rejected": -282.2215270996094,
      "loss": 1.1978,
      "margin_dpo/margin_mean": 52.26536560058594,
      "margin_dpo/margin_std": 110.63622283935547,
      "step": 525
    },
    {
      "KL/chosen_KL_mean": -155.30621337890625,
      "KL/mean": -178.01229858398438,
      "KL/rejected_KL_mean": -200.71835327148438,
      "KL/std": 93.90998840332031,
      "epoch": 0.7951625094482238,
      "fcm_dpo/beta": 0.0061467778868973255,
      "fcm_dpo/delta": -0.02072247304022312,
      "fcm_dpo/margin": 45.412132263183594,
      "fcm_dpo/q_t": 0.4357995390892029,
      "grad_norm": 15.068552017211914,
      "learning_rate": 6.193111425735515e-08,
      "logits/chosen": 0.7207432985305786,
      "logits/rejected": 0.6462384462356567,
      "logps/chosen": -216.5986328125,
      "logps/ref_chosen": -61.29241943359375,
      "logps/ref_rejected": -82.47763061523438,
      "logps/rejected": -283.19598388671875,
      "loss": 1.2208,
      "margin_dpo/margin_mean": 45.412132263183594,
      "margin_dpo/margin_std": 99.66590881347656,
      "step": 526
    },
    {
      "KL/chosen_KL_mean": -163.244873046875,
      "KL/mean": -184.04534912109375,
      "KL/rejected_KL_mean": -204.84579467773438,
      "KL/std": 94.02388000488281,
      "epoch": 0.7966742252456538,
      "fcm_dpo/beta": 0.006165428087115288,
      "fcm_dpo/delta": 0.03024955466389656,
      "fcm_dpo/margin": 41.60092544555664,
      "fcm_dpo/q_t": 0.4428751468658447,
      "grad_norm": 16.113893508911133,
      "learning_rate": 6.106260641143546e-08,
      "logits/chosen": 0.7807217836380005,
      "logits/rejected": 0.6930861473083496,
      "logps/chosen": -224.71749877929688,
      "logps/ref_chosen": -61.472625732421875,
      "logps/ref_rejected": -90.52831268310547,
      "logps/rejected": -295.3741149902344,
      "loss": 1.2541,
      "margin_dpo/margin_mean": 41.600921630859375,
      "margin_dpo/margin_std": 109.10176849365234,
      "step": 527
    },
    {
      "KL/chosen_KL_mean": -144.57591247558594,
      "KL/mean": -167.90296936035156,
      "KL/rejected_KL_mean": -191.2300262451172,
      "KL/std": 91.32858276367188,
      "epoch": 0.7981859410430839,
      "fcm_dpo/beta": 0.006270756013691425,
      "fcm_dpo/delta": 0.11083254963159561,
      "fcm_dpo/margin": 46.65412139892578,
      "fcm_dpo/q_t": 0.43469613790512085,
      "grad_norm": 17.359731674194336,
      "learning_rate": 6.019938355056422e-08,
      "logits/chosen": 0.6385272741317749,
      "logits/rejected": 0.5581063628196716,
      "logps/chosen": -203.367919921875,
      "logps/ref_chosen": -58.792015075683594,
      "logps/ref_rejected": -71.82516479492188,
      "logps/rejected": -263.05517578125,
      "loss": 1.2421,
      "margin_dpo/margin_mean": 46.65412139892578,
      "margin_dpo/margin_std": 116.03971862792969,
      "step": 528
    },
    {
      "KL/chosen_KL_mean": -135.99566650390625,
      "KL/mean": -183.17657470703125,
      "KL/rejected_KL_mean": -230.35748291015625,
      "KL/std": 90.82351684570312,
      "epoch": 0.799697656840514,
      "fcm_dpo/beta": 0.00614023394882679,
      "fcm_dpo/delta": -0.19065029919147491,
      "fcm_dpo/margin": 94.36182403564453,
      "fcm_dpo/q_t": 0.36667758226394653,
      "grad_norm": 16.461719512939453,
      "learning_rate": 5.934146982094049e-08,
      "logits/chosen": 0.6083083152770996,
      "logits/rejected": 0.5538345575332642,
      "logps/chosen": -191.06661987304688,
      "logps/ref_chosen": -55.070960998535156,
      "logps/ref_rejected": -75.44007873535156,
      "logps/rejected": -305.79754638671875,
      "loss": 0.973,
      "margin_dpo/margin_mean": 94.36182403564453,
      "margin_dpo/margin_std": 98.53756713867188,
      "step": 529
    },
    {
      "KL/chosen_KL_mean": -139.13638305664062,
      "KL/mean": -168.36148071289062,
      "KL/rejected_KL_mean": -197.58657836914062,
      "KL/std": 94.34196472167969,
      "epoch": 0.8012093726379441,
      "fcm_dpo/beta": 0.006126364227384329,
      "fcm_dpo/delta": 0.04346451163291931,
      "fcm_dpo/margin": 58.45021057128906,
      "fcm_dpo/q_t": 0.42002660036087036,
      "grad_norm": 18.474821090698242,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": 0.7120848298072815,
      "logits/rejected": 0.6645527482032776,
      "logps/chosen": -195.88018798828125,
      "logps/ref_chosen": -56.743812561035156,
      "logps/ref_rejected": -76.6692123413086,
      "logps/rejected": -274.25579833984375,
      "loss": 1.149,
      "margin_dpo/margin_mean": 58.45021057128906,
      "margin_dpo/margin_std": 99.46220397949219,
      "step": 530
    },
    {
      "KL/chosen_KL_mean": -140.04896545410156,
      "KL/mean": -170.93316650390625,
      "KL/rejected_KL_mean": -201.8173828125,
      "KL/std": 93.3708724975586,
      "epoch": 0.8027210884353742,
      "fcm_dpo/beta": 0.0061726756393909454,
      "fcm_dpo/delta": 0.019283978268504143,
      "fcm_dpo/margin": 61.76841735839844,
      "fcm_dpo/q_t": 0.41390424966812134,
      "grad_norm": 14.578618049621582,
      "learning_rate": 5.7641665597021435e-08,
      "logits/chosen": 0.6841608285903931,
      "logits/rejected": 0.6026010513305664,
      "logps/chosen": -191.16542053222656,
      "logps/ref_chosen": -51.116455078125,
      "logps/ref_rejected": -79.52884674072266,
      "logps/rejected": -281.3462219238281,
      "loss": 1.132,
      "margin_dpo/margin_mean": 61.76841735839844,
      "margin_dpo/margin_std": 100.71135711669922,
      "step": 531
    },
    {
      "KL/chosen_KL_mean": -160.59451293945312,
      "KL/mean": -193.44097900390625,
      "KL/rejected_KL_mean": -226.28744506835938,
      "KL/std": 92.31813049316406,
      "epoch": 0.8042328042328042,
      "fcm_dpo/beta": 0.006159262731671333,
      "fcm_dpo/delta": -0.004814588464796543,
      "fcm_dpo/margin": 65.69293212890625,
      "fcm_dpo/q_t": 0.4081898033618927,
      "grad_norm": 15.817337036132812,
      "learning_rate": 5.679982264990424e-08,
      "logits/chosen": 0.6366969347000122,
      "logits/rejected": 0.5827762484550476,
      "logps/chosen": -218.87445068359375,
      "logps/ref_chosen": -58.279945373535156,
      "logps/ref_rejected": -78.05426788330078,
      "logps/rejected": -304.3417053222656,
      "loss": 1.1197,
      "margin_dpo/margin_mean": 65.69293212890625,
      "margin_dpo/margin_std": 104.62611389160156,
      "step": 532
    },
    {
      "KL/chosen_KL_mean": -114.49920654296875,
      "KL/mean": -149.4227294921875,
      "KL/rejected_KL_mean": -184.3462371826172,
      "KL/std": 94.08676147460938,
      "epoch": 0.8057445200302343,
      "fcm_dpo/beta": 0.0061393016949296,
      "fcm_dpo/delta": -0.030106620863080025,
      "fcm_dpo/margin": 69.84703826904297,
      "fcm_dpo/q_t": 0.4025030732154846,
      "grad_norm": 15.609317779541016,
      "learning_rate": 5.596338392706076e-08,
      "logits/chosen": 0.7968940734863281,
      "logits/rejected": 0.7260788679122925,
      "logps/chosen": -170.91722106933594,
      "logps/ref_chosen": -56.41801071166992,
      "logps/ref_rejected": -73.89324951171875,
      "logps/rejected": -258.239501953125,
      "loss": 1.0933,
      "margin_dpo/margin_mean": 69.84703063964844,
      "margin_dpo/margin_std": 100.53176879882812,
      "step": 533
    },
    {
      "KL/chosen_KL_mean": -142.42276000976562,
      "KL/mean": -173.40286254882812,
      "KL/rejected_KL_mean": -204.3829803466797,
      "KL/std": 92.6493911743164,
      "epoch": 0.8072562358276644,
      "fcm_dpo/beta": 0.006117708049714565,
      "fcm_dpo/delta": 0.02158135361969471,
      "fcm_dpo/margin": 61.96025466918945,
      "fcm_dpo/q_t": 0.4157974123954773,
      "grad_norm": 14.248810768127441,
      "learning_rate": 5.513237282548033e-08,
      "logits/chosen": 0.6835423707962036,
      "logits/rejected": 0.6452208757400513,
      "logps/chosen": -203.17144775390625,
      "logps/ref_chosen": -60.748687744140625,
      "logps/ref_rejected": -73.8623046875,
      "logps/rejected": -278.24530029296875,
      "loss": 1.1557,
      "margin_dpo/margin_mean": 61.96025466918945,
      "margin_dpo/margin_std": 113.3079833984375,
      "step": 534
    },
    {
      "KL/chosen_KL_mean": -153.73721313476562,
      "KL/mean": -181.03717041015625,
      "KL/rejected_KL_mean": -208.3371124267578,
      "KL/std": 96.67320251464844,
      "epoch": 0.8087679516250945,
      "fcm_dpo/beta": 0.006213212385773659,
      "fcm_dpo/delta": 0.06276258826255798,
      "fcm_dpo/margin": 54.59989929199219,
      "fcm_dpo/q_t": 0.4240074157714844,
      "grad_norm": 16.0213565826416,
      "learning_rate": 5.430681259032957e-08,
      "logits/chosen": 0.5884385704994202,
      "logits/rejected": 0.5250898599624634,
      "logps/chosen": -215.3746337890625,
      "logps/ref_chosen": -61.637413024902344,
      "logps/ref_rejected": -80.93138885498047,
      "logps/rejected": -289.26849365234375,
      "loss": 1.1783,
      "margin_dpo/margin_mean": 54.59989929199219,
      "margin_dpo/margin_std": 105.70285034179688,
      "step": 535
    },
    {
      "KL/chosen_KL_mean": -132.82046508789062,
      "KL/mean": -176.50271606445312,
      "KL/rejected_KL_mean": -220.1849365234375,
      "KL/std": 98.14479064941406,
      "epoch": 0.8102796674225246,
      "fcm_dpo/beta": 0.006084546912461519,
      "fcm_dpo/delta": -0.13932110369205475,
      "fcm_dpo/margin": 87.3644790649414,
      "fcm_dpo/q_t": 0.3791520893573761,
      "grad_norm": 12.060877799987793,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": 0.756862461566925,
      "logits/rejected": 0.6645947694778442,
      "logps/chosen": -184.70944213867188,
      "logps/ref_chosen": -51.88897705078125,
      "logps/ref_rejected": -73.34864044189453,
      "logps/rejected": -293.5335693359375,
      "loss": 1.001,
      "margin_dpo/margin_mean": 87.3644790649414,
      "margin_dpo/margin_std": 95.73563385009766,
      "step": 536
    },
    {
      "KL/chosen_KL_mean": -151.81515502929688,
      "KL/mean": -186.49417114257812,
      "KL/rejected_KL_mean": -221.17320251464844,
      "KL/std": 101.94618225097656,
      "epoch": 0.8117913832199547,
      "fcm_dpo/beta": 0.006012958474457264,
      "fcm_dpo/delta": -0.018182016909122467,
      "fcm_dpo/margin": 69.3580322265625,
      "fcm_dpo/q_t": 0.40668776631355286,
      "grad_norm": 14.026582717895508,
      "learning_rate": 5.267213693697695e-08,
      "logits/chosen": 0.7806311249732971,
      "logits/rejected": 0.6831108331680298,
      "logps/chosen": -206.06378173828125,
      "logps/ref_chosen": -54.248619079589844,
      "logps/ref_rejected": -94.94343566894531,
      "logps/rejected": -316.11663818359375,
      "loss": 1.1146,
      "margin_dpo/margin_mean": 69.3580322265625,
      "margin_dpo/margin_std": 109.73101043701172,
      "step": 537
    },
    {
      "KL/chosen_KL_mean": -148.46551513671875,
      "KL/mean": -185.07318115234375,
      "KL/rejected_KL_mean": -221.68084716796875,
      "KL/std": 98.37266540527344,
      "epoch": 0.8133030990173847,
      "fcm_dpo/beta": 0.0060086022131145,
      "fcm_dpo/delta": -0.04177962988615036,
      "fcm_dpo/margin": 73.21534729003906,
      "fcm_dpo/q_t": 0.39960160851478577,
      "grad_norm": 13.386337280273438,
      "learning_rate": 5.1863067244167144e-08,
      "logits/chosen": 0.6958510279655457,
      "logits/rejected": 0.6673502326011658,
      "logps/chosen": -218.55905151367188,
      "logps/ref_chosen": -70.09353637695312,
      "logps/ref_rejected": -79.49833679199219,
      "logps/rejected": -301.17919921875,
      "loss": 1.0744,
      "margin_dpo/margin_mean": 73.21534729003906,
      "margin_dpo/margin_std": 98.02046203613281,
      "step": 538
    },
    {
      "KL/chosen_KL_mean": -158.19003295898438,
      "KL/mean": -186.1972198486328,
      "KL/rejected_KL_mean": -214.20443725585938,
      "KL/std": 93.99584197998047,
      "epoch": 0.8148148148148148,
      "fcm_dpo/beta": 0.0060254549607634544,
      "fcm_dpo/delta": 0.06471256166696548,
      "fcm_dpo/margin": 56.014408111572266,
      "fcm_dpo/q_t": 0.42504042387008667,
      "grad_norm": 15.207216262817383,
      "learning_rate": 5.105953986729195e-08,
      "logits/chosen": 0.6740202903747559,
      "logits/rejected": 0.5857997536659241,
      "logps/chosen": -220.12171936035156,
      "logps/ref_chosen": -61.93169403076172,
      "logps/ref_rejected": -84.08946228027344,
      "logps/rejected": -298.29388427734375,
      "loss": 1.1614,
      "margin_dpo/margin_mean": 56.01441192626953,
      "margin_dpo/margin_std": 100.07429504394531,
      "step": 539
    },
    {
      "KL/chosen_KL_mean": -143.1622314453125,
      "KL/mean": -185.54034423828125,
      "KL/rejected_KL_mean": -227.91848754882812,
      "KL/std": 105.14231872558594,
      "epoch": 0.8163265306122449,
      "fcm_dpo/beta": 0.005985685158520937,
      "fcm_dpo/delta": -0.11297339200973511,
      "fcm_dpo/margin": 84.75627136230469,
      "fcm_dpo/q_t": 0.38413751125335693,
      "grad_norm": 12.883346557617188,
      "learning_rate": 5.026157728273966e-08,
      "logits/chosen": 0.767681360244751,
      "logits/rejected": 0.6636344194412231,
      "logps/chosen": -205.86647033691406,
      "logps/ref_chosen": -62.704254150390625,
      "logps/ref_rejected": -95.63597106933594,
      "logps/rejected": -323.554443359375,
      "loss": 1.0263,
      "margin_dpo/margin_mean": 84.75627136230469,
      "margin_dpo/margin_std": 99.300537109375,
      "step": 540
    },
    {
      "KL/chosen_KL_mean": -140.2425994873047,
      "KL/mean": -176.7427215576172,
      "KL/rejected_KL_mean": -213.24285888671875,
      "KL/std": 95.12239074707031,
      "epoch": 0.817838246409675,
      "fcm_dpo/beta": 0.005870661698281765,
      "fcm_dpo/delta": -0.030379291623830795,
      "fcm_dpo/margin": 73.00025939941406,
      "fcm_dpo/q_t": 0.4007849395275116,
      "grad_norm": 12.794107437133789,
      "learning_rate": 4.9469201811239035e-08,
      "logits/chosen": 0.747472882270813,
      "logits/rejected": 0.7737694382667542,
      "logps/chosen": -202.72344970703125,
      "logps/ref_chosen": -62.48084259033203,
      "logps/ref_rejected": -57.55541229248047,
      "logps/rejected": -270.79827880859375,
      "loss": 1.0775,
      "margin_dpo/margin_mean": 73.00025939941406,
      "margin_dpo/margin_std": 94.49057006835938,
      "step": 541
    },
    {
      "KL/chosen_KL_mean": -122.75123596191406,
      "KL/mean": -163.3633270263672,
      "KL/rejected_KL_mean": -203.9754180908203,
      "KL/std": 92.215576171875,
      "epoch": 0.8193499622071051,
      "fcm_dpo/beta": 0.005811762064695358,
      "fcm_dpo/delta": -0.07581393420696259,
      "fcm_dpo/margin": 81.22418212890625,
      "fcm_dpo/q_t": 0.3921007513999939,
      "grad_norm": 13.943346977233887,
      "learning_rate": 4.868243561723534e-08,
      "logits/chosen": 0.7975116968154907,
      "logits/rejected": 0.7448440194129944,
      "logps/chosen": -172.2061309814453,
      "logps/ref_chosen": -49.454891204833984,
      "logps/ref_rejected": -65.33275604248047,
      "logps/rejected": -269.30816650390625,
      "loss": 1.0688,
      "margin_dpo/margin_mean": 81.22417449951172,
      "margin_dpo/margin_std": 112.6666488647461,
      "step": 542
    },
    {
      "KL/chosen_KL_mean": -131.66566467285156,
      "KL/mean": -170.76913452148438,
      "KL/rejected_KL_mean": -209.87257385253906,
      "KL/std": 92.56333923339844,
      "epoch": 0.8208616780045351,
      "fcm_dpo/beta": 0.005771012045443058,
      "fcm_dpo/delta": -0.05379205569624901,
      "fcm_dpo/margin": 78.20692443847656,
      "fcm_dpo/q_t": 0.3958283066749573,
      "grad_norm": 11.740777015686035,
      "learning_rate": 4.790130070827028e-08,
      "logits/chosen": 0.7076966762542725,
      "logits/rejected": 0.616827130317688,
      "logps/chosen": -182.7665252685547,
      "logps/ref_chosen": -51.100860595703125,
      "logps/ref_rejected": -76.06130981445312,
      "logps/rejected": -285.93389892578125,
      "loss": 1.0689,
      "margin_dpo/margin_mean": 78.20692443847656,
      "margin_dpo/margin_std": 101.98219299316406,
      "step": 543
    },
    {
      "KL/chosen_KL_mean": -143.2991180419922,
      "KL/mean": -186.08511352539062,
      "KL/rejected_KL_mean": -228.87106323242188,
      "KL/std": 100.58622741699219,
      "epoch": 0.8223733938019653,
      "fcm_dpo/beta": 0.005664612166583538,
      "fcm_dpo/delta": -0.08900754153728485,
      "fcm_dpo/margin": 85.57198333740234,
      "fcm_dpo/q_t": 0.39036205410957336,
      "grad_norm": 15.742673873901367,
      "learning_rate": 4.7125818934366454e-08,
      "logits/chosen": 0.7208126187324524,
      "logits/rejected": 0.6367508769035339,
      "logps/chosen": -203.57635498046875,
      "logps/ref_chosen": -60.2772331237793,
      "logps/ref_rejected": -88.40553283691406,
      "logps/rejected": -317.276611328125,
      "loss": 1.06,
      "margin_dpo/margin_mean": 85.57197570800781,
      "margin_dpo/margin_std": 116.65727233886719,
      "step": 544
    },
    {
      "KL/chosen_KL_mean": -154.10137939453125,
      "KL/mean": -179.682373046875,
      "KL/rejected_KL_mean": -205.26336669921875,
      "KL/std": 94.35951232910156,
      "epoch": 0.8238851095993953,
      "fcm_dpo/beta": 0.005731325596570969,
      "fcm_dpo/delta": 0.10995464026927948,
      "fcm_dpo/margin": 51.16197967529297,
      "fcm_dpo/q_t": 0.43320369720458984,
      "grad_norm": 14.339609146118164,
      "learning_rate": 4.635601198741607e-08,
      "logits/chosen": 0.6638723611831665,
      "logits/rejected": 0.603476881980896,
      "logps/chosen": -215.7166290283203,
      "logps/ref_chosen": -61.61524963378906,
      "logps/ref_rejected": -78.71266174316406,
      "logps/rejected": -283.97601318359375,
      "loss": 1.201,
      "margin_dpo/margin_mean": 51.16197967529297,
      "margin_dpo/margin_std": 104.99940490722656,
      "step": 545
    },
    {
      "KL/chosen_KL_mean": -147.88327026367188,
      "KL/mean": -176.1870880126953,
      "KL/rejected_KL_mean": -204.4909210205078,
      "KL/std": 91.73049926757812,
      "epoch": 0.8253968253968254,
      "fcm_dpo/beta": 0.00581570016220212,
      "fcm_dpo/delta": 0.0732608512043953,
      "fcm_dpo/margin": 56.60765075683594,
      "fcm_dpo/q_t": 0.4246191382408142,
      "grad_norm": 16.442094802856445,
      "learning_rate": 4.559190140057428e-08,
      "logits/chosen": 0.799730122089386,
      "logits/rejected": 0.791517436504364,
      "logps/chosen": -207.196533203125,
      "logps/ref_chosen": -59.313262939453125,
      "logps/ref_rejected": -64.73631286621094,
      "logps/rejected": -269.22723388671875,
      "loss": 1.187,
      "margin_dpo/margin_mean": 56.60765075683594,
      "margin_dpo/margin_std": 113.25538635253906,
      "step": 546
    },
    {
      "KL/chosen_KL_mean": -129.1564178466797,
      "KL/mean": -169.7764892578125,
      "KL/rejected_KL_mean": -210.39654541015625,
      "KL/std": 95.73387145996094,
      "epoch": 0.8269085411942555,
      "fcm_dpo/beta": 0.005766263697296381,
      "fcm_dpo/delta": -0.07221996039152145,
      "fcm_dpo/margin": 81.24012756347656,
      "fcm_dpo/q_t": 0.39244258403778076,
      "grad_norm": 13.574936866760254,
      "learning_rate": 4.483350854765672e-08,
      "logits/chosen": 0.6180684566497803,
      "logits/rejected": 0.5523202419281006,
      "logps/chosen": -184.13316345214844,
      "logps/ref_chosen": -54.97674560546875,
      "logps/ref_rejected": -75.35922241210938,
      "logps/rejected": -285.7557678222656,
      "loss": 1.0638,
      "margin_dpo/margin_mean": 81.24012756347656,
      "margin_dpo/margin_std": 109.19973754882812,
      "step": 547
    },
    {
      "KL/chosen_KL_mean": -150.80990600585938,
      "KL/mean": -176.61753845214844,
      "KL/rejected_KL_mean": -202.4251708984375,
      "KL/std": 95.62950897216797,
      "epoch": 0.8284202569916855,
      "fcm_dpo/beta": 0.005864979233592749,
      "fcm_dpo/delta": 0.09988602250814438,
      "fcm_dpo/margin": 51.615257263183594,
      "fcm_dpo/q_t": 0.43169891834259033,
      "grad_norm": 16.31439781188965,
      "learning_rate": 4.4080854642541826e-08,
      "logits/chosen": 0.6168273687362671,
      "logits/rejected": 0.5545735359191895,
      "logps/chosen": -214.02056884765625,
      "logps/ref_chosen": -63.21067428588867,
      "logps/ref_rejected": -81.23347473144531,
      "logps/rejected": -283.65863037109375,
      "loss": 1.1924,
      "margin_dpo/margin_mean": 51.615264892578125,
      "margin_dpo/margin_std": 102.42247009277344,
      "step": 548
    },
    {
      "KL/chosen_KL_mean": -147.26806640625,
      "KL/mean": -178.75881958007812,
      "KL/rejected_KL_mean": -210.2495880126953,
      "KL/std": 100.26600646972656,
      "epoch": 0.8299319727891157,
      "fcm_dpo/beta": 0.005888701416552067,
      "fcm_dpo/delta": 0.03016788512468338,
      "fcm_dpo/margin": 62.98152160644531,
      "fcm_dpo/q_t": 0.41702839732170105,
      "grad_norm": 16.012353897094727,
      "learning_rate": 4.333396073857723e-08,
      "logits/chosen": 0.8080065250396729,
      "logits/rejected": 0.7336448431015015,
      "logps/chosen": -211.54156494140625,
      "logps/ref_chosen": -64.27351379394531,
      "logps/ref_rejected": -92.31663513183594,
      "logps/rejected": -302.56622314453125,
      "loss": 1.1638,
      "margin_dpo/margin_mean": 62.981529235839844,
      "margin_dpo/margin_std": 118.73297882080078,
      "step": 549
    },
    {
      "KL/chosen_KL_mean": -156.357177734375,
      "KL/mean": -177.01620483398438,
      "KL/rejected_KL_mean": -197.6752471923828,
      "KL/std": 91.97258758544922,
      "epoch": 0.8314436885865457,
      "fcm_dpo/beta": 0.0059481412172317505,
      "fcm_dpo/delta": 0.028638044372200966,
      "fcm_dpo/margin": 41.31805419921875,
      "fcm_dpo/q_t": 0.4438709020614624,
      "grad_norm": 17.320735931396484,
      "learning_rate": 4.259284772799099e-08,
      "logits/chosen": 0.7289705872535706,
      "logits/rejected": 0.6985296010971069,
      "logps/chosen": -212.58761596679688,
      "logps/ref_chosen": -56.230438232421875,
      "logps/ref_rejected": -62.59788513183594,
      "logps/rejected": -260.27313232421875,
      "loss": 1.2429,
      "margin_dpo/margin_mean": 41.31805419921875,
      "margin_dpo/margin_std": 100.11188507080078,
      "step": 550
    },
    {
      "KL/chosen_KL_mean": -157.56686401367188,
      "KL/mean": -184.38162231445312,
      "KL/rejected_KL_mean": -211.19638061523438,
      "KL/std": 98.75320434570312,
      "epoch": 0.8329554043839759,
      "fcm_dpo/beta": 0.006031910888850689,
      "fcm_dpo/delta": 0.07879273593425751,
      "fcm_dpo/margin": 53.629493713378906,
      "fcm_dpo/q_t": 0.42806869745254517,
      "grad_norm": 14.624547004699707,
      "learning_rate": 4.1857536341307176e-08,
      "logits/chosen": 0.754467248916626,
      "logits/rejected": 0.7172669172286987,
      "logps/chosen": -225.31407165527344,
      "logps/ref_chosen": -67.74720764160156,
      "logps/ref_rejected": -87.04285430908203,
      "logps/rejected": -298.2392272949219,
      "loss": 1.1645,
      "margin_dpo/margin_mean": 53.629493713378906,
      "margin_dpo/margin_std": 95.92138671875,
      "step": 551
    },
    {
      "KL/chosen_KL_mean": -146.32579040527344,
      "KL/mean": -177.65493774414062,
      "KL/rejected_KL_mean": -208.98410034179688,
      "KL/std": 98.06755828857422,
      "epoch": 0.8344671201814059,
      "fcm_dpo/beta": 0.006087047979235649,
      "fcm_dpo/delta": 0.01883266121149063,
      "fcm_dpo/margin": 62.6583251953125,
      "fcm_dpo/q_t": 0.41055458784103394,
      "grad_norm": 15.157283782958984,
      "learning_rate": 4.112804714676593e-08,
      "logits/chosen": 0.6982611417770386,
      "logits/rejected": 0.641166090965271,
      "logps/chosen": -209.25204467773438,
      "logps/ref_chosen": -62.92625427246094,
      "logps/ref_rejected": -82.98365783691406,
      "logps/rejected": -291.9677734375,
      "loss": 1.1185,
      "margin_dpo/margin_mean": 62.658329010009766,
      "margin_dpo/margin_std": 93.13668060302734,
      "step": 552
    },
    {
      "KL/chosen_KL_mean": -157.8448028564453,
      "KL/mean": -188.88116455078125,
      "KL/rejected_KL_mean": -219.91757202148438,
      "KL/std": 94.49806213378906,
      "epoch": 0.8359788359788359,
      "fcm_dpo/beta": 0.0060878656804561615,
      "fcm_dpo/delta": 0.022969983518123627,
      "fcm_dpo/margin": 62.072757720947266,
      "fcm_dpo/q_t": 0.41634491086006165,
      "grad_norm": 16.90045928955078,
      "learning_rate": 4.0404400549748144e-08,
      "logits/chosen": 0.6653603315353394,
      "logits/rejected": 0.5574727058410645,
      "logps/chosen": -213.88330078125,
      "logps/ref_chosen": -56.038490295410156,
      "logps/ref_rejected": -84.48454284667969,
      "logps/rejected": -304.402099609375,
      "loss": 1.1546,
      "margin_dpo/margin_mean": 62.07276153564453,
      "margin_dpo/margin_std": 114.04953002929688,
      "step": 553
    },
    {
      "KL/chosen_KL_mean": -141.66587829589844,
      "KL/mean": -178.45956420898438,
      "KL/rejected_KL_mean": -215.25323486328125,
      "KL/std": 93.54683685302734,
      "epoch": 0.8374905517762661,
      "fcm_dpo/beta": 0.006051028147339821,
      "fcm_dpo/delta": -0.047385621815919876,
      "fcm_dpo/margin": 73.58735656738281,
      "fcm_dpo/q_t": 0.3975900411605835,
      "grad_norm": 13.925719261169434,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 0.6994329690933228,
      "logits/rejected": 0.6819012761116028,
      "logps/chosen": -206.19647216796875,
      "logps/ref_chosen": -64.53059387207031,
      "logps/ref_rejected": -71.2155990600586,
      "logps/rejected": -286.4688415527344,
      "loss": 1.0809,
      "margin_dpo/margin_mean": 73.58736419677734,
      "margin_dpo/margin_std": 102.7331771850586,
      "step": 554
    },
    {
      "KL/chosen_KL_mean": -155.81317138671875,
      "KL/mean": -185.680908203125,
      "KL/rejected_KL_mean": -215.54864501953125,
      "KL/std": 93.25511169433594,
      "epoch": 0.8390022675736961,
      "fcm_dpo/beta": 0.006112195551395416,
      "fcm_dpo/delta": 0.03484828397631645,
      "fcm_dpo/margin": 59.7354736328125,
      "fcm_dpo/q_t": 0.41532590985298157,
      "grad_norm": 15.105382919311523,
      "learning_rate": 3.89747159520904e-08,
      "logits/chosen": 0.6931326389312744,
      "logits/rejected": 0.6678953170776367,
      "logps/chosen": -222.465087890625,
      "logps/ref_chosen": -66.65191650390625,
      "logps/ref_rejected": -68.6667251586914,
      "logps/rejected": -284.2153625488281,
      "loss": 1.1716,
      "margin_dpo/margin_mean": 59.73548126220703,
      "margin_dpo/margin_std": 111.55022430419922,
      "step": 555
    },
    {
      "KL/chosen_KL_mean": -156.04660034179688,
      "KL/mean": -182.6348876953125,
      "KL/rejected_KL_mean": -209.22317504882812,
      "KL/std": 97.96969604492188,
      "epoch": 0.8405139833711263,
      "fcm_dpo/beta": 0.006134449504315853,
      "fcm_dpo/delta": 0.076349176466465,
      "fcm_dpo/margin": 53.17655944824219,
      "fcm_dpo/q_t": 0.42776405811309814,
      "grad_norm": 13.785261154174805,
      "learning_rate": 3.826871794280192e-08,
      "logits/chosen": 0.7225127816200256,
      "logits/rejected": 0.6713939905166626,
      "logps/chosen": -208.87896728515625,
      "logps/ref_chosen": -52.832366943359375,
      "logps/ref_rejected": -64.49044036865234,
      "logps/rejected": -273.713623046875,
      "loss": 1.2011,
      "margin_dpo/margin_mean": 53.17656707763672,
      "margin_dpo/margin_std": 112.81788635253906,
      "step": 556
    },
    {
      "KL/chosen_KL_mean": -152.7425537109375,
      "KL/mean": -193.62136840820312,
      "KL/rejected_KL_mean": -234.50018310546875,
      "KL/std": 95.47584533691406,
      "epoch": 0.8420256991685563,
      "fcm_dpo/beta": 0.006056217011064291,
      "fcm_dpo/delta": -0.10079901665449142,
      "fcm_dpo/margin": 81.75762176513672,
      "fcm_dpo/q_t": 0.38686493039131165,
      "grad_norm": 11.868200302124023,
      "learning_rate": 3.756864251262143e-08,
      "logits/chosen": 0.7944482564926147,
      "logits/rejected": 0.7143831849098206,
      "logps/chosen": -207.77853393554688,
      "logps/ref_chosen": -55.03598403930664,
      "logps/ref_rejected": -75.80644989013672,
      "logps/rejected": -310.306640625,
      "loss": 1.0292,
      "margin_dpo/margin_mean": 81.75762939453125,
      "margin_dpo/margin_std": 94.34634399414062,
      "step": 557
    },
    {
      "KL/chosen_KL_mean": -140.5734405517578,
      "KL/mean": -184.28152465820312,
      "KL/rejected_KL_mean": -227.9896240234375,
      "KL/std": 102.52485656738281,
      "epoch": 0.8435374149659864,
      "fcm_dpo/beta": 0.005914529785513878,
      "fcm_dpo/delta": -0.1242096945643425,
      "fcm_dpo/margin": 87.41616821289062,
      "fcm_dpo/q_t": 0.38339143991470337,
      "grad_norm": 11.268230438232422,
      "learning_rate": 3.687450924416341e-08,
      "logits/chosen": 0.7653758525848389,
      "logits/rejected": 0.7072293758392334,
      "logps/chosen": -203.79977416992188,
      "logps/ref_chosen": -63.226348876953125,
      "logps/ref_rejected": -91.46881866455078,
      "logps/rejected": -319.45843505859375,
      "loss": 1.0255,
      "margin_dpo/margin_mean": 87.41616821289062,
      "margin_dpo/margin_std": 106.01040649414062,
      "step": 558
    },
    {
      "KL/chosen_KL_mean": -144.50631713867188,
      "KL/mean": -182.7941436767578,
      "KL/rejected_KL_mean": -221.08197021484375,
      "KL/std": 99.27385711669922,
      "epoch": 0.8450491307634165,
      "fcm_dpo/beta": 0.005818785633891821,
      "fcm_dpo/delta": -0.048915110528469086,
      "fcm_dpo/margin": 76.57566833496094,
      "fcm_dpo/q_t": 0.40228039026260376,
      "grad_norm": 12.22198486328125,
      "learning_rate": 3.6186337553827743e-08,
      "logits/chosen": 0.6875864267349243,
      "logits/rejected": 0.6185659170150757,
      "logps/chosen": -206.0279541015625,
      "logps/ref_chosen": -61.521644592285156,
      "logps/ref_rejected": -82.83859252929688,
      "logps/rejected": -303.9205627441406,
      "loss": 1.1004,
      "margin_dpo/margin_mean": 76.57566833496094,
      "margin_dpo/margin_std": 118.04710388183594,
      "step": 559
    },
    {
      "KL/chosen_KL_mean": -158.76681518554688,
      "KL/mean": -192.46304321289062,
      "KL/rejected_KL_mean": -226.1592559814453,
      "KL/std": 97.42121124267578,
      "epoch": 0.8465608465608465,
      "fcm_dpo/beta": 0.005872940644621849,
      "fcm_dpo/delta": 0.003934595733880997,
      "fcm_dpo/margin": 67.39244079589844,
      "fcm_dpo/q_t": 0.4091563820838928,
      "grad_norm": 17.246580123901367,
      "learning_rate": 3.550414669125573e-08,
      "logits/chosen": 0.7269736528396606,
      "logits/rejected": 0.6843345165252686,
      "logps/chosen": -219.40805053710938,
      "logps/ref_chosen": -60.64122009277344,
      "logps/ref_rejected": -78.75474548339844,
      "logps/rejected": -304.91400146484375,
      "loss": 1.1045,
      "margin_dpo/margin_mean": 67.39244842529297,
      "margin_dpo/margin_std": 94.21781158447266,
      "step": 560
    },
    {
      "KL/chosen_KL_mean": -143.70648193359375,
      "KL/mean": -176.48004150390625,
      "KL/rejected_KL_mean": -209.2535858154297,
      "KL/std": 96.2630844116211,
      "epoch": 0.8480725623582767,
      "fcm_dpo/beta": 0.005853408016264439,
      "fcm_dpo/delta": 0.016958223655819893,
      "fcm_dpo/margin": 65.54713439941406,
      "fcm_dpo/q_t": 0.41396719217300415,
      "grad_norm": 14.105753898620605,
      "learning_rate": 3.482795573879241e-08,
      "logits/chosen": 0.6962438225746155,
      "logits/rejected": 0.6599966287612915,
      "logps/chosen": -206.205078125,
      "logps/ref_chosen": -62.49859619140625,
      "logps/ref_rejected": -78.72064208984375,
      "logps/rejected": -287.9742431640625,
      "loss": 1.1289,
      "margin_dpo/margin_mean": 65.54713439941406,
      "margin_dpo/margin_std": 105.32086181640625,
      "step": 561
    },
    {
      "KL/chosen_KL_mean": -153.3737030029297,
      "KL/mean": -195.14984130859375,
      "KL/rejected_KL_mean": -236.9259490966797,
      "KL/std": 107.42462158203125,
      "epoch": 0.8495842781557067,
      "fcm_dpo/beta": 0.005758739076554775,
      "fcm_dpo/delta": -0.08644125610589981,
      "fcm_dpo/margin": 83.55224609375,
      "fcm_dpo/q_t": 0.391997754573822,
      "grad_norm": 17.710552215576172,
      "learning_rate": 3.415778361095226e-08,
      "logits/chosen": 0.6928203105926514,
      "logits/rejected": 0.652666449546814,
      "logps/chosen": -228.1554412841797,
      "logps/ref_chosen": -74.78173828125,
      "logps/ref_rejected": -92.63499450683594,
      "logps/rejected": -329.5609436035156,
      "loss": 1.0525,
      "margin_dpo/margin_mean": 83.55224609375,
      "margin_dpo/margin_std": 107.55873107910156,
      "step": 562
    },
    {
      "KL/chosen_KL_mean": -129.497802734375,
      "KL/mean": -165.84393310546875,
      "KL/rejected_KL_mean": -202.19003295898438,
      "KL/std": 87.41785430908203,
      "epoch": 0.8510959939531368,
      "fcm_dpo/beta": 0.005751899443566799,
      "fcm_dpo/delta": -0.018905367702245712,
      "fcm_dpo/margin": 72.69223022460938,
      "fcm_dpo/q_t": 0.40398576855659485,
      "grad_norm": 19.509660720825195,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": 0.7855877876281738,
      "logits/rejected": 0.7325365543365479,
      "logps/chosen": -179.69631958007812,
      "logps/ref_chosen": -50.19850158691406,
      "logps/ref_rejected": -66.76687622070312,
      "logps/rejected": -268.9569091796875,
      "loss": 1.1107,
      "margin_dpo/margin_mean": 72.69223022460938,
      "margin_dpo/margin_std": 112.92652130126953,
      "step": 563
    },
    {
      "KL/chosen_KL_mean": -131.23855590820312,
      "KL/mean": -177.26199340820312,
      "KL/rejected_KL_mean": -223.28543090820312,
      "KL/std": 95.011962890625,
      "epoch": 0.8526077097505669,
      "fcm_dpo/beta": 0.005634985864162445,
      "fcm_dpo/delta": -0.12527057528495789,
      "fcm_dpo/margin": 92.04689025878906,
      "fcm_dpo/q_t": 0.38040876388549805,
      "grad_norm": 13.060510635375977,
      "learning_rate": 3.283557064487785e-08,
      "logits/chosen": 0.6486942172050476,
      "logits/rejected": 0.6173498630523682,
      "logps/chosen": -186.97940063476562,
      "logps/ref_chosen": -55.7408447265625,
      "logps/ref_rejected": -74.82323455810547,
      "logps/rejected": -298.1086730957031,
      "loss": 1.0291,
      "margin_dpo/margin_mean": 92.04689025878906,
      "margin_dpo/margin_std": 113.59528350830078,
      "step": 564
    },
    {
      "KL/chosen_KL_mean": -160.48941040039062,
      "KL/mean": -192.53794860839844,
      "KL/rejected_KL_mean": -224.58648681640625,
      "KL/std": 95.77909851074219,
      "epoch": 0.854119425547997,
      "fcm_dpo/beta": 0.005659112706780434,
      "fcm_dpo/delta": 0.03811845928430557,
      "fcm_dpo/margin": 64.09708404541016,
      "fcm_dpo/q_t": 0.4162023067474365,
      "grad_norm": 15.256197929382324,
      "learning_rate": 3.218356679178252e-08,
      "logits/chosen": 0.7203613519668579,
      "logits/rejected": 0.6639231443405151,
      "logps/chosen": -218.82679748535156,
      "logps/ref_chosen": -58.33738327026367,
      "logps/ref_rejected": -78.31776428222656,
      "logps/rejected": -302.90423583984375,
      "loss": 1.1286,
      "margin_dpo/margin_mean": 64.09708404541016,
      "margin_dpo/margin_std": 96.95319366455078,
      "step": 565
    },
    {
      "KL/chosen_KL_mean": -148.97732543945312,
      "KL/mean": -179.01806640625,
      "KL/rejected_KL_mean": -209.05882263183594,
      "KL/std": 99.58465576171875,
      "epoch": 0.8556311413454271,
      "fcm_dpo/beta": 0.005717899184674025,
      "fcm_dpo/delta": 0.05788592994213104,
      "fcm_dpo/margin": 60.08150100708008,
      "fcm_dpo/q_t": 0.42354559898376465,
      "grad_norm": 15.784143447875977,
      "learning_rate": 3.1537655732553764e-08,
      "logits/chosen": 0.7294129133224487,
      "logits/rejected": 0.7139770984649658,
      "logps/chosen": -220.20106506347656,
      "logps/ref_chosen": -71.22373962402344,
      "logps/ref_rejected": -71.11601257324219,
      "logps/rejected": -280.1748352050781,
      "loss": 1.1943,
      "margin_dpo/margin_mean": 60.08149719238281,
      "margin_dpo/margin_std": 126.13102722167969,
      "step": 566
    },
    {
      "KL/chosen_KL_mean": -143.3767852783203,
      "KL/mean": -181.28350830078125,
      "KL/rejected_KL_mean": -219.1902313232422,
      "KL/std": 95.1152572631836,
      "epoch": 0.8571428571428571,
      "fcm_dpo/beta": 0.00565255805850029,
      "fcm_dpo/delta": -0.0306740440428257,
      "fcm_dpo/margin": 75.81346130371094,
      "fcm_dpo/q_t": 0.4009702801704407,
      "grad_norm": 11.552154541015625,
      "learning_rate": 3.089785553471233e-08,
      "logits/chosen": 0.7283965349197388,
      "logits/rejected": 0.6318632364273071,
      "logps/chosen": -196.04605102539062,
      "logps/ref_chosen": -52.669273376464844,
      "logps/ref_rejected": -74.34785461425781,
      "logps/rejected": -293.5380859375,
      "loss": 1.0862,
      "margin_dpo/margin_mean": 75.8134536743164,
      "margin_dpo/margin_std": 103.04086303710938,
      "step": 567
    },
    {
      "KL/chosen_KL_mean": -130.6129150390625,
      "KL/mean": -177.60072326660156,
      "KL/rejected_KL_mean": -224.58853149414062,
      "KL/std": 100.37284851074219,
      "epoch": 0.8586545729402872,
      "fcm_dpo/beta": 0.005599203985184431,
      "fcm_dpo/delta": -0.13314224779605865,
      "fcm_dpo/margin": 93.97561645507812,
      "fcm_dpo/q_t": 0.3804738223552704,
      "grad_norm": 16.543067932128906,
      "learning_rate": 3.026418409484513e-08,
      "logits/chosen": 0.7331607341766357,
      "logits/rejected": 0.6472818851470947,
      "logps/chosen": -182.79090881347656,
      "logps/ref_chosen": -52.178001403808594,
      "logps/ref_rejected": -85.8277587890625,
      "logps/rejected": -310.4162902832031,
      "loss": 1.0064,
      "margin_dpo/margin_mean": 93.97561645507812,
      "margin_dpo/margin_std": 101.29707336425781,
      "step": 568
    },
    {
      "KL/chosen_KL_mean": -154.01739501953125,
      "KL/mean": -176.99273681640625,
      "KL/rejected_KL_mean": -199.96804809570312,
      "KL/std": 98.98895263671875,
      "epoch": 0.8601662887377173,
      "fcm_dpo/beta": 0.005527706816792488,
      "fcm_dpo/delta": 0.0005241321050561965,
      "fcm_dpo/margin": 45.95063781738281,
      "fcm_dpo/q_t": 0.44138121604919434,
      "grad_norm": 15.738585472106934,
      "learning_rate": 2.963665913810451e-08,
      "logits/chosen": 0.6584327220916748,
      "logits/rejected": 0.6224997043609619,
      "logps/chosen": -216.66665649414062,
      "logps/ref_chosen": -62.649261474609375,
      "logps/ref_rejected": -75.4298324584961,
      "logps/rejected": -275.39788818359375,
      "loss": 1.2256,
      "margin_dpo/margin_mean": 45.95063018798828,
      "margin_dpo/margin_std": 99.63298797607422,
      "step": 569
    },
    {
      "KL/chosen_KL_mean": -135.53909301757812,
      "KL/mean": -181.07962036132812,
      "KL/rejected_KL_mean": -226.62013244628906,
      "KL/std": 93.45390319824219,
      "epoch": 0.8616780045351474,
      "fcm_dpo/beta": 0.005470400210469961,
      "fcm_dpo/delta": -0.10338807851076126,
      "fcm_dpo/margin": 91.0810546875,
      "fcm_dpo/q_t": 0.3853939473628998,
      "grad_norm": 13.147180557250977,
      "learning_rate": 2.9015298217712453e-08,
      "logits/chosen": 0.6849209070205688,
      "logits/rejected": 0.5985517501831055,
      "logps/chosen": -185.58087158203125,
      "logps/ref_chosen": -50.04179382324219,
      "logps/ref_rejected": -78.27146911621094,
      "logps/rejected": -304.8916015625,
      "loss": 1.0267,
      "margin_dpo/margin_mean": 91.08104705810547,
      "margin_dpo/margin_std": 104.20199584960938,
      "step": 570
    },
    {
      "KL/chosen_KL_mean": -147.98048400878906,
      "KL/mean": -173.66644287109375,
      "KL/rejected_KL_mean": -199.3524169921875,
      "KL/std": 91.55722045898438,
      "epoch": 0.8631897203325775,
      "fcm_dpo/beta": 0.005426807329058647,
      "fcm_dpo/delta": 0.022530177608132362,
      "fcm_dpo/margin": 51.371917724609375,
      "fcm_dpo/q_t": 0.4348105192184448,
      "grad_norm": 13.500092506408691,
      "learning_rate": 2.840011871446962e-08,
      "logits/chosen": 0.7210831642150879,
      "logits/rejected": 0.6914381384849548,
      "logps/chosen": -201.63729858398438,
      "logps/ref_chosen": -53.65681457519531,
      "logps/ref_rejected": -66.13298034667969,
      "logps/rejected": -265.48541259765625,
      "loss": 1.2113,
      "margin_dpo/margin_mean": 51.371917724609375,
      "margin_dpo/margin_std": 107.35842895507812,
      "step": 571
    },
    {
      "KL/chosen_KL_mean": -147.66845703125,
      "KL/mean": -177.28985595703125,
      "KL/rejected_KL_mean": -206.91128540039062,
      "KL/std": 97.01789855957031,
      "epoch": 0.8647014361300076,
      "fcm_dpo/beta": 0.005522261373698711,
      "fcm_dpo/delta": 0.07467402517795563,
      "fcm_dpo/margin": 59.242828369140625,
      "fcm_dpo/q_t": 0.4242960214614868,
      "grad_norm": 13.124091148376465,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": 0.7015185356140137,
      "logits/rejected": 0.7461810111999512,
      "logps/chosen": -222.4863739013672,
      "logps/ref_chosen": -74.81792449951172,
      "logps/ref_rejected": -65.88681030273438,
      "logps/rejected": -272.798095703125,
      "loss": 1.1573,
      "margin_dpo/margin_mean": 59.242828369140625,
      "margin_dpo/margin_std": 98.15217590332031,
      "step": 572
    },
    {
      "KL/chosen_KL_mean": -161.7481689453125,
      "KL/mean": -194.27401733398438,
      "KL/rejected_KL_mean": -226.7998504638672,
      "KL/std": 103.3853530883789,
      "epoch": 0.8662131519274376,
      "fcm_dpo/beta": 0.0055539412423968315,
      "fcm_dpo/delta": 0.040158383548259735,
      "fcm_dpo/margin": 65.05167388916016,
      "fcm_dpo/q_t": 0.41949892044067383,
      "grad_norm": 15.037415504455566,
      "learning_rate": 2.718837261761528e-08,
      "logits/chosen": 0.6950033903121948,
      "logits/rejected": 0.6469535231590271,
      "logps/chosen": -230.47381591796875,
      "logps/ref_chosen": -68.72564697265625,
      "logps/ref_rejected": -88.16201782226562,
      "logps/rejected": -314.96185302734375,
      "loss": 1.1657,
      "margin_dpo/margin_mean": 65.05168151855469,
      "margin_dpo/margin_std": 123.31416320800781,
      "step": 573
    },
    {
      "KL/chosen_KL_mean": -144.3804931640625,
      "KL/mean": -185.24703979492188,
      "KL/rejected_KL_mean": -226.11361694335938,
      "KL/std": 94.29296112060547,
      "epoch": 0.8677248677248677,
      "fcm_dpo/beta": 0.005549177527427673,
      "fcm_dpo/delta": -0.056394994258880615,
      "fcm_dpo/margin": 81.73311614990234,
      "fcm_dpo/q_t": 0.39483287930488586,
      "grad_norm": 11.877057075500488,
      "learning_rate": 2.659183991914696e-08,
      "logits/chosen": 0.7845852971076965,
      "logits/rejected": 0.713538646697998,
      "logps/chosen": -200.6938934326172,
      "logps/ref_chosen": -56.31340026855469,
      "logps/ref_rejected": -83.91553497314453,
      "logps/rejected": -310.0291442871094,
      "loss": 1.0423,
      "margin_dpo/margin_mean": 81.73310852050781,
      "margin_dpo/margin_std": 88.43424987792969,
      "step": 574
    },
    {
      "KL/chosen_KL_mean": -148.43624877929688,
      "KL/mean": -178.46737670898438,
      "KL/rejected_KL_mean": -208.49853515625,
      "KL/std": 98.57743072509766,
      "epoch": 0.8692365835222978,
      "fcm_dpo/beta": 0.005458875559270382,
      "fcm_dpo/delta": -0.040230460464954376,
      "fcm_dpo/margin": 60.06227111816406,
      "fcm_dpo/q_t": 0.42702460289001465,
      "grad_norm": 13.435763359069824,
      "learning_rate": 2.600155642716606e-08,
      "logits/chosen": 0.7627922296524048,
      "logits/rejected": 0.6764528751373291,
      "logps/chosen": -213.0203857421875,
      "logps/ref_chosen": -64.5841293334961,
      "logps/ref_rejected": -93.47034454345703,
      "logps/rejected": -301.9688720703125,
      "loss": 1.1933,
      "margin_dpo/margin_mean": 60.06227493286133,
      "margin_dpo/margin_std": 120.7169189453125,
      "step": 575
    },
    {
      "KL/chosen_KL_mean": -131.35055541992188,
      "KL/mean": -173.777587890625,
      "KL/rejected_KL_mean": -216.20462036132812,
      "KL/std": 97.3672103881836,
      "epoch": 0.8707482993197279,
      "fcm_dpo/beta": 0.005376887507736683,
      "fcm_dpo/delta": -0.06009761244058609,
      "fcm_dpo/margin": 84.85408020019531,
      "fcm_dpo/q_t": 0.3950416147708893,
      "grad_norm": 13.129989624023438,
      "learning_rate": 2.5417538653170754e-08,
      "logits/chosen": 0.7097625732421875,
      "logits/rejected": 0.600039005279541,
      "logps/chosen": -184.63108825683594,
      "logps/ref_chosen": -53.28052520751953,
      "logps/ref_rejected": -84.2000503540039,
      "logps/rejected": -300.4046630859375,
      "loss": 1.0627,
      "margin_dpo/margin_mean": 84.85408782958984,
      "margin_dpo/margin_std": 107.62837982177734,
      "step": 576
    },
    {
      "KL/chosen_KL_mean": -147.17095947265625,
      "KL/mean": -175.99276733398438,
      "KL/rejected_KL_mean": -204.8145751953125,
      "KL/std": 96.7497787475586,
      "epoch": 0.872260015117158,
      "fcm_dpo/beta": 0.005462226457893848,
      "fcm_dpo/delta": 0.08794426172971725,
      "fcm_dpo/margin": 57.643611907958984,
      "fcm_dpo/q_t": 0.42714783549308777,
      "grad_norm": 13.27270221710205,
      "learning_rate": 2.4839802933393607e-08,
      "logits/chosen": 0.7166399955749512,
      "logits/rejected": 0.7043805122375488,
      "logps/chosen": -209.4956512451172,
      "logps/ref_chosen": -62.32468795776367,
      "logps/ref_rejected": -67.300537109375,
      "logps/rejected": -272.1151123046875,
      "loss": 1.1798,
      "margin_dpo/margin_mean": 57.643611907958984,
      "margin_dpo/margin_std": 106.11869812011719,
      "step": 577
    },
    {
      "KL/chosen_KL_mean": -139.62893676757812,
      "KL/mean": -167.9611358642578,
      "KL/rejected_KL_mean": -196.2933349609375,
      "KL/std": 99.08432006835938,
      "epoch": 0.873771730914588,
      "fcm_dpo/beta": 0.00555716548115015,
      "fcm_dpo/delta": 0.08793404698371887,
      "fcm_dpo/margin": 56.664405822753906,
      "fcm_dpo/q_t": 0.42954006791114807,
      "grad_norm": 14.082164764404297,
      "learning_rate": 2.4268365428344733e-08,
      "logits/chosen": 0.7510101795196533,
      "logits/rejected": 0.7321392297744751,
      "logps/chosen": -196.28451538085938,
      "logps/ref_chosen": -56.65557861328125,
      "logps/ref_rejected": -68.21835327148438,
      "logps/rejected": -264.5116882324219,
      "loss": 1.1913,
      "margin_dpo/margin_mean": 56.664405822753906,
      "margin_dpo/margin_std": 113.96426391601562,
      "step": 578
    },
    {
      "KL/chosen_KL_mean": -146.20864868164062,
      "KL/mean": -186.79234313964844,
      "KL/rejected_KL_mean": -227.3760528564453,
      "KL/std": 92.79012298583984,
      "epoch": 0.8752834467120182,
      "fcm_dpo/beta": 0.005536979530006647,
      "fcm_dpo/delta": -0.05184290185570717,
      "fcm_dpo/margin": 81.16737365722656,
      "fcm_dpo/q_t": 0.3948771059513092,
      "grad_norm": 13.722906112670898,
      "learning_rate": 2.3703242122359357e-08,
      "logits/chosen": 0.6737359166145325,
      "logits/rejected": 0.6492637395858765,
      "logps/chosen": -203.018310546875,
      "logps/ref_chosen": -56.809661865234375,
      "logps/ref_rejected": -68.09613037109375,
      "logps/rejected": -295.47216796875,
      "loss": 1.0524,
      "margin_dpo/margin_mean": 81.16737365722656,
      "margin_dpo/margin_std": 95.31343078613281,
      "step": 579
    },
    {
      "KL/chosen_KL_mean": -147.4833984375,
      "KL/mean": -181.59197998046875,
      "KL/rejected_KL_mean": -215.7005615234375,
      "KL/std": 105.0927734375,
      "epoch": 0.8767951625094482,
      "fcm_dpo/beta": 0.00556798093020916,
      "fcm_dpo/delta": 0.020568108186125755,
      "fcm_dpo/margin": 68.2171630859375,
      "fcm_dpo/q_t": 0.4152269959449768,
      "grad_norm": 13.51451301574707,
      "learning_rate": 2.3144448823151392e-08,
      "logits/chosen": 0.6773139238357544,
      "logits/rejected": 0.6209636926651001,
      "logps/chosen": -205.18353271484375,
      "logps/ref_chosen": -57.70011520385742,
      "logps/ref_rejected": -77.90664672851562,
      "logps/rejected": -293.6072082519531,
      "loss": 1.1451,
      "margin_dpo/margin_mean": 68.21715545654297,
      "margin_dpo/margin_std": 118.87091064453125,
      "step": 580
    },
    {
      "KL/chosen_KL_mean": -159.50442504882812,
      "KL/mean": -193.21817016601562,
      "KL/rejected_KL_mean": -226.93190002441406,
      "KL/std": 96.07111358642578,
      "epoch": 0.8783068783068783,
      "fcm_dpo/beta": 0.005554153583943844,
      "fcm_dpo/delta": 0.026327921077609062,
      "fcm_dpo/margin": 67.4274673461914,
      "fcm_dpo/q_t": 0.4155082106590271,
      "grad_norm": 13.451879501342773,
      "learning_rate": 2.259200116137039e-08,
      "logits/chosen": 0.72639000415802,
      "logits/rejected": 0.6580033898353577,
      "logps/chosen": -218.8367919921875,
      "logps/ref_chosen": -59.332359313964844,
      "logps/ref_rejected": -83.64482116699219,
      "logps/rejected": -310.57672119140625,
      "loss": 1.136,
      "margin_dpo/margin_mean": 67.4274673461914,
      "margin_dpo/margin_std": 110.48698425292969,
      "step": 581
    },
    {
      "KL/chosen_KL_mean": -145.05770874023438,
      "KL/mean": -176.64620971679688,
      "KL/rejected_KL_mean": -208.2346954345703,
      "KL/std": 92.65220642089844,
      "epoch": 0.8798185941043084,
      "fcm_dpo/beta": 0.005630874074995518,
      "fcm_dpo/delta": 0.04571300745010376,
      "fcm_dpo/margin": 63.17696762084961,
      "fcm_dpo/q_t": 0.41909968852996826,
      "grad_norm": 11.4200439453125,
      "learning_rate": 2.204591459016525e-08,
      "logits/chosen": 0.6998355388641357,
      "logits/rejected": 0.7280929684638977,
      "logps/chosen": -209.2205810546875,
      "logps/ref_chosen": -64.16285705566406,
      "logps/ref_rejected": -58.632896423339844,
      "logps/rejected": -266.86761474609375,
      "loss": 1.1468,
      "margin_dpo/margin_mean": 63.176971435546875,
      "margin_dpo/margin_std": 105.90766906738281,
      "step": 582
    },
    {
      "KL/chosen_KL_mean": -144.9822998046875,
      "KL/mean": -183.77243041992188,
      "KL/rejected_KL_mean": -222.56256103515625,
      "KL/std": 102.11319732666016,
      "epoch": 0.8813303099017384,
      "fcm_dpo/beta": 0.005599405616521835,
      "fcm_dpo/delta": -0.03599086403846741,
      "fcm_dpo/margin": 77.58027648925781,
      "fcm_dpo/q_t": 0.4010327458381653,
      "grad_norm": 15.530279159545898,
      "learning_rate": 2.1506204384751064e-08,
      "logits/chosen": 0.8185715675354004,
      "logits/rejected": 0.706648588180542,
      "logps/chosen": -196.85467529296875,
      "logps/ref_chosen": -51.87239456176758,
      "logps/ref_rejected": -83.86331176757812,
      "logps/rejected": -306.4258728027344,
      "loss": 1.1013,
      "margin_dpo/margin_mean": 77.58027648925781,
      "margin_dpo/margin_std": 117.77641296386719,
      "step": 583
    },
    {
      "KL/chosen_KL_mean": -132.80816650390625,
      "KL/mean": -166.947509765625,
      "KL/rejected_KL_mean": -201.08685302734375,
      "KL/std": 95.08578491210938,
      "epoch": 0.8828420256991686,
      "fcm_dpo/beta": 0.005589952692389488,
      "fcm_dpo/delta": 0.018903149291872978,
      "fcm_dpo/margin": 68.2786865234375,
      "fcm_dpo/q_t": 0.41556039452552795,
      "grad_norm": 13.357123374938965,
      "learning_rate": 2.09728856419826e-08,
      "logits/chosen": 0.8511885404586792,
      "logits/rejected": 0.7385942935943604,
      "logps/chosen": -179.37954711914062,
      "logps/ref_chosen": -46.571388244628906,
      "logps/ref_rejected": -80.67969512939453,
      "logps/rejected": -281.76654052734375,
      "loss": 1.1511,
      "margin_dpo/margin_mean": 68.2786865234375,
      "margin_dpo/margin_std": 122.87632751464844,
      "step": 584
    },
    {
      "KL/chosen_KL_mean": -155.32931518554688,
      "KL/mean": -179.2681884765625,
      "KL/rejected_KL_mean": -203.20704650878906,
      "KL/std": 103.95755004882812,
      "epoch": 0.8843537414965986,
      "fcm_dpo/beta": 0.005628950893878937,
      "fcm_dpo/delta": 0.027845166623592377,
      "fcm_dpo/margin": 47.87772750854492,
      "fcm_dpo/q_t": 0.43734651803970337,
      "grad_norm": 12.0934419631958,
      "learning_rate": 2.044597327993153e-08,
      "logits/chosen": 0.6991287469863892,
      "logits/rejected": 0.643784761428833,
      "logps/chosen": -213.453857421875,
      "logps/ref_chosen": -58.124534606933594,
      "logps/ref_rejected": -79.00538635253906,
      "logps/rejected": -282.2124328613281,
      "loss": 1.2184,
      "margin_dpo/margin_mean": 47.877723693847656,
      "margin_dpo/margin_std": 104.39326477050781,
      "step": 585
    },
    {
      "KL/chosen_KL_mean": -143.0966339111328,
      "KL/mean": -177.208251953125,
      "KL/rejected_KL_mean": -211.3198699951172,
      "KL/std": 89.04827117919922,
      "epoch": 0.8858654572940288,
      "fcm_dpo/beta": 0.005642901174724102,
      "fcm_dpo/delta": 0.015524804592132568,
      "fcm_dpo/margin": 68.22322082519531,
      "fcm_dpo/q_t": 0.41047054529190063,
      "grad_norm": 15.621770858764648,
      "learning_rate": 1.9925482037469187e-08,
      "logits/chosen": 0.7698843479156494,
      "logits/rejected": 0.7214852571487427,
      "logps/chosen": -197.19827270507812,
      "logps/ref_chosen": -54.10163879394531,
      "logps/ref_rejected": -63.72113037109375,
      "logps/rejected": -275.041015625,
      "loss": 1.0877,
      "margin_dpo/margin_mean": 68.22322082519531,
      "margin_dpo/margin_std": 81.10868835449219,
      "step": 586
    },
    {
      "KL/chosen_KL_mean": -149.34744262695312,
      "KL/mean": -182.81503295898438,
      "KL/rejected_KL_mean": -216.28262329101562,
      "KL/std": 101.19564819335938,
      "epoch": 0.8873771730914588,
      "fcm_dpo/beta": 0.00567442923784256,
      "fcm_dpo/delta": 0.020979033783078194,
      "fcm_dpo/margin": 66.9351806640625,
      "fcm_dpo/q_t": 0.4098867177963257,
      "grad_norm": 14.616544723510742,
      "learning_rate": 1.9411426473854687e-08,
      "logits/chosen": 0.7673693895339966,
      "logits/rejected": 0.7625389099121094,
      "logps/chosen": -212.7646484375,
      "logps/ref_chosen": -63.41719436645508,
      "logps/ref_rejected": -63.47003936767578,
      "logps/rejected": -279.752685546875,
      "loss": 1.1725,
      "margin_dpo/margin_mean": 66.9351806640625,
      "margin_dpo/margin_std": 134.6204833984375,
      "step": 587
    },
    {
      "KL/chosen_KL_mean": -150.87796020507812,
      "KL/mean": -189.13902282714844,
      "KL/rejected_KL_mean": -227.4000701904297,
      "KL/std": 100.95319366455078,
      "epoch": 0.8888888888888888,
      "fcm_dpo/beta": 0.005642802454531193,
      "fcm_dpo/delta": -0.03346514701843262,
      "fcm_dpo/margin": 76.5220947265625,
      "fcm_dpo/q_t": 0.4033350944519043,
      "grad_norm": 15.386234283447266,
      "learning_rate": 1.890382096832699e-08,
      "logits/chosen": 0.7514165639877319,
      "logits/rejected": 0.7071614265441895,
      "logps/chosen": -213.07901000976562,
      "logps/ref_chosen": -62.20103454589844,
      "logps/ref_rejected": -82.10249328613281,
      "logps/rejected": -309.5025634765625,
      "loss": 1.1105,
      "margin_dpo/margin_mean": 76.5220947265625,
      "margin_dpo/margin_std": 122.594482421875,
      "step": 588
    },
    {
      "KL/chosen_KL_mean": -142.32830810546875,
      "KL/mean": -180.05657958984375,
      "KL/rejected_KL_mean": -217.78482055664062,
      "KL/std": 97.2964859008789,
      "epoch": 0.890400604686319,
      "fcm_dpo/beta": 0.005636701360344887,
      "fcm_dpo/delta": -0.026475675404071808,
      "fcm_dpo/margin": 75.45652770996094,
      "fcm_dpo/q_t": 0.40082281827926636,
      "grad_norm": 11.952903747558594,
      "learning_rate": 1.840267971970344e-08,
      "logits/chosen": 0.7151200771331787,
      "logits/rejected": 0.6838746666908264,
      "logps/chosen": -199.04193115234375,
      "logps/ref_chosen": -56.71361541748047,
      "logps/ref_rejected": -76.7366943359375,
      "logps/rejected": -294.5215148925781,
      "loss": 1.0668,
      "margin_dpo/margin_mean": 75.45652770996094,
      "margin_dpo/margin_std": 90.1925048828125,
      "step": 589
    },
    {
      "KL/chosen_KL_mean": -157.22772216796875,
      "KL/mean": -192.367919921875,
      "KL/rejected_KL_mean": -227.5081329345703,
      "KL/std": 91.61934661865234,
      "epoch": 0.891912320483749,
      "fcm_dpo/beta": 0.005639345850795507,
      "fcm_dpo/delta": 0.0035936329513788223,
      "fcm_dpo/margin": 70.28041076660156,
      "fcm_dpo/q_t": 0.4088994860649109,
      "grad_norm": 16.716114044189453,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": 0.6473318934440613,
      "logits/rejected": 0.6127752065658569,
      "logps/chosen": -223.74154663085938,
      "logps/ref_chosen": -66.5138168334961,
      "logps/ref_rejected": -85.70820617675781,
      "logps/rejected": -313.2163391113281,
      "loss": 1.107,
      "margin_dpo/margin_mean": 70.28040313720703,
      "margin_dpo/margin_std": 101.08212280273438,
      "step": 590
    },
    {
      "KL/chosen_KL_mean": -133.28314208984375,
      "KL/mean": -177.31515502929688,
      "KL/rejected_KL_mean": -221.34716796875,
      "KL/std": 105.98800659179688,
      "epoch": 0.8934240362811792,
      "fcm_dpo/beta": 0.00550592876970768,
      "fcm_dpo/delta": -0.09098677337169647,
      "fcm_dpo/margin": 88.06401062011719,
      "fcm_dpo/q_t": 0.39215224981307983,
      "grad_norm": 14.096073150634766,
      "learning_rate": 1.7419845883949098e-08,
      "logits/chosen": 0.8193856477737427,
      "logits/rejected": 0.7526212334632874,
      "logps/chosen": -193.98033142089844,
      "logps/ref_chosen": -60.697181701660156,
      "logps/ref_rejected": -86.12278747558594,
      "logps/rejected": -307.469970703125,
      "loss": 1.0807,
      "margin_dpo/margin_mean": 88.06401062011719,
      "margin_dpo/margin_std": 130.5552978515625,
      "step": 591
    },
    {
      "KL/chosen_KL_mean": -146.2674560546875,
      "KL/mean": -179.26779174804688,
      "KL/rejected_KL_mean": -212.26812744140625,
      "KL/std": 91.21923828125,
      "epoch": 0.8949357520786092,
      "fcm_dpo/beta": 0.005524776875972748,
      "fcm_dpo/delta": 0.03641321882605553,
      "fcm_dpo/margin": 66.00065612792969,
      "fcm_dpo/q_t": 0.4181811809539795,
      "grad_norm": 13.774628639221191,
      "learning_rate": 1.6938180788793556e-08,
      "logits/chosen": 0.7695102691650391,
      "logits/rejected": 0.6523857116699219,
      "logps/chosen": -197.50479125976562,
      "logps/ref_chosen": -51.237327575683594,
      "logps/ref_rejected": -81.60242462158203,
      "logps/rejected": -293.87054443359375,
      "loss": 1.1327,
      "margin_dpo/margin_mean": 66.00065612792969,
      "margin_dpo/margin_std": 102.60389709472656,
      "step": 592
    },
    {
      "KL/chosen_KL_mean": -121.18894958496094,
      "KL/mean": -157.242431640625,
      "KL/rejected_KL_mean": -193.29591369628906,
      "KL/std": 93.52127075195312,
      "epoch": 0.8964474678760394,
      "fcm_dpo/beta": 0.005549794062972069,
      "fcm_dpo/delta": -0.00027018971741199493,
      "fcm_dpo/margin": 72.10696411132812,
      "fcm_dpo/q_t": 0.4079144597053528,
      "grad_norm": 16.033967971801758,
      "learning_rate": 1.6463034933723336e-08,
      "logits/chosen": 0.7926292419433594,
      "logits/rejected": 0.6890050172805786,
      "logps/chosen": -163.26895141601562,
      "logps/ref_chosen": -42.08000183105469,
      "logps/ref_rejected": -68.47499084472656,
      "logps/rejected": -261.7709045410156,
      "loss": 1.1112,
      "margin_dpo/margin_mean": 72.10696411132812,
      "margin_dpo/margin_std": 108.24049377441406,
      "step": 593
    },
    {
      "KL/chosen_KL_mean": -153.3001708984375,
      "KL/mean": -185.24551391601562,
      "KL/rejected_KL_mean": -217.19085693359375,
      "KL/std": 92.95860290527344,
      "epoch": 0.8979591836734694,
      "fcm_dpo/beta": 0.005595002323389053,
      "fcm_dpo/delta": 0.04412460699677467,
      "fcm_dpo/margin": 63.89067459106445,
      "fcm_dpo/q_t": 0.41660457849502563,
      "grad_norm": 13.056547164916992,
      "learning_rate": 1.5994421609589385e-08,
      "logits/chosen": 0.6857548952102661,
      "logits/rejected": 0.6704928278923035,
      "logps/chosen": -216.95884704589844,
      "logps/ref_chosen": -63.658668518066406,
      "logps/ref_rejected": -70.35597229003906,
      "logps/rejected": -287.54681396484375,
      "loss": 1.1295,
      "margin_dpo/margin_mean": 63.89067077636719,
      "margin_dpo/margin_std": 95.78257751464844,
      "step": 594
    },
    {
      "KL/chosen_KL_mean": -141.01893615722656,
      "KL/mean": -182.9136962890625,
      "KL/rejected_KL_mean": -224.8084716796875,
      "KL/std": 98.6803970336914,
      "epoch": 0.8994708994708994,
      "fcm_dpo/beta": 0.005570332985371351,
      "fcm_dpo/delta": -0.06999208778142929,
      "fcm_dpo/margin": 83.7895278930664,
      "fcm_dpo/q_t": 0.39455899596214294,
      "grad_norm": 11.194549560546875,
      "learning_rate": 1.553235392451377e-08,
      "logits/chosen": 0.8061363697052002,
      "logits/rejected": 0.7098953723907471,
      "logps/chosen": -197.23770141601562,
      "logps/ref_chosen": -56.21875762939453,
      "logps/ref_rejected": -83.95773315429688,
      "logps/rejected": -308.7662048339844,
      "loss": 1.0802,
      "margin_dpo/margin_mean": 83.78953552246094,
      "margin_dpo/margin_std": 121.9223403930664,
      "step": 595
    },
    {
      "KL/chosen_KL_mean": -162.91629028320312,
      "KL/mean": -180.66098022460938,
      "KL/rejected_KL_mean": -198.40570068359375,
      "KL/std": 92.42216491699219,
      "epoch": 0.9009826152683296,
      "fcm_dpo/beta": 0.005609722808003426,
      "fcm_dpo/delta": 0.06954170018434525,
      "fcm_dpo/margin": 35.489410400390625,
      "fcm_dpo/q_t": 0.4551619291305542,
      "grad_norm": 13.614740371704102,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": 0.6191815137863159,
      "logits/rejected": 0.6415808200836182,
      "logps/chosen": -231.39715576171875,
      "logps/ref_chosen": -68.48088073730469,
      "logps/ref_rejected": -61.732967376708984,
      "logps/rejected": -260.138671875,
      "loss": 1.2714,
      "margin_dpo/margin_mean": 35.489410400390625,
      "margin_dpo/margin_std": 98.60360717773438,
      "step": 596
    },
    {
      "KL/chosen_KL_mean": -125.30804443359375,
      "KL/mean": -157.54745483398438,
      "KL/rejected_KL_mean": -189.786865234375,
      "KL/std": 83.25325012207031,
      "epoch": 0.9024943310657596,
      "fcm_dpo/beta": 0.005650757811963558,
      "fcm_dpo/delta": 0.0368620865046978,
      "fcm_dpo/margin": 64.47880554199219,
      "fcm_dpo/q_t": 0.4167628884315491,
      "grad_norm": 11.625533103942871,
      "learning_rate": 1.4627906988186111e-08,
      "logits/chosen": 0.7446720600128174,
      "logits/rejected": 0.7268559336662292,
      "logps/chosen": -174.16555786132812,
      "logps/ref_chosen": -48.85750961303711,
      "logps/ref_rejected": -55.068084716796875,
      "logps/rejected": -244.85494995117188,
      "loss": 1.1306,
      "margin_dpo/margin_mean": 64.47881317138672,
      "margin_dpo/margin_std": 100.05552673339844,
      "step": 597
    },
    {
      "KL/chosen_KL_mean": -164.51393127441406,
      "KL/mean": -184.85238647460938,
      "KL/rejected_KL_mean": -205.19082641601562,
      "KL/std": 95.56716918945312,
      "epoch": 0.9040060468631897,
      "fcm_dpo/beta": 0.00568807777017355,
      "fcm_dpo/delta": 0.06477639079093933,
      "fcm_dpo/margin": 40.676902770996094,
      "fcm_dpo/q_t": 0.447678804397583,
      "grad_norm": 12.799821853637695,
      "learning_rate": 1.4185553036259095e-08,
      "logits/chosen": 0.7193522453308105,
      "logits/rejected": 0.6358869075775146,
      "logps/chosen": -223.40109252929688,
      "logps/ref_chosen": -58.88715362548828,
      "logps/ref_rejected": -81.43145751953125,
      "logps/rejected": -286.6222839355469,
      "loss": 1.2513,
      "margin_dpo/margin_mean": 40.67690658569336,
      "margin_dpo/margin_std": 103.65243530273438,
      "step": 598
    },
    {
      "KL/chosen_KL_mean": -166.65670776367188,
      "KL/mean": -191.69293212890625,
      "KL/rejected_KL_mean": -216.72915649414062,
      "KL/std": 93.59307861328125,
      "epoch": 0.9055177626606198,
      "fcm_dpo/beta": 0.005834928713738918,
      "fcm_dpo/delta": 0.11067037284374237,
      "fcm_dpo/margin": 50.07246780395508,
      "fcm_dpo/q_t": 0.4342125356197357,
      "grad_norm": 16.525049209594727,
      "learning_rate": 1.3749795321332885e-08,
      "logits/chosen": 0.785170316696167,
      "logits/rejected": 0.7406322956085205,
      "logps/chosen": -224.26388549804688,
      "logps/ref_chosen": -57.60719299316406,
      "logps/ref_rejected": -71.80469512939453,
      "logps/rejected": -288.53387451171875,
      "loss": 1.2064,
      "margin_dpo/margin_mean": 50.072471618652344,
      "margin_dpo/margin_std": 106.24748229980469,
      "step": 599
    },
    {
      "KL/chosen_KL_mean": -151.4080810546875,
      "KL/mean": -180.9127960205078,
      "KL/rejected_KL_mean": -210.41751098632812,
      "KL/std": 100.85147857666016,
      "epoch": 0.9070294784580499,
      "fcm_dpo/beta": 0.005830493755638599,
      "fcm_dpo/delta": -0.03868885338306427,
      "fcm_dpo/margin": 59.0093994140625,
      "fcm_dpo/q_t": 0.4230087399482727,
      "grad_norm": 15.823807716369629,
      "learning_rate": 1.3320646032487393e-08,
      "logits/chosen": 0.7570271492004395,
      "logits/rejected": 0.6990246772766113,
      "logps/chosen": -209.85040283203125,
      "logps/ref_chosen": -58.44231414794922,
      "logps/ref_rejected": -83.64639282226562,
      "logps/rejected": -294.06390380859375,
      "loss": 1.1649,
      "margin_dpo/margin_mean": 59.0093994140625,
      "margin_dpo/margin_std": 106.41853332519531,
      "step": 600
    },
    {
      "KL/chosen_KL_mean": -134.89620971679688,
      "KL/mean": -174.92724609375,
      "KL/rejected_KL_mean": -214.958251953125,
      "KL/std": 104.88700866699219,
      "epoch": 0.90854119425548,
      "fcm_dpo/beta": 0.005732652731239796,
      "fcm_dpo/delta": -0.062330782413482666,
      "fcm_dpo/margin": 80.06205749511719,
      "fcm_dpo/q_t": 0.39689481258392334,
      "grad_norm": 11.354548454284668,
      "learning_rate": 1.2898117173950868e-08,
      "logits/chosen": 0.7024219036102295,
      "logits/rejected": 0.622978687286377,
      "logps/chosen": -190.49053955078125,
      "logps/ref_chosen": -55.59432601928711,
      "logps/ref_rejected": -83.68630981445312,
      "logps/rejected": -298.64459228515625,
      "loss": 1.0866,
      "margin_dpo/margin_mean": 80.06205749511719,
      "margin_dpo/margin_std": 118.5966796875,
      "step": 601
    },
    {
      "KL/chosen_KL_mean": -123.71614074707031,
      "KL/mean": -161.18551635742188,
      "KL/rejected_KL_mean": -198.65487670898438,
      "KL/std": 92.09284973144531,
      "epoch": 0.91005291005291,
      "fcm_dpo/beta": 0.005728420335799456,
      "fcm_dpo/delta": -0.030709169805049896,
      "fcm_dpo/margin": 74.938720703125,
      "fcm_dpo/q_t": 0.40118837356567383,
      "grad_norm": 13.959485054016113,
      "learning_rate": 1.2482220564763667e-08,
      "logits/chosen": 0.7402975559234619,
      "logits/rejected": 0.7094787359237671,
      "logps/chosen": -180.06533813476562,
      "logps/ref_chosen": -56.349185943603516,
      "logps/ref_rejected": -71.9959716796875,
      "logps/rejected": -270.6508483886719,
      "loss": 1.0784,
      "margin_dpo/margin_mean": 74.938720703125,
      "margin_dpo/margin_std": 99.05213165283203,
      "step": 602
    },
    {
      "KL/chosen_KL_mean": -137.94265747070312,
      "KL/mean": -173.8961181640625,
      "KL/rejected_KL_mean": -209.84957885742188,
      "KL/std": 91.87611389160156,
      "epoch": 0.9115646258503401,
      "fcm_dpo/beta": 0.005682522896677256,
      "fcm_dpo/delta": -0.00906812772154808,
      "fcm_dpo/margin": 71.90690612792969,
      "fcm_dpo/q_t": 0.406727135181427,
      "grad_norm": 14.226873397827148,
      "learning_rate": 1.2072967838448051e-08,
      "logits/chosen": 0.7089002132415771,
      "logits/rejected": 0.651750385761261,
      "logps/chosen": -191.11105346679688,
      "logps/ref_chosen": -53.16838836669922,
      "logps/ref_rejected": -73.8604736328125,
      "logps/rejected": -283.7100524902344,
      "loss": 1.1073,
      "margin_dpo/margin_mean": 71.90690612792969,
      "margin_dpo/margin_std": 107.62342834472656,
      "step": 603
    },
    {
      "KL/chosen_KL_mean": -133.23109436035156,
      "KL/mean": -164.9775390625,
      "KL/rejected_KL_mean": -196.72398376464844,
      "KL/std": 87.64952087402344,
      "epoch": 0.9130763416477702,
      "fcm_dpo/beta": 0.005730690900236368,
      "fcm_dpo/delta": 0.03740895539522171,
      "fcm_dpo/margin": 63.492889404296875,
      "fcm_dpo/q_t": 0.41807228326797485,
      "grad_norm": 15.897056579589844,
      "learning_rate": 1.1670370442682459e-08,
      "logits/chosen": 0.6813480257987976,
      "logits/rejected": 0.6873229742050171,
      "logps/chosen": -205.88052368164062,
      "logps/ref_chosen": -72.64942169189453,
      "logps/ref_rejected": -69.8792724609375,
      "logps/rejected": -266.6032409667969,
      "loss": 1.1534,
      "margin_dpo/margin_mean": 63.492889404296875,
      "margin_dpo/margin_std": 113.17750549316406,
      "step": 604
    },
    {
      "KL/chosen_KL_mean": -152.93955993652344,
      "KL/mean": -185.7640380859375,
      "KL/rejected_KL_mean": -218.58853149414062,
      "KL/std": 91.53559875488281,
      "epoch": 0.9145880574452003,
      "fcm_dpo/beta": 0.005740322172641754,
      "fcm_dpo/delta": 0.024004101753234863,
      "fcm_dpo/margin": 65.6489486694336,
      "fcm_dpo/q_t": 0.4135058522224426,
      "grad_norm": 14.684964179992676,
      "learning_rate": 1.1274439638981532e-08,
      "logits/chosen": 0.7383975982666016,
      "logits/rejected": 0.6835330128669739,
      "logps/chosen": -214.55239868164062,
      "logps/ref_chosen": -61.61284637451172,
      "logps/ref_rejected": -79.34398651123047,
      "logps/rejected": -297.9324951171875,
      "loss": 1.1398,
      "margin_dpo/margin_mean": 65.6489486694336,
      "margin_dpo/margin_std": 109.92323303222656,
      "step": 605
    },
    {
      "KL/chosen_KL_mean": -133.11740112304688,
      "KL/mean": -173.52670288085938,
      "KL/rejected_KL_mean": -213.93597412109375,
      "KL/std": 94.95388793945312,
      "epoch": 0.9160997732426304,
      "fcm_dpo/beta": 0.005727029405534267,
      "fcm_dpo/delta": -0.06595481932163239,
      "fcm_dpo/margin": 80.81857299804688,
      "fcm_dpo/q_t": 0.39591366052627563,
      "grad_norm": 15.443764686584473,
      "learning_rate": 1.0885186502381016e-08,
      "logits/chosen": 0.706656277179718,
      "logits/rejected": 0.637535810470581,
      "logps/chosen": -187.58163452148438,
      "logps/ref_chosen": -54.46424102783203,
      "logps/ref_rejected": -79.62708282470703,
      "logps/rejected": -293.56304931640625,
      "loss": 1.0711,
      "margin_dpo/margin_mean": 80.8185806274414,
      "margin_dpo/margin_std": 111.27831268310547,
      "step": 606
    },
    {
      "KL/chosen_KL_mean": -149.36038208007812,
      "KL/mean": -183.9228973388672,
      "KL/rejected_KL_mean": -218.48541259765625,
      "KL/std": 93.28308868408203,
      "epoch": 0.9176114890400605,
      "fcm_dpo/beta": 0.005645174998790026,
      "fcm_dpo/delta": 0.008793435990810394,
      "fcm_dpo/margin": 69.12503051757812,
      "fcm_dpo/q_t": 0.41042715311050415,
      "grad_norm": 13.01208209991455,
      "learning_rate": 1.0502621921127774e-08,
      "logits/chosen": 0.7263978719711304,
      "logits/rejected": 0.6995840072631836,
      "logps/chosen": -212.22125244140625,
      "logps/ref_chosen": -62.86086654663086,
      "logps/ref_rejected": -72.5501937866211,
      "logps/rejected": -291.03558349609375,
      "loss": 1.1273,
      "margin_dpo/margin_mean": 69.12503051757812,
      "margin_dpo/margin_std": 107.19122314453125,
      "step": 607
    },
    {
      "KL/chosen_KL_mean": -151.8929443359375,
      "KL/mean": -188.0191192626953,
      "KL/rejected_KL_mean": -224.14529418945312,
      "KL/std": 99.14498901367188,
      "epoch": 0.9191232048374905,
      "fcm_dpo/beta": 0.005693910177797079,
      "fcm_dpo/delta": -0.011916290037333965,
      "fcm_dpo/margin": 72.25234985351562,
      "fcm_dpo/q_t": 0.40606489777565,
      "grad_norm": 12.95539665222168,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": 0.7004761695861816,
      "logits/rejected": 0.6175021529197693,
      "logps/chosen": -215.07366943359375,
      "logps/ref_chosen": -63.18071746826172,
      "logps/ref_rejected": -99.15888214111328,
      "logps/rejected": -323.3041687011719,
      "loss": 1.095,
      "margin_dpo/margin_mean": 72.2523422241211,
      "margin_dpo/margin_std": 101.28202056884766,
      "step": 608
    },
    {
      "KL/chosen_KL_mean": -134.8228759765625,
      "KL/mean": -174.05133056640625,
      "KL/rejected_KL_mean": -213.27976989746094,
      "KL/std": 92.3570327758789,
      "epoch": 0.9206349206349206,
      "fcm_dpo/beta": 0.005622707773000002,
      "fcm_dpo/delta": -0.043535713106393814,
      "fcm_dpo/margin": 78.45690155029297,
      "fcm_dpo/q_t": 0.3969118297100067,
      "grad_norm": 12.496787071228027,
      "learning_rate": 9.757601041885694e-09,
      "logits/chosen": 0.80591881275177,
      "logits/rejected": 0.7658596038818359,
      "logps/chosen": -183.4460906982422,
      "logps/ref_chosen": -48.62322235107422,
      "logps/ref_rejected": -68.28271484375,
      "logps/rejected": -281.5625,
      "loss": 1.0624,
      "margin_dpo/margin_mean": 78.45689392089844,
      "margin_dpo/margin_std": 93.21892547607422,
      "step": 609
    },
    {
      "KL/chosen_KL_mean": -146.20834350585938,
      "KL/mean": -184.7145233154297,
      "KL/rejected_KL_mean": -223.220703125,
      "KL/std": 99.93855285644531,
      "epoch": 0.9221466364323507,
      "fcm_dpo/beta": 0.00558491563424468,
      "fcm_dpo/delta": -0.03186805918812752,
      "fcm_dpo/margin": 77.01237487792969,
      "fcm_dpo/q_t": 0.40332934260368347,
      "grad_norm": 13.486494064331055,
      "learning_rate": 9.395165583732379e-09,
      "logits/chosen": 0.7073228359222412,
      "logits/rejected": 0.7007800340652466,
      "logps/chosen": -218.87347412109375,
      "logps/ref_chosen": -72.66513061523438,
      "logps/ref_rejected": -87.15310668945312,
      "logps/rejected": -310.3738098144531,
      "loss": 1.095,
      "margin_dpo/margin_mean": 77.01237487792969,
      "margin_dpo/margin_std": 112.86748504638672,
      "step": 610
    },
    {
      "KL/chosen_KL_mean": -136.69102478027344,
      "KL/mean": -167.19656372070312,
      "KL/rejected_KL_mean": -197.70208740234375,
      "KL/std": 89.79940795898438,
      "epoch": 0.9236583522297808,
      "fcm_dpo/beta": 0.00563270878046751,
      "fcm_dpo/delta": 0.05837348476052284,
      "fcm_dpo/margin": 61.01106643676758,
      "fcm_dpo/q_t": 0.420263409614563,
      "grad_norm": 15.119729995727539,
      "learning_rate": 9.03946036001449e-09,
      "logits/chosen": 0.7547829151153564,
      "logits/rejected": 0.7037972807884216,
      "logps/chosen": -184.99960327148438,
      "logps/ref_chosen": -48.30857849121094,
      "logps/ref_rejected": -70.6141128540039,
      "logps/rejected": -268.31622314453125,
      "loss": 1.1383,
      "margin_dpo/margin_mean": 61.01106643676758,
      "margin_dpo/margin_std": 93.35009002685547,
      "step": 611
    },
    {
      "KL/chosen_KL_mean": -148.52883911132812,
      "KL/mean": -193.0296630859375,
      "KL/rejected_KL_mean": -237.53045654296875,
      "KL/std": 95.53738403320312,
      "epoch": 0.9251700680272109,
      "fcm_dpo/beta": 0.005580402445048094,
      "fcm_dpo/delta": -0.10167094320058823,
      "fcm_dpo/margin": 89.00161743164062,
      "fcm_dpo/q_t": 0.38511383533477783,
      "grad_norm": 12.341629028320312,
      "learning_rate": 8.690495320571839e-09,
      "logits/chosen": 0.6185309290885925,
      "logits/rejected": 0.5481315851211548,
      "logps/chosen": -209.76040649414062,
      "logps/ref_chosen": -61.23155975341797,
      "logps/ref_rejected": -94.37979888916016,
      "logps/rejected": -331.9102783203125,
      "loss": 1.0305,
      "margin_dpo/margin_mean": 89.00161743164062,
      "margin_dpo/margin_std": 105.5512466430664,
      "step": 612
    },
    {
      "KL/chosen_KL_mean": -128.20095825195312,
      "KL/mean": -168.02951049804688,
      "KL/rejected_KL_mean": -207.8580322265625,
      "KL/std": 99.59346771240234,
      "epoch": 0.926681783824641,
      "fcm_dpo/beta": 0.00553030613809824,
      "fcm_dpo/delta": -0.04245399683713913,
      "fcm_dpo/margin": 79.65707397460938,
      "fcm_dpo/q_t": 0.3979625701904297,
      "grad_norm": 11.432785987854004,
      "learning_rate": 8.348280226706722e-09,
      "logits/chosen": 0.69444340467453,
      "logits/rejected": 0.6887099742889404,
      "logps/chosen": -182.18406677246094,
      "logps/ref_chosen": -53.98310852050781,
      "logps/ref_rejected": -58.32208251953125,
      "logps/rejected": -266.18011474609375,
      "loss": 1.071,
      "margin_dpo/margin_mean": 79.65707397460938,
      "margin_dpo/margin_std": 103.14279174804688,
      "step": 613
    },
    {
      "KL/chosen_KL_mean": -149.01048278808594,
      "KL/mean": -186.2037811279297,
      "KL/rejected_KL_mean": -223.39706420898438,
      "KL/std": 87.71038818359375,
      "epoch": 0.9281934996220711,
      "fcm_dpo/beta": 0.005520460195839405,
      "fcm_dpo/delta": -0.01158231869339943,
      "fcm_dpo/margin": 74.38656616210938,
      "fcm_dpo/q_t": 0.40441012382507324,
      "grad_norm": 15.124855041503906,
      "learning_rate": 8.012824650910937e-09,
      "logits/chosen": 0.7338849306106567,
      "logits/rejected": 0.7256481647491455,
      "logps/chosen": -209.25350952148438,
      "logps/ref_chosen": -60.24303436279297,
      "logps/ref_rejected": -72.26258850097656,
      "logps/rejected": -295.65966796875,
      "loss": 1.0923,
      "margin_dpo/margin_mean": 74.38656616210938,
      "margin_dpo/margin_std": 100.2391357421875,
      "step": 614
    },
    {
      "KL/chosen_KL_mean": -156.41226196289062,
      "KL/mean": -194.71534729003906,
      "KL/rejected_KL_mean": -233.01844787597656,
      "KL/std": 110.10960388183594,
      "epoch": 0.9297052154195011,
      "fcm_dpo/beta": 0.005440958775579929,
      "fcm_dpo/delta": -0.01839909330010414,
      "fcm_dpo/margin": 76.60619354248047,
      "fcm_dpo/q_t": 0.4055163860321045,
      "grad_norm": 12.36121940612793,
      "learning_rate": 7.684137976598088e-09,
      "logits/chosen": 0.6608693599700928,
      "logits/rejected": 0.6032828092575073,
      "logps/chosen": -228.50692749023438,
      "logps/ref_chosen": -72.09467315673828,
      "logps/ref_rejected": -104.02980041503906,
      "logps/rejected": -337.0482482910156,
      "loss": 1.1181,
      "margin_dpo/margin_mean": 76.60618591308594,
      "margin_dpo/margin_std": 122.05841064453125,
      "step": 615
    },
    {
      "KL/chosen_KL_mean": -147.6690673828125,
      "KL/mean": -180.33358764648438,
      "KL/rejected_KL_mean": -212.99810791015625,
      "KL/std": 102.95178985595703,
      "epoch": 0.9312169312169312,
      "fcm_dpo/beta": 0.0054956115782260895,
      "fcm_dpo/delta": 0.042521800845861435,
      "fcm_dpo/margin": 65.32905578613281,
      "fcm_dpo/q_t": 0.41801419854164124,
      "grad_norm": 12.493739128112793,
      "learning_rate": 7.36222939784098e-09,
      "logits/chosen": 0.7943388223648071,
      "logits/rejected": 0.7104923725128174,
      "logps/chosen": -206.1997833251953,
      "logps/ref_chosen": -58.530723571777344,
      "logps/ref_rejected": -75.48025512695312,
      "logps/rejected": -288.4783630371094,
      "loss": 1.1356,
      "margin_dpo/margin_mean": 65.32905578613281,
      "margin_dpo/margin_std": 103.19471740722656,
      "step": 616
    },
    {
      "KL/chosen_KL_mean": -158.54544067382812,
      "KL/mean": -186.70318603515625,
      "KL/rejected_KL_mean": -214.86094665527344,
      "KL/std": 94.73501586914062,
      "epoch": 0.9327286470143613,
      "fcm_dpo/beta": 0.005482667591422796,
      "fcm_dpo/delta": -0.029780426993966103,
      "fcm_dpo/margin": 56.31550216674805,
      "fcm_dpo/q_t": 0.4290716350078583,
      "grad_norm": 16.646713256835938,
      "learning_rate": 7.047107919114586e-09,
      "logits/chosen": 0.7297405004501343,
      "logits/rejected": 0.6762892007827759,
      "logps/chosen": -216.15411376953125,
      "logps/ref_chosen": -57.608673095703125,
      "logps/ref_rejected": -81.22109985351562,
      "logps/rejected": -296.08203125,
      "loss": 1.1793,
      "margin_dpo/margin_mean": 56.31550216674805,
      "margin_dpo/margin_std": 100.77790069580078,
      "step": 617
    },
    {
      "KL/chosen_KL_mean": -142.28543090820312,
      "KL/mean": -177.02438354492188,
      "KL/rejected_KL_mean": -211.76336669921875,
      "KL/std": 103.45098876953125,
      "epoch": 0.9342403628117913,
      "fcm_dpo/beta": 0.005484769586473703,
      "fcm_dpo/delta": 0.019597385078668594,
      "fcm_dpo/margin": 69.4779281616211,
      "fcm_dpo/q_t": 0.4140332341194153,
      "grad_norm": 16.582569122314453,
      "learning_rate": 6.738782355044048e-09,
      "logits/chosen": 0.7223110198974609,
      "logits/rejected": 0.6146073341369629,
      "logps/chosen": -198.9813690185547,
      "logps/ref_chosen": -56.69594192504883,
      "logps/ref_rejected": -85.92362976074219,
      "logps/rejected": -297.68701171875,
      "loss": 1.1169,
      "margin_dpo/margin_mean": 69.4779281616211,
      "margin_dpo/margin_std": 104.17143249511719,
      "step": 618
    },
    {
      "KL/chosen_KL_mean": -138.02951049804688,
      "KL/mean": -175.8691864013672,
      "KL/rejected_KL_mean": -213.7088623046875,
      "KL/std": 99.71603393554688,
      "epoch": 0.9357520786092215,
      "fcm_dpo/beta": 0.0054851071909070015,
      "fcm_dpo/delta": -0.015807051211595535,
      "fcm_dpo/margin": 75.67935180664062,
      "fcm_dpo/q_t": 0.40444010496139526,
      "grad_norm": 12.825272560119629,
      "learning_rate": 6.437261330158206e-09,
      "logits/chosen": 0.8090226054191589,
      "logits/rejected": 0.7290970683097839,
      "logps/chosen": -192.08792114257812,
      "logps/ref_chosen": -54.05841827392578,
      "logps/ref_rejected": -83.55493927001953,
      "logps/rejected": -297.2637939453125,
      "loss": 1.0981,
      "margin_dpo/margin_mean": 75.67935180664062,
      "margin_dpo/margin_std": 109.75007629394531,
      "step": 619
    },
    {
      "KL/chosen_KL_mean": -140.25616455078125,
      "KL/mean": -169.9412384033203,
      "KL/rejected_KL_mean": -199.62631225585938,
      "KL/std": 89.34503173828125,
      "epoch": 0.9372637944066515,
      "fcm_dpo/beta": 0.005455330945551395,
      "fcm_dpo/delta": -0.05773269385099411,
      "fcm_dpo/margin": 59.37013626098633,
      "fcm_dpo/q_t": 0.42477214336395264,
      "grad_norm": 13.20751667022705,
      "learning_rate": 6.142553278648238e-09,
      "logits/chosen": 0.7635496854782104,
      "logits/rejected": 0.7635151147842407,
      "logps/chosen": -203.62588500976562,
      "logps/ref_chosen": -63.36971664428711,
      "logps/ref_rejected": -65.68269348144531,
      "logps/rejected": -265.3089904785156,
      "loss": 1.1647,
      "margin_dpo/margin_mean": 59.37013244628906,
      "margin_dpo/margin_std": 97.37005615234375,
      "step": 620
    },
    {
      "KL/chosen_KL_mean": -141.00387573242188,
      "KL/mean": -172.40478515625,
      "KL/rejected_KL_mean": -203.80572509765625,
      "KL/std": 95.36180114746094,
      "epoch": 0.9387755102040817,
      "fcm_dpo/beta": 0.005485064350068569,
      "fcm_dpo/delta": 0.05720696970820427,
      "fcm_dpo/margin": 62.80182647705078,
      "fcm_dpo/q_t": 0.42262840270996094,
      "grad_norm": 15.133943557739258,
      "learning_rate": 5.854666444131934e-09,
      "logits/chosen": 0.7632216215133667,
      "logits/rejected": 0.6463322639465332,
      "logps/chosen": -193.32510375976562,
      "logps/ref_chosen": -52.321224212646484,
      "logps/ref_rejected": -88.09001159667969,
      "logps/rejected": -291.8957214355469,
      "loss": 1.1633,
      "margin_dpo/margin_mean": 62.80183410644531,
      "margin_dpo/margin_std": 113.8005142211914,
      "step": 621
    },
    {
      "KL/chosen_KL_mean": -153.46717834472656,
      "KL/mean": -186.71771240234375,
      "KL/rejected_KL_mean": -219.96827697753906,
      "KL/std": 100.89706420898438,
      "epoch": 0.9402872260015117,
      "fcm_dpo/beta": 0.005528050474822521,
      "fcm_dpo/delta": 0.033387791365385056,
      "fcm_dpo/margin": 66.50111389160156,
      "fcm_dpo/q_t": 0.4158519506454468,
      "grad_norm": 15.99815845489502,
      "learning_rate": 5.573608879422875e-09,
      "logits/chosen": 0.6899482011795044,
      "logits/rejected": 0.6423511505126953,
      "logps/chosen": -213.3326416015625,
      "logps/ref_chosen": -59.86545944213867,
      "logps/ref_rejected": -81.86668395996094,
      "logps/rejected": -301.8349609375,
      "loss": 1.135,
      "margin_dpo/margin_mean": 66.50110626220703,
      "margin_dpo/margin_std": 106.6271743774414,
      "step": 622
    },
    {
      "KL/chosen_KL_mean": -162.5045928955078,
      "KL/mean": -197.54281616210938,
      "KL/rejected_KL_mean": -232.58099365234375,
      "KL/std": 105.14082336425781,
      "epoch": 0.9417989417989417,
      "fcm_dpo/beta": 0.005511360242962837,
      "fcm_dpo/delta": 0.014034271240234375,
      "fcm_dpo/margin": 70.07640075683594,
      "fcm_dpo/q_t": 0.41175198554992676,
      "grad_norm": 14.25440502166748,
      "learning_rate": 5.299388446305342e-09,
      "logits/chosen": 0.7139912843704224,
      "logits/rejected": 0.6572809219360352,
      "logps/chosen": -229.873046875,
      "logps/ref_chosen": -67.36846160888672,
      "logps/ref_rejected": -82.02733612060547,
      "logps/rejected": -314.60833740234375,
      "loss": 1.1251,
      "margin_dpo/margin_mean": 70.07640075683594,
      "margin_dpo/margin_std": 110.23150634765625,
      "step": 623
    },
    {
      "KL/chosen_KL_mean": -136.90478515625,
      "KL/mean": -175.5767822265625,
      "KL/rejected_KL_mean": -214.248779296875,
      "KL/std": 101.36776733398438,
      "epoch": 0.9433106575963719,
      "fcm_dpo/beta": 0.005502481944859028,
      "fcm_dpo/delta": -0.026918943971395493,
      "fcm_dpo/margin": 77.34397888183594,
      "fcm_dpo/q_t": 0.40445658564567566,
      "grad_norm": 13.976579666137695,
      "learning_rate": 5.03201281531429e-09,
      "logits/chosen": 0.7619487047195435,
      "logits/rejected": 0.6611573696136475,
      "logps/chosen": -187.93133544921875,
      "logps/ref_chosen": -51.02655029296875,
      "logps/ref_rejected": -76.49203491210938,
      "logps/rejected": -290.74078369140625,
      "loss": 1.0969,
      "margin_dpo/margin_mean": 77.34397888183594,
      "margin_dpo/margin_std": 113.79231262207031,
      "step": 624
    },
    {
      "KL/chosen_KL_mean": -138.4292755126953,
      "KL/mean": -168.58859252929688,
      "KL/rejected_KL_mean": -198.74790954589844,
      "KL/std": 96.82426452636719,
      "epoch": 0.9448223733938019,
      "fcm_dpo/beta": 0.005562103819102049,
      "fcm_dpo/delta": 0.0667605847120285,
      "fcm_dpo/margin": 60.318634033203125,
      "fcm_dpo/q_t": 0.42588430643081665,
      "grad_norm": 13.17888069152832,
      "learning_rate": 4.7714894655209174e-09,
      "logits/chosen": 0.8270108699798584,
      "logits/rejected": 0.7326474189758301,
      "logps/chosen": -192.63690185546875,
      "logps/ref_chosen": -54.20761489868164,
      "logps/ref_rejected": -84.93669128417969,
      "logps/rejected": -283.6846008300781,
      "loss": 1.1778,
      "margin_dpo/margin_mean": 60.31863784790039,
      "margin_dpo/margin_std": 117.32594299316406,
      "step": 625
    },
    {
      "KL/chosen_KL_mean": -131.81886291503906,
      "KL/mean": -172.271240234375,
      "KL/rejected_KL_mean": -212.72360229492188,
      "KL/std": 100.97776794433594,
      "epoch": 0.9463340891912321,
      "fcm_dpo/beta": 0.005533743184059858,
      "fcm_dpo/delta": -0.050050437450408936,
      "fcm_dpo/margin": 80.90475463867188,
      "fcm_dpo/q_t": 0.40123170614242554,
      "grad_norm": 13.59911823272705,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 0.8097890615463257,
      "logits/rejected": 0.6727601289749146,
      "logps/chosen": -176.88088989257812,
      "logps/ref_chosen": -45.06201934814453,
      "logps/ref_rejected": -89.66368103027344,
      "logps/rejected": -302.38726806640625,
      "loss": 1.0932,
      "margin_dpo/margin_mean": 80.90474700927734,
      "margin_dpo/margin_std": 123.7414321899414,
      "step": 626
    },
    {
      "KL/chosen_KL_mean": -150.9131317138672,
      "KL/mean": -193.84359741210938,
      "KL/rejected_KL_mean": -236.77407836914062,
      "KL/std": 104.66654968261719,
      "epoch": 0.9478458049886621,
      "fcm_dpo/beta": 0.00545249180868268,
      "fcm_dpo/delta": -0.07188767939805984,
      "fcm_dpo/margin": 85.86093139648438,
      "fcm_dpo/q_t": 0.3917366564273834,
      "grad_norm": 14.100014686584473,
      "learning_rate": 4.271028567242818e-09,
      "logits/chosen": 0.6746641397476196,
      "logits/rejected": 0.5507217049598694,
      "logps/chosen": -209.70419311523438,
      "logps/ref_chosen": -58.791053771972656,
      "logps/ref_rejected": -94.90802001953125,
      "logps/rejected": -331.6820983886719,
      "loss": 1.0585,
      "margin_dpo/margin_mean": 85.86093139648438,
      "margin_dpo/margin_std": 110.94627380371094,
      "step": 627
    },
    {
      "KL/chosen_KL_mean": -137.35072326660156,
      "KL/mean": -178.21011352539062,
      "KL/rejected_KL_mean": -219.06948852539062,
      "KL/std": 98.84043884277344,
      "epoch": 0.9493575207860923,
      "fcm_dpo/beta": 0.005451854318380356,
      "fcm_dpo/delta": -0.04851195216178894,
      "fcm_dpo/margin": 81.71876525878906,
      "fcm_dpo/q_t": 0.3969580829143524,
      "grad_norm": 15.095354080200195,
      "learning_rate": 4.0311050177251895e-09,
      "logits/chosen": 0.7564040422439575,
      "logits/rejected": 0.71360182762146,
      "logps/chosen": -190.154296875,
      "logps/ref_chosen": -52.80357360839844,
      "logps/ref_rejected": -76.49468994140625,
      "logps/rejected": -295.5641784667969,
      "loss": 1.0937,
      "margin_dpo/margin_mean": 81.7187728881836,
      "margin_dpo/margin_std": 113.85972595214844,
      "step": 628
    },
    {
      "KL/chosen_KL_mean": -148.08657836914062,
      "KL/mean": -178.578857421875,
      "KL/rejected_KL_mean": -209.07113647460938,
      "KL/std": 93.7113037109375,
      "epoch": 0.9508692365835223,
      "fcm_dpo/beta": 0.005464477464556694,
      "fcm_dpo/delta": 0.06876949220895767,
      "fcm_dpo/margin": 60.98457336425781,
      "fcm_dpo/q_t": 0.42434054613113403,
      "grad_norm": 12.859718322753906,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": 0.7155510187149048,
      "logits/rejected": 0.7113825082778931,
      "logps/chosen": -218.8040771484375,
      "logps/ref_chosen": -70.71749877929688,
      "logps/ref_rejected": -78.96273803710938,
      "logps/rejected": -288.03387451171875,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 60.98456954956055,
      "margin_dpo/margin_std": 98.41184997558594,
      "step": 629
    },
    {
      "KL/chosen_KL_mean": -137.65347290039062,
      "KL/mean": -179.70785522460938,
      "KL/rejected_KL_mean": -221.7622528076172,
      "KL/std": 100.41184997558594,
      "epoch": 0.9523809523809523,
      "fcm_dpo/beta": 0.005434257909655571,
      "fcm_dpo/delta": -0.0597710907459259,
      "fcm_dpo/margin": 84.10877990722656,
      "fcm_dpo/q_t": 0.39515233039855957,
      "grad_norm": 11.467988967895508,
      "learning_rate": 3.5719052736323806e-09,
      "logits/chosen": 0.6823030710220337,
      "logits/rejected": 0.6375913619995117,
      "logps/chosen": -193.85487365722656,
      "logps/ref_chosen": -56.201412200927734,
      "logps/ref_rejected": -74.69807434082031,
      "logps/rejected": -296.4603271484375,
      "loss": 1.0598,
      "margin_dpo/margin_mean": 84.1087875366211,
      "margin_dpo/margin_std": 107.00636291503906,
      "step": 630
    },
    {
      "KL/chosen_KL_mean": -132.22628784179688,
      "KL/mean": -179.32887268066406,
      "KL/rejected_KL_mean": -226.43145751953125,
      "KL/std": 106.06401062011719,
      "epoch": 0.9538926681783825,
      "fcm_dpo/beta": 0.005281176418066025,
      "fcm_dpo/delta": -0.10464085638523102,
      "fcm_dpo/margin": 94.20515441894531,
      "fcm_dpo/q_t": 0.38737034797668457,
      "grad_norm": 13.388460159301758,
      "learning_rate": 3.352641923861144e-09,
      "logits/chosen": 0.8165959119796753,
      "logits/rejected": 0.6994968056678772,
      "logps/chosen": -191.04689025878906,
      "logps/ref_chosen": -58.82059860229492,
      "logps/ref_rejected": -96.51437377929688,
      "logps/rejected": -322.9458312988281,
      "loss": 1.0421,
      "margin_dpo/margin_mean": 94.20515441894531,
      "margin_dpo/margin_std": 116.7836685180664,
      "step": 631
    },
    {
      "KL/chosen_KL_mean": -134.43540954589844,
      "KL/mean": -178.71347045898438,
      "KL/rejected_KL_mean": -222.99151611328125,
      "KL/std": 94.78572845458984,
      "epoch": 0.9554043839758125,
      "fcm_dpo/beta": 0.005254029296338558,
      "fcm_dpo/delta": -0.06842543184757233,
      "fcm_dpo/margin": 88.55610656738281,
      "fcm_dpo/q_t": 0.39199209213256836,
      "grad_norm": 11.932879447937012,
      "learning_rate": 3.140277830901428e-09,
      "logits/chosen": 0.7434148788452148,
      "logits/rejected": 0.7235583066940308,
      "logps/chosen": -193.22146606445312,
      "logps/ref_chosen": -58.786048889160156,
      "logps/ref_rejected": -67.21923828125,
      "logps/rejected": -290.21075439453125,
      "loss": 1.0471,
      "margin_dpo/margin_mean": 88.55609893798828,
      "margin_dpo/margin_std": 105.25202941894531,
      "step": 632
    },
    {
      "KL/chosen_KL_mean": -127.76835632324219,
      "KL/mean": -162.04751586914062,
      "KL/rejected_KL_mean": -196.32669067382812,
      "KL/std": 90.17171478271484,
      "epoch": 0.9569160997732427,
      "fcm_dpo/beta": 0.005269904620945454,
      "fcm_dpo/delta": 0.03980453684926033,
      "fcm_dpo/margin": 68.55831909179688,
      "fcm_dpo/q_t": 0.41761648654937744,
      "grad_norm": 13.597479820251465,
      "learning_rate": 2.9348189350335007e-09,
      "logits/chosen": 0.7004154324531555,
      "logits/rejected": 0.6402037143707275,
      "logps/chosen": -179.8985595703125,
      "logps/ref_chosen": -52.13019561767578,
      "logps/ref_rejected": -67.23016357421875,
      "logps/rejected": -263.5568542480469,
      "loss": 1.1284,
      "margin_dpo/margin_mean": 68.5583267211914,
      "margin_dpo/margin_std": 103.29327392578125,
      "step": 633
    },
    {
      "KL/chosen_KL_mean": -156.1773223876953,
      "KL/mean": -170.20523071289062,
      "KL/rejected_KL_mean": -184.233154296875,
      "KL/std": 93.95570373535156,
      "epoch": 0.9584278155706727,
      "fcm_dpo/beta": 0.005263281520456076,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 28.05582046508789,
      "fcm_dpo/q_t": 0.46669408679008484,
      "grad_norm": 16.815614700317383,
      "learning_rate": 2.736270983384276e-09,
      "logits/chosen": 0.8039923310279846,
      "logits/rejected": 0.8182891607284546,
      "logps/chosen": -217.15711975097656,
      "logps/ref_chosen": -60.97979736328125,
      "logps/ref_rejected": -58.50825119018555,
      "logps/rejected": -242.74139404296875,
      "loss": 1.3224,
      "margin_dpo/margin_mean": 28.05582046508789,
      "margin_dpo/margin_std": 108.4864273071289,
      "step": 634
    },
    {
      "KL/chosen_KL_mean": -171.27197265625,
      "KL/mean": -199.12741088867188,
      "KL/rejected_KL_mean": -226.98287963867188,
      "KL/std": 94.59959411621094,
      "epoch": 0.9599395313681028,
      "fcm_dpo/beta": 0.0053599514067173,
      "fcm_dpo/delta": 0.10409200191497803,
      "fcm_dpo/margin": 55.71092224121094,
      "fcm_dpo/q_t": 0.4323340654373169,
      "grad_norm": 13.661896705627441,
      "learning_rate": 2.5446395297668287e-09,
      "logits/chosen": 0.5933520197868347,
      "logits/rejected": 0.5343912243843079,
      "logps/chosen": -237.24497985839844,
      "logps/ref_chosen": -65.9730224609375,
      "logps/ref_rejected": -85.61317443847656,
      "logps/rejected": -312.5960693359375,
      "loss": 1.2028,
      "margin_dpo/margin_mean": 55.71092224121094,
      "margin_dpo/margin_std": 115.75862121582031,
      "step": 635
    },
    {
      "KL/chosen_KL_mean": -133.72576904296875,
      "KL/mean": -171.8177490234375,
      "KL/rejected_KL_mean": -209.9097442626953,
      "KL/std": 94.93038940429688,
      "epoch": 0.9614512471655329,
      "fcm_dpo/beta": 0.005354847759008408,
      "fcm_dpo/delta": -0.008421150967478752,
      "fcm_dpo/margin": 76.18397521972656,
      "fcm_dpo/q_t": 0.40504512190818787,
      "grad_norm": 11.020319938659668,
      "learning_rate": 2.359929934524829e-09,
      "logits/chosen": 0.7218972444534302,
      "logits/rejected": 0.6205803155899048,
      "logps/chosen": -182.86593627929688,
      "logps/ref_chosen": -49.140167236328125,
      "logps/ref_rejected": -81.26971435546875,
      "logps/rejected": -291.179443359375,
      "loss": 1.0889,
      "margin_dpo/margin_mean": 76.18397521972656,
      "margin_dpo/margin_std": 100.74605560302734,
      "step": 636
    },
    {
      "KL/chosen_KL_mean": -177.77621459960938,
      "KL/mean": -205.72189331054688,
      "KL/rejected_KL_mean": -233.66754150390625,
      "KL/std": 93.27995300292969,
      "epoch": 0.9629629629629629,
      "fcm_dpo/beta": 0.005460776388645172,
      "fcm_dpo/delta": 0.09725769609212875,
      "fcm_dpo/margin": 55.891334533691406,
      "fcm_dpo/q_t": 0.43144917488098145,
      "grad_norm": 15.637556076049805,
      "learning_rate": 2.1821473643827137e-09,
      "logits/chosen": 0.7164098024368286,
      "logits/rejected": 0.6493207216262817,
      "logps/chosen": -251.47279357910156,
      "logps/ref_chosen": -73.69658660888672,
      "logps/ref_rejected": -83.01487731933594,
      "logps/rejected": -316.68243408203125,
      "loss": 1.2024,
      "margin_dpo/margin_mean": 55.891334533691406,
      "margin_dpo/margin_std": 118.63633728027344,
      "step": 637
    },
    {
      "KL/chosen_KL_mean": -155.15830993652344,
      "KL/mean": -188.7733154296875,
      "KL/rejected_KL_mean": -222.38832092285156,
      "KL/std": 94.15514373779297,
      "epoch": 0.9644746787603931,
      "fcm_dpo/beta": 0.005497739650309086,
      "fcm_dpo/delta": 0.03154220059514046,
      "fcm_dpo/margin": 67.22999572753906,
      "fcm_dpo/q_t": 0.4159843921661377,
      "grad_norm": 13.306710243225098,
      "learning_rate": 2.0112967923011646e-09,
      "logits/chosen": 0.7447936534881592,
      "logits/rejected": 0.6957993507385254,
      "logps/chosen": -217.93988037109375,
      "logps/ref_chosen": -62.78158187866211,
      "logps/ref_rejected": -85.40478515625,
      "logps/rejected": -307.7930908203125,
      "loss": 1.1272,
      "margin_dpo/margin_mean": 67.22999572753906,
      "margin_dpo/margin_std": 103.98416137695312,
      "step": 638
    },
    {
      "KL/chosen_KL_mean": -137.9173583984375,
      "KL/mean": -178.12657165527344,
      "KL/rejected_KL_mean": -218.33580017089844,
      "KL/std": 101.79522705078125,
      "epoch": 0.9659863945578231,
      "fcm_dpo/beta": 0.005474994890391827,
      "fcm_dpo/delta": -0.04212556779384613,
      "fcm_dpo/margin": 80.41844940185547,
      "fcm_dpo/q_t": 0.4007149934768677,
      "grad_norm": 14.170092582702637,
      "learning_rate": 1.847382997337943e-09,
      "logits/chosen": 0.7185821533203125,
      "logits/rejected": 0.6204472780227661,
      "logps/chosen": -191.68392944335938,
      "logps/ref_chosen": -53.76658630371094,
      "logps/ref_rejected": -72.30009460449219,
      "logps/rejected": -290.6358947753906,
      "loss": 1.0804,
      "margin_dpo/margin_mean": 80.41845703125,
      "margin_dpo/margin_std": 111.70866394042969,
      "step": 639
    },
    {
      "KL/chosen_KL_mean": -144.30633544921875,
      "KL/mean": -180.83792114257812,
      "KL/rejected_KL_mean": -217.36947631835938,
      "KL/std": 97.84783935546875,
      "epoch": 0.9674981103552532,
      "fcm_dpo/beta": 0.005446083843708038,
      "fcm_dpo/delta": 0.0020423419773578644,
      "fcm_dpo/margin": 73.06314086914062,
      "fcm_dpo/q_t": 0.40859144926071167,
      "grad_norm": 12.568567276000977,
      "learning_rate": 1.690410564514244e-09,
      "logits/chosen": 0.7867799997329712,
      "logits/rejected": 0.7209002375602722,
      "logps/chosen": -195.72412109375,
      "logps/ref_chosen": -51.41777801513672,
      "logps/ref_rejected": -77.27879333496094,
      "logps/rejected": -294.64825439453125,
      "loss": 1.1067,
      "margin_dpo/margin_mean": 73.06314086914062,
      "margin_dpo/margin_std": 106.24955749511719,
      "step": 640
    },
    {
      "KL/chosen_KL_mean": -152.33514404296875,
      "KL/mean": -184.97947692871094,
      "KL/rejected_KL_mean": -217.6238250732422,
      "KL/std": 91.95304107666016,
      "epoch": 0.9690098261526833,
      "fcm_dpo/beta": 0.005483964458107948,
      "fcm_dpo/delta": 0.04350946471095085,
      "fcm_dpo/margin": 65.2886962890625,
      "fcm_dpo/q_t": 0.4163803160190582,
      "grad_norm": 13.97402286529541,
      "learning_rate": 1.5403838846864692e-09,
      "logits/chosen": 0.6977224349975586,
      "logits/rejected": 0.6800275444984436,
      "logps/chosen": -223.38980102539062,
      "logps/ref_chosen": -71.0546646118164,
      "logps/ref_rejected": -82.2440185546875,
      "logps/rejected": -299.86785888671875,
      "loss": 1.1159,
      "margin_dpo/margin_mean": 65.2886962890625,
      "margin_dpo/margin_std": 88.158935546875,
      "step": 641
    },
    {
      "KL/chosen_KL_mean": -161.3411865234375,
      "KL/mean": -185.21339416503906,
      "KL/rejected_KL_mean": -209.08560180664062,
      "KL/std": 88.95892333984375,
      "epoch": 0.9705215419501134,
      "fcm_dpo/beta": 0.005519367288798094,
      "fcm_dpo/delta": 0.009645511396229267,
      "fcm_dpo/margin": 47.74443054199219,
      "fcm_dpo/q_t": 0.4397560954093933,
      "grad_norm": 15.762700080871582,
      "learning_rate": 1.3973071544233218e-09,
      "logits/chosen": 0.6300410032272339,
      "logits/rejected": 0.6499575972557068,
      "logps/chosen": -230.27044677734375,
      "logps/ref_chosen": -68.92927551269531,
      "logps/ref_rejected": -70.85682678222656,
      "logps/rejected": -279.94244384765625,
      "loss": 1.2279,
      "margin_dpo/margin_mean": 47.74443054199219,
      "margin_dpo/margin_std": 107.56587219238281,
      "step": 642
    },
    {
      "KL/chosen_KL_mean": -144.30145263671875,
      "KL/mean": -181.84938049316406,
      "KL/rejected_KL_mean": -219.39732360839844,
      "KL/std": 95.93026733398438,
      "epoch": 0.9720332577475435,
      "fcm_dpo/beta": 0.005500371567904949,
      "fcm_dpo/delta": -0.013677622191607952,
      "fcm_dpo/margin": 75.09587097167969,
      "fcm_dpo/q_t": 0.40612655878067017,
      "grad_norm": 19.423891067504883,
      "learning_rate": 1.261184375888541e-09,
      "logits/chosen": 0.656221330165863,
      "logits/rejected": 0.5695391893386841,
      "logps/chosen": -209.6104736328125,
      "logps/ref_chosen": -65.30903625488281,
      "logps/ref_rejected": -83.61613464355469,
      "logps/rejected": -303.0134582519531,
      "loss": 1.108,
      "margin_dpo/margin_mean": 75.09587097167969,
      "margin_dpo/margin_std": 113.25881958007812,
      "step": 643
    },
    {
      "KL/chosen_KL_mean": -136.59500122070312,
      "KL/mean": -163.04470825195312,
      "KL/rejected_KL_mean": -189.49440002441406,
      "KL/std": 87.87973022460938,
      "epoch": 0.9735449735449735,
      "fcm_dpo/beta": 0.005500611383467913,
      "fcm_dpo/delta": -0.006686890963464975,
      "fcm_dpo/margin": 52.899391174316406,
      "fcm_dpo/q_t": 0.43249207735061646,
      "grad_norm": 12.692774772644043,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": 0.7841014862060547,
      "logits/rejected": 0.7521142363548279,
      "logps/chosen": -187.5976104736328,
      "logps/ref_chosen": -51.002601623535156,
      "logps/ref_rejected": -64.46372985839844,
      "logps/rejected": -253.9581298828125,
      "loss": 1.2057,
      "margin_dpo/margin_mean": 52.899391174316406,
      "margin_dpo/margin_std": 108.9871826171875,
      "step": 644
    },
    {
      "KL/chosen_KL_mean": -147.3602752685547,
      "KL/mean": -180.54623413085938,
      "KL/rejected_KL_mean": -213.732177734375,
      "KL/std": 93.51528930664062,
      "epoch": 0.9750566893424036,
      "fcm_dpo/beta": 0.005523581989109516,
      "fcm_dpo/delta": 0.03466928005218506,
      "fcm_dpo/margin": 66.37191009521484,
      "fcm_dpo/q_t": 0.41463106870651245,
      "grad_norm": 14.304610252380371,
      "learning_rate": 1.0098157099674987e-09,
      "logits/chosen": 0.6800580620765686,
      "logits/rejected": 0.6638115644454956,
      "logps/chosen": -208.3236846923828,
      "logps/ref_chosen": -60.963409423828125,
      "logps/ref_rejected": -69.73353576660156,
      "logps/rejected": -283.4656982421875,
      "loss": 1.1165,
      "margin_dpo/margin_mean": 66.37191009521484,
      "margin_dpo/margin_std": 93.798828125,
      "step": 645
    },
    {
      "KL/chosen_KL_mean": -163.3536376953125,
      "KL/mean": -192.60321044921875,
      "KL/rejected_KL_mean": -221.852783203125,
      "KL/std": 104.98648834228516,
      "epoch": 0.9765684051398337,
      "fcm_dpo/beta": 0.005599304102361202,
      "fcm_dpo/delta": 0.07487069070339203,
      "fcm_dpo/margin": 58.4991455078125,
      "fcm_dpo/q_t": 0.425899475812912,
      "grad_norm": 12.293761253356934,
      "learning_rate": 8.945768539031783e-10,
      "logits/chosen": 0.7638056874275208,
      "logits/rejected": 0.7025067806243896,
      "logps/chosen": -225.64370727539062,
      "logps/ref_chosen": -62.290069580078125,
      "logps/ref_rejected": -85.54812622070312,
      "logps/rejected": -307.4009094238281,
      "loss": 1.1757,
      "margin_dpo/margin_mean": 58.4991455078125,
      "margin_dpo/margin_std": 110.7708740234375,
      "step": 646
    },
    {
      "KL/chosen_KL_mean": -160.01625061035156,
      "KL/mean": -207.0872344970703,
      "KL/rejected_KL_mean": -254.158203125,
      "KL/std": 105.69502258300781,
      "epoch": 0.9780801209372638,
      "fcm_dpo/beta": 0.005524728447198868,
      "fcm_dpo/delta": -0.12653151154518127,
      "fcm_dpo/margin": 94.14195251464844,
      "fcm_dpo/q_t": 0.38018798828125,
      "grad_norm": 14.664588928222656,
      "learning_rate": 7.863060120144316e-10,
      "logits/chosen": 0.7419285774230957,
      "logits/rejected": 0.6430102586746216,
      "logps/chosen": -227.53213500976562,
      "logps/ref_chosen": -67.515869140625,
      "logps/ref_rejected": -101.50871276855469,
      "logps/rejected": -355.66693115234375,
      "loss": 1.0092,
      "margin_dpo/margin_mean": 94.14195251464844,
      "margin_dpo/margin_std": 103.87393188476562,
      "step": 647
    },
    {
      "KL/chosen_KL_mean": -158.1337432861328,
      "KL/mean": -191.36920166015625,
      "KL/rejected_KL_mean": -224.6046905517578,
      "KL/std": 92.80807495117188,
      "epoch": 0.9795918367346939,
      "fcm_dpo/beta": 0.0054849558509886265,
      "fcm_dpo/delta": 0.03653711825609207,
      "fcm_dpo/margin": 66.47093200683594,
      "fcm_dpo/q_t": 0.4177197217941284,
      "grad_norm": 14.50339412689209,
      "learning_rate": 6.850062128694045e-10,
      "logits/chosen": 0.6602978706359863,
      "logits/rejected": 0.5971434116363525,
      "logps/chosen": -222.72967529296875,
      "logps/ref_chosen": -64.59593963623047,
      "logps/ref_rejected": -83.384033203125,
      "logps/rejected": -307.98870849609375,
      "loss": 1.1602,
      "margin_dpo/margin_mean": 66.47093200683594,
      "margin_dpo/margin_std": 119.66780090332031,
      "step": 648
    },
    {
      "KL/chosen_KL_mean": -145.18980407714844,
      "KL/mean": -179.31549072265625,
      "KL/rejected_KL_mean": -213.44117736816406,
      "KL/std": 92.50328826904297,
      "epoch": 0.981103552532124,
      "fcm_dpo/beta": 0.005515716038644314,
      "fcm_dpo/delta": 0.024219004437327385,
      "fcm_dpo/margin": 68.25140380859375,
      "fcm_dpo/q_t": 0.4137144982814789,
      "grad_norm": 16.997325897216797,
      "learning_rate": 5.906802900412788e-10,
      "logits/chosen": 0.7402960062026978,
      "logits/rejected": 0.678533136844635,
      "logps/chosen": -194.49945068359375,
      "logps/ref_chosen": -49.30964660644531,
      "logps/ref_rejected": -73.73710632324219,
      "logps/rejected": -287.17828369140625,
      "loss": 1.1374,
      "margin_dpo/margin_mean": 68.25140380859375,
      "margin_dpo/margin_std": 112.25713348388672,
      "step": 649
    },
    {
      "KL/chosen_KL_mean": -141.2061309814453,
      "KL/mean": -175.8365936279297,
      "KL/rejected_KL_mean": -210.46705627441406,
      "KL/std": 93.37223815917969,
      "epoch": 0.982615268329554,
      "fcm_dpo/beta": 0.005547208711504936,
      "fcm_dpo/delta": 0.016371339559555054,
      "fcm_dpo/margin": 69.26091003417969,
      "fcm_dpo/q_t": 0.4124113619327545,
      "grad_norm": 13.100647926330566,
      "learning_rate": 5.033308820289184e-10,
      "logits/chosen": 0.7958236932754517,
      "logits/rejected": 0.7285829186439514,
      "logps/chosen": -196.26939392089844,
      "logps/ref_chosen": -55.06325912475586,
      "logps/ref_rejected": -77.39610290527344,
      "logps/rejected": -287.8631591796875,
      "loss": 1.1346,
      "margin_dpo/margin_mean": 69.26091003417969,
      "margin_dpo/margin_std": 114.14212799072266,
      "step": 650
    },
    {
      "KL/chosen_KL_mean": -140.88888549804688,
      "KL/mean": -170.99203491210938,
      "KL/rejected_KL_mean": -201.09519958496094,
      "KL/std": 94.2713851928711,
      "epoch": 0.9841269841269841,
      "fcm_dpo/beta": 0.005626247264444828,
      "fcm_dpo/delta": 0.0632336363196373,
      "fcm_dpo/margin": 60.20629119873047,
      "fcm_dpo/q_t": 0.42402487993240356,
      "grad_norm": 12.568854331970215,
      "learning_rate": 4.2296043218295606e-10,
      "logits/chosen": 0.7908520698547363,
      "logits/rejected": 0.7132381200790405,
      "logps/chosen": -194.9540557861328,
      "logps/ref_chosen": -54.065162658691406,
      "logps/ref_rejected": -77.79080200195312,
      "logps/rejected": -278.885986328125,
      "loss": 1.1529,
      "margin_dpo/margin_mean": 60.20629119873047,
      "margin_dpo/margin_std": 101.39066314697266,
      "step": 651
    },
    {
      "KL/chosen_KL_mean": -165.57644653320312,
      "KL/mean": -194.77255249023438,
      "KL/rejected_KL_mean": -223.96865844726562,
      "KL/std": 101.62708282470703,
      "epoch": 0.9856386999244142,
      "fcm_dpo/beta": 0.005613422952592373,
      "fcm_dpo/delta": -0.01851782761514187,
      "fcm_dpo/margin": 58.39221954345703,
      "fcm_dpo/q_t": 0.42721042037010193,
      "grad_norm": 13.84463119506836,
      "learning_rate": 3.4957118863768176e-10,
      "logits/chosen": 0.7539942264556885,
      "logits/rejected": 0.7029706239700317,
      "logps/chosen": -229.21673583984375,
      "logps/ref_chosen": -63.64030456542969,
      "logps/ref_rejected": -78.86882019042969,
      "logps/rejected": -302.8374938964844,
      "loss": 1.1948,
      "margin_dpo/margin_mean": 58.39221954345703,
      "margin_dpo/margin_std": 121.28067016601562,
      "step": 652
    },
    {
      "KL/chosen_KL_mean": -147.64129638671875,
      "KL/mean": -181.91357421875,
      "KL/rejected_KL_mean": -216.18582153320312,
      "KL/std": 92.51074981689453,
      "epoch": 0.9871504157218443,
      "fcm_dpo/beta": 0.0056394971907138824,
      "fcm_dpo/delta": 0.013813050463795662,
      "fcm_dpo/margin": 68.54454040527344,
      "fcm_dpo/q_t": 0.41165587306022644,
      "grad_norm": 14.583548545837402,
      "learning_rate": 2.831652042480093e-10,
      "logits/chosen": 0.7122687697410583,
      "logits/rejected": 0.6665648818016052,
      "logps/chosen": -209.30966186523438,
      "logps/ref_chosen": -61.668373107910156,
      "logps/ref_rejected": -73.83012390136719,
      "logps/rejected": -290.0159606933594,
      "loss": 1.128,
      "margin_dpo/margin_mean": 68.54454040527344,
      "margin_dpo/margin_std": 109.81883239746094,
      "step": 653
    },
    {
      "KL/chosen_KL_mean": -153.53273010253906,
      "KL/mean": -184.33224487304688,
      "KL/rejected_KL_mean": -215.13174438476562,
      "KL/std": 106.68663024902344,
      "epoch": 0.9886621315192744,
      "fcm_dpo/beta": 0.005561579950153828,
      "fcm_dpo/delta": -0.06020544096827507,
      "fcm_dpo/margin": 61.598995208740234,
      "fcm_dpo/q_t": 0.4215339124202728,
      "grad_norm": 13.458853721618652,
      "learning_rate": 2.2374433653205016e-10,
      "logits/chosen": 0.7000631093978882,
      "logits/rejected": 0.5972336530685425,
      "logps/chosen": -211.1009979248047,
      "logps/ref_chosen": -57.568267822265625,
      "logps/ref_rejected": -87.74789428710938,
      "logps/rejected": -302.879638671875,
      "loss": 1.1535,
      "margin_dpo/margin_mean": 61.5989990234375,
      "margin_dpo/margin_std": 99.90827941894531,
      "step": 654
    },
    {
      "KL/chosen_KL_mean": -120.32594299316406,
      "KL/mean": -167.99766540527344,
      "KL/rejected_KL_mean": -215.66940307617188,
      "KL/std": 95.54719543457031,
      "epoch": 0.9901738473167044,
      "fcm_dpo/beta": 0.005412455648183823,
      "fcm_dpo/delta": -0.12495169043540955,
      "fcm_dpo/margin": 95.34344482421875,
      "fcm_dpo/q_t": 0.37979695200920105,
      "grad_norm": 11.774541854858398,
      "learning_rate": 1.7131024761923852e-10,
      "logits/chosen": 0.7104381322860718,
      "logits/rejected": 0.6201817989349365,
      "logps/chosen": -172.47308349609375,
      "logps/ref_chosen": -52.14714813232422,
      "logps/ref_rejected": -80.85014343261719,
      "logps/rejected": -296.51953125,
      "loss": 0.9992,
      "margin_dpo/margin_mean": 95.34344482421875,
      "margin_dpo/margin_std": 92.38569641113281,
      "step": 655
    },
    {
      "KL/chosen_KL_mean": -157.343505859375,
      "KL/mean": -192.9163818359375,
      "KL/rejected_KL_mean": -228.48927307128906,
      "KL/std": 96.09796905517578,
      "epoch": 0.9916855631141346,
      "fcm_dpo/beta": 0.005436629056930542,
      "fcm_dpo/delta": 0.013734135776758194,
      "fcm_dpo/margin": 71.14578247070312,
      "fcm_dpo/q_t": 0.41111651062965393,
      "grad_norm": 10.782207489013672,
      "learning_rate": 1.2586440420372934e-10,
      "logits/chosen": 0.6520262956619263,
      "logits/rejected": 0.6041054725646973,
      "logps/chosen": -230.6002197265625,
      "logps/ref_chosen": -73.25672912597656,
      "logps/ref_rejected": -85.35127258300781,
      "logps/rejected": -313.8405456542969,
      "loss": 1.1172,
      "margin_dpo/margin_mean": 71.14578247070312,
      "margin_dpo/margin_std": 107.85639953613281,
      "step": 656
    },
    {
      "KL/chosen_KL_mean": -141.95278930664062,
      "KL/mean": -183.90524291992188,
      "KL/rejected_KL_mean": -225.85768127441406,
      "KL/std": 106.96556091308594,
      "epoch": 0.9931972789115646,
      "fcm_dpo/beta": 0.005397680681198835,
      "fcm_dpo/delta": -0.05537598580121994,
      "fcm_dpo/margin": 83.90487670898438,
      "fcm_dpo/q_t": 0.3969414234161377,
      "grad_norm": 10.755281448364258,
      "learning_rate": 8.740807750345913e-11,
      "logits/chosen": 0.845678448677063,
      "logits/rejected": 0.7539495825767517,
      "logps/chosen": -191.6761932373047,
      "logps/ref_chosen": -49.72339630126953,
      "logps/ref_rejected": -75.1568603515625,
      "logps/rejected": -301.0145263671875,
      "loss": 1.0785,
      "margin_dpo/margin_mean": 83.90487670898438,
      "margin_dpo/margin_std": 118.01260375976562,
      "step": 657
    },
    {
      "KL/chosen_KL_mean": -149.97967529296875,
      "KL/mean": -183.7830810546875,
      "KL/rejected_KL_mean": -217.5864715576172,
      "KL/std": 107.20687866210938,
      "epoch": 0.9947089947089947,
      "fcm_dpo/beta": 0.005438681226223707,
      "fcm_dpo/delta": 0.03258253261446953,
      "fcm_dpo/margin": 67.60679626464844,
      "fcm_dpo/q_t": 0.41690129041671753,
      "grad_norm": 11.799734115600586,
      "learning_rate": 5.594234322453539e-11,
      "logits/chosen": 0.772599995136261,
      "logits/rejected": 0.7247228622436523,
      "logps/chosen": -213.02601623535156,
      "logps/ref_chosen": -63.04634094238281,
      "logps/ref_rejected": -83.44963073730469,
      "logps/rejected": -301.0361022949219,
      "loss": 1.1637,
      "margin_dpo/margin_mean": 67.60679626464844,
      "margin_dpo/margin_std": 125.44908142089844,
      "step": 658
    },
    {
      "KL/chosen_KL_mean": -153.94338989257812,
      "KL/mean": -180.20620727539062,
      "KL/rejected_KL_mean": -206.46902465820312,
      "KL/std": 96.60218811035156,
      "epoch": 0.9962207105064248,
      "fcm_dpo/beta": 0.005414203740656376,
      "fcm_dpo/delta": -0.0002711827401071787,
      "fcm_dpo/margin": 52.52562713623047,
      "fcm_dpo/q_t": 0.4343125522136688,
      "grad_norm": 17.4157657623291,
      "learning_rate": 3.146808153123293e-11,
      "logits/chosen": 0.8189652562141418,
      "logits/rejected": 0.7509829998016357,
      "logps/chosen": -209.02359008789062,
      "logps/ref_chosen": -55.0802001953125,
      "logps/ref_rejected": -71.91049194335938,
      "logps/rejected": -278.3795166015625,
      "loss": 1.2126,
      "margin_dpo/margin_mean": 52.52562713623047,
      "margin_dpo/margin_std": 111.74625396728516,
      "step": 659
    },
    {
      "KL/chosen_KL_mean": -145.05416870117188,
      "KL/mean": -187.25155639648438,
      "KL/rejected_KL_mean": -229.44894409179688,
      "KL/std": 102.2318344116211,
      "epoch": 0.9977324263038548,
      "fcm_dpo/beta": 0.005373704247176647,
      "fcm_dpo/delta": -0.05602237209677696,
      "fcm_dpo/margin": 84.39476013183594,
      "fcm_dpo/q_t": 0.3970376253128052,
      "grad_norm": 12.2911958694458,
      "learning_rate": 1.3985977021235829e-11,
      "logits/chosen": 0.8505597114562988,
      "logits/rejected": 0.7764712572097778,
      "logps/chosen": -199.580078125,
      "logps/ref_chosen": -54.525917053222656,
      "logps/ref_rejected": -81.23604583740234,
      "logps/rejected": -310.68499755859375,
      "loss": 1.0603,
      "margin_dpo/margin_mean": 84.39476013183594,
      "margin_dpo/margin_std": 108.99366760253906,
      "step": 660
    },
    {
      "KL/chosen_KL_mean": -166.950439453125,
      "KL/mean": -194.38804626464844,
      "KL/rejected_KL_mean": -221.82568359375,
      "KL/std": 104.34062194824219,
      "epoch": 0.999244142101285,
      "fcm_dpo/beta": 0.005447630304843187,
      "fcm_dpo/delta": 0.10391321033239365,
      "fcm_dpo/margin": 54.875244140625,
      "fcm_dpo/q_t": 0.4321562647819519,
      "grad_norm": 15.33483600616455,
      "learning_rate": 3.4965187065971735e-12,
      "logits/chosen": 0.7138886451721191,
      "logits/rejected": 0.6324626803398132,
      "logps/chosen": -227.3230743408203,
      "logps/ref_chosen": -60.37263870239258,
      "logps/ref_rejected": -77.42874145507812,
      "logps/rejected": -299.2544250488281,
      "loss": 1.2112,
      "margin_dpo/margin_mean": 54.875244140625,
      "margin_dpo/margin_std": 120.78031158447266,
      "step": 661
    },
    {
      "epoch": 0.999244142101285,
      "step": 661,
      "total_flos": 0.0,
      "train_loss": 1.1404347123068148,
      "train_runtime": 1649.8929,
      "train_samples_per_second": 25.66,
      "train_steps_per_second": 0.401
    }
  ],
  "logging_steps": 1,
  "max_steps": 661,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}