{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 200,
  "global_step": 681,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0014684287812041115,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.02287006378173828,
      "fcm_dpo/q_t": 0.5005706548690796,
      "grad_norm": 83.51022338867188,
      "learning_rate": 0.0,
      "logits/chosen": -0.4974287748336792,
      "logits/rejected": -0.43299180269241333,
      "logps/chosen": -50.1435661315918,
      "logps/ref_chosen": -50.14883804321289,
      "logps/ref_rejected": -74.1280517578125,
      "logps/rejected": -74.09991455078125,
      "loss": 1.389,
      "margin_dpo/margin_mean": -0.02287048101425171,
      "margin_dpo/margin_std": 0.41920793056488037,
      "step": 1
    },
    {
      "epoch": 0.007342143906020558,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.007774516940116882,
      "fcm_dpo/q_t": 0.500194251537323,
      "grad_norm": 89.51734924316406,
      "learning_rate": 2.898550724637681e-08,
      "logits/chosen": -0.4900396764278412,
      "logits/rejected": -0.4533483386039734,
      "logps/chosen": -56.05889892578125,
      "logps/ref_chosen": -56.05734634399414,
      "logps/ref_rejected": -78.69325256347656,
      "logps/rejected": -78.68702697753906,
      "loss": 1.3874,
      "margin_dpo/margin_mean": -0.007774434983730316,
      "margin_dpo/margin_std": 0.37205374240875244,
      "step": 5
    },
    {
      "epoch": 0.014684287812041116,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.017793914303183556,
      "fcm_dpo/q_t": 0.49955543875694275,
      "grad_norm": 70.37712097167969,
      "learning_rate": 6.521739130434782e-08,
      "logits/chosen": -0.49457064270973206,
      "logits/rejected": -0.4558273255825043,
      "logps/chosen": -59.53853225708008,
      "logps/ref_chosen": -59.54457473754883,
      "logps/ref_rejected": -91.17041778564453,
      "logps/rejected": -91.18217468261719,
      "loss": 1.3849,
      "margin_dpo/margin_mean": 0.017794013023376465,
      "margin_dpo/margin_std": 0.40502458810806274,
      "step": 10
    },
    {
      "epoch": 0.022026431718061675,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.013358664698898792,
      "fcm_dpo/q_t": 0.4996658265590668,
      "grad_norm": 64.22943878173828,
      "learning_rate": 1.0144927536231885e-07,
      "logits/chosen": -0.48777151107788086,
      "logits/rejected": -0.4582846164703369,
      "logps/chosen": -58.85475540161133,
      "logps/ref_chosen": -58.83195877075195,
      "logps/ref_rejected": -92.93949890136719,
      "logps/rejected": -92.97565460205078,
      "loss": 1.3854,
      "margin_dpo/margin_mean": 0.01335877738893032,
      "margin_dpo/margin_std": 0.3935544788837433,
      "step": 15
    },
    {
      "epoch": 0.02936857562408223,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09099045395851135,
      "fcm_dpo/q_t": 0.49772581458091736,
      "grad_norm": 73.8072509765625,
      "learning_rate": 1.3768115942028986e-07,
      "logits/chosen": -0.5034265518188477,
      "logits/rejected": -0.45439839363098145,
      "logps/chosen": -59.618431091308594,
      "logps/ref_chosen": -59.6396598815918,
      "logps/ref_rejected": -82.76026916503906,
      "logps/rejected": -82.83003234863281,
      "loss": 1.3775,
      "margin_dpo/margin_mean": 0.09099055081605911,
      "margin_dpo/margin_std": 0.35805386304855347,
      "step": 20
    },
    {
      "epoch": 0.03671071953010279,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.21624115109443665,
      "fcm_dpo/q_t": 0.49459710717201233,
      "grad_norm": 73.70997619628906,
      "learning_rate": 1.7391304347826085e-07,
      "logits/chosen": -0.5030972361564636,
      "logits/rejected": -0.47600775957107544,
      "logps/chosen": -53.1678352355957,
      "logps/ref_chosen": -53.205284118652344,
      "logps/ref_rejected": -88.99608612060547,
      "logps/rejected": -89.17488098144531,
      "loss": 1.3652,
      "margin_dpo/margin_mean": 0.21624140441417694,
      "margin_dpo/margin_std": 0.4134984016418457,
      "step": 25
    },
    {
      "epoch": 0.04405286343612335,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4783309996128082,
      "fcm_dpo/q_t": 0.4880555272102356,
      "grad_norm": 87.57738494873047,
      "learning_rate": 2.1014492753623187e-07,
      "logits/chosen": -0.5126634240150452,
      "logits/rejected": -0.4724017083644867,
      "logps/chosen": -53.46284103393555,
      "logps/ref_chosen": -53.5526008605957,
      "logps/ref_rejected": -97.87371826171875,
      "logps/rejected": -98.26229858398438,
      "loss": 1.3399,
      "margin_dpo/margin_mean": 0.47833117842674255,
      "margin_dpo/margin_std": 0.5532158017158508,
      "step": 30
    },
    {
      "epoch": 0.0513950073421439,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7770208120346069,
      "fcm_dpo/q_t": 0.48065242171287537,
      "grad_norm": 82.80741119384766,
      "learning_rate": 2.463768115942029e-07,
      "logits/chosen": -0.5122802257537842,
      "logits/rejected": -0.4792337417602539,
      "logps/chosen": -56.210533142089844,
      "logps/ref_chosen": -56.3298454284668,
      "logps/ref_rejected": -91.76858520507812,
      "logps/rejected": -92.4262924194336,
      "loss": 1.3121,
      "margin_dpo/margin_mean": 0.7770206332206726,
      "margin_dpo/margin_std": 0.8431359529495239,
      "step": 35
    },
    {
      "epoch": 0.05873715124816446,
      "fcm_dpo/beta": 0.10000000894069672,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3613355159759521,
      "fcm_dpo/q_t": 0.46629637479782104,
      "grad_norm": 60.205745697021484,
      "learning_rate": 2.8260869565217386e-07,
      "logits/chosen": -0.5219647884368896,
      "logits/rejected": -0.4850676953792572,
      "logps/chosen": -54.278953552246094,
      "logps/ref_chosen": -54.38492965698242,
      "logps/ref_rejected": -82.94353485107422,
      "logps/rejected": -84.19891357421875,
      "loss": 1.2605,
      "margin_dpo/margin_mean": 1.3613355159759521,
      "margin_dpo/margin_std": 1.4631270170211792,
      "step": 40
    },
    {
      "epoch": 0.06607929515418502,
      "fcm_dpo/beta": 0.10669933259487152,
      "fcm_dpo/delta": 0.12330988794565201,
      "fcm_dpo/margin": 2.4058146476745605,
      "fcm_dpo/q_t": 0.4380177855491638,
      "grad_norm": 68.1404800415039,
      "learning_rate": 3.188405797101449e-07,
      "logits/chosen": -0.5121681094169617,
      "logits/rejected": -0.4841387867927551,
      "logps/chosen": -54.63410186767578,
      "logps/ref_chosen": -54.862335205078125,
      "logps/ref_rejected": -98.0264663696289,
      "logps/rejected": -100.20404052734375,
      "loss": 1.1601,
      "margin_dpo/margin_mean": 2.4058141708374023,
      "margin_dpo/margin_std": 2.224526882171631,
      "step": 45
    },
    {
      "epoch": 0.07342143906020558,
      "fcm_dpo/beta": 0.11535553634166718,
      "fcm_dpo/delta": 0.0067912861704826355,
      "fcm_dpo/margin": 3.4076931476593018,
      "fcm_dpo/q_t": 0.4073707163333893,
      "grad_norm": 68.53203582763672,
      "learning_rate": 3.5507246376811595e-07,
      "logits/chosen": -0.562477707862854,
      "logits/rejected": -0.5116110444068909,
      "logps/chosen": -58.12030029296875,
      "logps/ref_chosen": -58.304595947265625,
      "logps/ref_rejected": -91.69480895996094,
      "logps/rejected": -94.91819763183594,
      "loss": 1.0694,
      "margin_dpo/margin_mean": 3.4076931476593018,
      "margin_dpo/margin_std": 3.365309953689575,
      "step": 50
    },
    {
      "epoch": 0.08076358296622614,
      "fcm_dpo/beta": 0.10359964519739151,
      "fcm_dpo/delta": -0.19395628571510315,
      "fcm_dpo/margin": 5.653986930847168,
      "fcm_dpo/q_t": 0.3696584403514862,
      "grad_norm": 52.22880935668945,
      "learning_rate": 3.9130434782608694e-07,
      "logits/chosen": -0.5961982011795044,
      "logits/rejected": -0.5513918399810791,
      "logps/chosen": -56.34511184692383,
      "logps/ref_chosen": -56.06591796875,
      "logps/ref_rejected": -85.69367980957031,
      "logps/rejected": -91.62686157226562,
      "loss": 0.968,
      "margin_dpo/margin_mean": 5.653987407684326,
      "margin_dpo/margin_std": 5.778587818145752,
      "step": 55
    },
    {
      "epoch": 0.0881057268722467,
      "fcm_dpo/beta": 0.08351375162601471,
      "fcm_dpo/delta": -0.2116357386112213,
      "fcm_dpo/margin": 7.191043853759766,
      "fcm_dpo/q_t": 0.3656514883041382,
      "grad_norm": 53.557796478271484,
      "learning_rate": 4.2753623188405794e-07,
      "logits/chosen": -0.614372193813324,
      "logits/rejected": -0.5657563209533691,
      "logps/chosen": -62.0140266418457,
      "logps/ref_chosen": -60.6871337890625,
      "logps/ref_rejected": -89.72715759277344,
      "logps/rejected": -98.24510192871094,
      "loss": 0.9663,
      "margin_dpo/margin_mean": 7.191043853759766,
      "margin_dpo/margin_std": 7.640544891357422,
      "step": 60
    },
    {
      "epoch": 0.09544787077826726,
      "fcm_dpo/beta": 0.06861739605665207,
      "fcm_dpo/delta": -0.19986796379089355,
      "fcm_dpo/margin": 8.604007720947266,
      "fcm_dpo/q_t": 0.37094324827194214,
      "grad_norm": 41.2183723449707,
      "learning_rate": 4.63768115942029e-07,
      "logits/chosen": -0.6187667846679688,
      "logits/rejected": -0.5872984528541565,
      "logps/chosen": -63.8182487487793,
      "logps/ref_chosen": -61.75325393676758,
      "logps/ref_rejected": -93.30108642578125,
      "logps/rejected": -103.9700927734375,
      "loss": 0.9904,
      "margin_dpo/margin_mean": 8.604008674621582,
      "margin_dpo/margin_std": 10.206991195678711,
      "step": 65
    },
    {
      "epoch": 0.1027900146842878,
      "fcm_dpo/beta": 0.05661847069859505,
      "fcm_dpo/delta": -0.1891469657421112,
      "fcm_dpo/margin": 10.259188652038574,
      "fcm_dpo/q_t": 0.3718249201774597,
      "grad_norm": 39.173946380615234,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6201697587966919,
      "logits/rejected": -0.5813671350479126,
      "logps/chosen": -63.28239059448242,
      "logps/ref_chosen": -59.548004150390625,
      "logps/ref_rejected": -84.01609802246094,
      "logps/rejected": -98.00967407226562,
      "loss": 0.98,
      "margin_dpo/margin_mean": 10.259187698364258,
      "margin_dpo/margin_std": 11.377448081970215,
      "step": 70
    },
    {
      "epoch": 0.11013215859030837,
      "fcm_dpo/beta": 0.04313134402036667,
      "fcm_dpo/delta": -0.354682594537735,
      "fcm_dpo/margin": 17.1324405670166,
      "fcm_dpo/q_t": 0.34925174713134766,
      "grad_norm": 31.45458221435547,
      "learning_rate": 4.999176576834721e-07,
      "logits/chosen": -0.6573521494865417,
      "logits/rejected": -0.6307379603385925,
      "logps/chosen": -67.56647491455078,
      "logps/ref_chosen": -59.86931228637695,
      "logps/ref_rejected": -98.05613708496094,
      "logps/rejected": -122.8857421875,
      "loss": 0.9415,
      "margin_dpo/margin_mean": 17.132436752319336,
      "margin_dpo/margin_std": 21.05853843688965,
      "step": 75
    },
    {
      "epoch": 0.11747430249632893,
      "fcm_dpo/beta": 0.03168341889977455,
      "fcm_dpo/delta": -0.22349825501441956,
      "fcm_dpo/margin": 19.243648529052734,
      "fcm_dpo/q_t": 0.3687431514263153,
      "grad_norm": 28.407453536987305,
      "learning_rate": 4.996706849759452e-07,
      "logits/chosen": -0.6761894822120667,
      "logits/rejected": -0.6410226821899414,
      "logps/chosen": -66.81375122070312,
      "logps/ref_chosen": -56.18925857543945,
      "logps/ref_rejected": -86.42393493652344,
      "logps/rejected": -116.2920913696289,
      "loss": 0.9939,
      "margin_dpo/margin_mean": 19.243648529052734,
      "margin_dpo/margin_std": 23.982927322387695,
      "step": 80
    },
    {
      "epoch": 0.12481644640234948,
      "fcm_dpo/beta": 0.025587420910596848,
      "fcm_dpo/delta": -0.18879233300685883,
      "fcm_dpo/margin": 22.462379455566406,
      "fcm_dpo/q_t": 0.3756103515625,
      "grad_norm": 26.530624389648438,
      "learning_rate": 4.992592445678582e-07,
      "logits/chosen": -0.6672359704971313,
      "logits/rejected": -0.6387097835540771,
      "logps/chosen": -73.97752380371094,
      "logps/ref_chosen": -60.018287658691406,
      "logps/ref_rejected": -98.01185607910156,
      "logps/rejected": -134.43350219726562,
      "loss": 1.0175,
      "margin_dpo/margin_mean": 22.462383270263672,
      "margin_dpo/margin_std": 29.141366958618164,
      "step": 85
    },
    {
      "epoch": 0.13215859030837004,
      "fcm_dpo/beta": 0.02217969484627247,
      "fcm_dpo/delta": -0.15027864277362823,
      "fcm_dpo/margin": 24.47287368774414,
      "fcm_dpo/q_t": 0.38759851455688477,
      "grad_norm": 26.1925106048584,
      "learning_rate": 4.986836074908615e-07,
      "logits/chosen": -0.6896823644638062,
      "logits/rejected": -0.6781142354011536,
      "logps/chosen": -78.13359069824219,
      "logps/ref_chosen": -59.8709831237793,
      "logps/ref_rejected": -96.78519439697266,
      "logps/rejected": -139.5206756591797,
      "loss": 1.0705,
      "margin_dpo/margin_mean": 24.47287368774414,
      "margin_dpo/margin_std": 37.618186950683594,
      "step": 90
    },
    {
      "epoch": 0.1395007342143906,
      "fcm_dpo/beta": 0.019353795796632767,
      "fcm_dpo/delta": -0.10450585186481476,
      "fcm_dpo/margin": 25.787700653076172,
      "fcm_dpo/q_t": 0.3899272084236145,
      "grad_norm": 21.816125869750977,
      "learning_rate": 4.979441529392784e-07,
      "logits/chosen": -0.690959095954895,
      "logits/rejected": -0.6617642641067505,
      "logps/chosen": -74.7010269165039,
      "logps/ref_chosen": -55.94385528564453,
      "logps/ref_rejected": -83.6790542602539,
      "logps/rejected": -128.2239227294922,
      "loss": 1.0485,
      "margin_dpo/margin_mean": 25.787700653076172,
      "margin_dpo/margin_std": 34.392967224121094,
      "step": 95
    },
    {
      "epoch": 0.14684287812041116,
      "fcm_dpo/beta": 0.017355434596538544,
      "fcm_dpo/delta": -0.09411351382732391,
      "fcm_dpo/margin": 28.09613037109375,
      "fcm_dpo/q_t": 0.3915005326271057,
      "grad_norm": 23.495582580566406,
      "learning_rate": 4.970413680203148e-07,
      "logits/chosen": -0.6664688587188721,
      "logits/rejected": -0.6419214010238647,
      "logps/chosen": -78.07569885253906,
      "logps/ref_chosen": -57.05888748168945,
      "logps/ref_rejected": -86.11727142333984,
      "logps/rejected": -135.23020935058594,
      "loss": 1.057,
      "margin_dpo/margin_mean": 28.09613037109375,
      "margin_dpo/margin_std": 37.85677719116211,
      "step": 100
    },
    {
      "epoch": 0.15418502202643172,
      "fcm_dpo/beta": 0.01605215296149254,
      "fcm_dpo/delta": -0.14830544590950012,
      "fcm_dpo/margin": 32.540122985839844,
      "fcm_dpo/q_t": 0.38781845569610596,
      "grad_norm": 22.918745040893555,
      "learning_rate": 4.959758474331832e-07,
      "logits/chosen": -0.6817123889923096,
      "logits/rejected": -0.6505634188652039,
      "logps/chosen": -84.82267761230469,
      "logps/ref_chosen": -59.20774459838867,
      "logps/ref_rejected": -86.49754333496094,
      "logps/rejected": -144.65260314941406,
      "loss": 1.052,
      "margin_dpo/margin_mean": 32.540122985839844,
      "margin_dpo/margin_std": 44.27156066894531,
      "step": 105
    },
    {
      "epoch": 0.16152716593245228,
      "fcm_dpo/beta": 0.013644491322338581,
      "fcm_dpo/delta": -0.1275637000799179,
      "fcm_dpo/margin": 38.23952102661133,
      "fcm_dpo/q_t": 0.38538408279418945,
      "grad_norm": 18.83212661743164,
      "learning_rate": 4.947482930773511e-07,
      "logits/chosen": -0.6280543804168701,
      "logits/rejected": -0.5978196859359741,
      "logps/chosen": -88.46613311767578,
      "logps/ref_chosen": -60.437957763671875,
      "logps/ref_rejected": -90.83917999267578,
      "logps/rejected": -157.1068878173828,
      "loss": 1.0403,
      "margin_dpo/margin_mean": 38.23952102661133,
      "margin_dpo/margin_std": 50.03200149536133,
      "step": 110
    },
    {
      "epoch": 0.16886930983847284,
      "fcm_dpo/beta": 0.012269817292690277,
      "fcm_dpo/delta": -0.06500422954559326,
      "fcm_dpo/margin": 37.55778121948242,
      "fcm_dpo/q_t": 0.39965710043907166,
      "grad_norm": 27.971012115478516,
      "learning_rate": 4.933595135901732e-07,
      "logits/chosen": -0.6194974184036255,
      "logits/rejected": -0.5893136262893677,
      "logps/chosen": -96.05851745605469,
      "logps/ref_chosen": -61.7908821105957,
      "logps/ref_rejected": -85.36943054199219,
      "logps/rejected": -157.19485473632812,
      "loss": 1.0871,
      "margin_dpo/margin_mean": 37.55778121948242,
      "margin_dpo/margin_std": 56.132057189941406,
      "step": 115
    },
    {
      "epoch": 0.1762114537444934,
      "fcm_dpo/beta": 0.01178265642374754,
      "fcm_dpo/delta": -0.05727796629071236,
      "fcm_dpo/margin": 38.61918640136719,
      "fcm_dpo/q_t": 0.39924171566963196,
      "grad_norm": 21.68465805053711,
      "learning_rate": 4.918104238142103e-07,
      "logits/chosen": -0.6018606424331665,
      "logits/rejected": -0.580163836479187,
      "logps/chosen": -104.28592681884766,
      "logps/ref_chosen": -65.3261489868164,
      "logps/ref_rejected": -86.75518798828125,
      "logps/rejected": -164.33416748046875,
      "loss": 1.0756,
      "margin_dpo/margin_mean": 38.61919021606445,
      "margin_dpo/margin_std": 53.98603057861328,
      "step": 120
    },
    {
      "epoch": 0.18355359765051396,
      "fcm_dpo/beta": 0.009986856020987034,
      "fcm_dpo/delta": -0.2121940553188324,
      "fcm_dpo/margin": 59.94220733642578,
      "fcm_dpo/q_t": 0.3725253939628601,
      "grad_norm": 19.076047897338867,
      "learning_rate": 4.90102044194588e-07,
      "logits/chosen": -0.5440986752510071,
      "logits/rejected": -0.5444971323013306,
      "logps/chosen": -102.51204681396484,
      "logps/ref_chosen": -58.323204040527344,
      "logps/ref_rejected": -101.2106704711914,
      "logps/rejected": -205.34170532226562,
      "loss": 1.0102,
      "margin_dpo/margin_mean": 59.94221115112305,
      "margin_dpo/margin_std": 79.08087158203125,
      "step": 125
    },
    {
      "epoch": 0.19089574155653452,
      "fcm_dpo/beta": 0.00899023748934269,
      "fcm_dpo/delta": -0.0574115514755249,
      "fcm_dpo/margin": 50.56972122192383,
      "fcm_dpo/q_t": 0.4008238911628723,
      "grad_norm": 21.011878967285156,
      "learning_rate": 4.882355001067891e-07,
      "logits/chosen": -0.5073991417884827,
      "logits/rejected": -0.49552878737449646,
      "logps/chosen": -103.4196548461914,
      "logps/ref_chosen": -56.38518524169922,
      "logps/ref_rejected": -86.15767669677734,
      "logps/rejected": -183.76187133789062,
      "loss": 1.0949,
      "margin_dpo/margin_mean": 50.569725036621094,
      "margin_dpo/margin_std": 77.31275939941406,
      "step": 130
    },
    {
      "epoch": 0.19823788546255505,
      "fcm_dpo/beta": 0.00818575732409954,
      "fcm_dpo/delta": -0.10119257867336273,
      "fcm_dpo/margin": 60.596702575683594,
      "fcm_dpo/q_t": 0.39124470949172974,
      "grad_norm": 21.952133178710938,
      "learning_rate": 4.862120211153265e-07,
      "logits/chosen": -0.487496942281723,
      "logits/rejected": -0.4898650050163269,
      "logps/chosen": -103.59019470214844,
      "logps/ref_chosen": -54.59065628051758,
      "logps/ref_rejected": -95.26080322265625,
      "logps/rejected": -204.8570556640625,
      "loss": 1.0502,
      "margin_dpo/margin_mean": 60.596702575683594,
      "margin_dpo/margin_std": 81.37339782714844,
      "step": 135
    },
    {
      "epoch": 0.2055800293685756,
      "fcm_dpo/beta": 0.007442955859005451,
      "fcm_dpo/delta": -0.09183903783559799,
      "fcm_dpo/margin": 65.49043273925781,
      "fcm_dpo/q_t": 0.39486831426620483,
      "grad_norm": 20.813894271850586,
      "learning_rate": 4.840329401637809e-07,
      "logits/chosen": -0.4298696517944336,
      "logits/rejected": -0.4265991747379303,
      "logps/chosen": -114.61155700683594,
      "logps/ref_chosen": -56.04347610473633,
      "logps/ref_rejected": -93.27880859375,
      "logps/rejected": -217.3373565673828,
      "loss": 1.064,
      "margin_dpo/margin_mean": 65.49043273925781,
      "margin_dpo/margin_std": 92.36592102050781,
      "step": 140
    },
    {
      "epoch": 0.21292217327459617,
      "fcm_dpo/beta": 0.007036598864942789,
      "fcm_dpo/delta": -0.012454366311430931,
      "fcm_dpo/margin": 58.48834991455078,
      "fcm_dpo/q_t": 0.40865078568458557,
      "grad_norm": 23.93450927734375,
      "learning_rate": 4.816996926967401e-07,
      "logits/chosen": -0.40944308042526245,
      "logits/rejected": -0.39997661113739014,
      "logps/chosen": -129.96218872070312,
      "logps/ref_chosen": -61.4414176940918,
      "logps/ref_rejected": -86.32813262939453,
      "logps/rejected": -213.33724975585938,
      "loss": 1.115,
      "margin_dpo/margin_mean": 58.48834991455078,
      "margin_dpo/margin_std": 92.11347961425781,
      "step": 145
    },
    {
      "epoch": 0.22026431718061673,
      "fcm_dpo/beta": 0.006964278873056173,
      "fcm_dpo/delta": -0.03296690061688423,
      "fcm_dpo/margin": 59.02332305908203,
      "fcm_dpo/q_t": 0.4086775779724121,
      "grad_norm": 18.16049575805664,
      "learning_rate": 4.792138157142157e-07,
      "logits/chosen": -0.44094228744506836,
      "logits/rejected": -0.42822694778442383,
      "logps/chosen": -123.40377044677734,
      "logps/ref_chosen": -57.70451736450195,
      "logps/ref_rejected": -87.76991271972656,
      "logps/rejected": -212.4925079345703,
      "loss": 1.1066,
      "margin_dpo/margin_mean": 59.02332305908203,
      "margin_dpo/margin_std": 87.62327575683594,
      "step": 150
    },
    {
      "epoch": 0.2276064610866373,
      "fcm_dpo/beta": 0.006572508253157139,
      "fcm_dpo/delta": -0.058337144553661346,
      "fcm_dpo/margin": 69.28470611572266,
      "fcm_dpo/q_t": 0.3986854553222656,
      "grad_norm": 19.04676628112793,
      "learning_rate": 4.7657694675916247e-07,
      "logits/chosen": -0.4826192855834961,
      "logits/rejected": -0.4734383523464203,
      "logps/chosen": -126.15814208984375,
      "logps/ref_chosen": -62.08925247192383,
      "logps/ref_rejected": -94.79930114746094,
      "logps/rejected": -228.1529083251953,
      "loss": 1.076,
      "margin_dpo/margin_mean": 69.28469848632812,
      "margin_dpo/margin_std": 97.56148529052734,
      "step": 155
    },
    {
      "epoch": 0.23494860499265785,
      "fcm_dpo/beta": 0.006233358755707741,
      "fcm_dpo/delta": -0.06433447450399399,
      "fcm_dpo/margin": 70.75523376464844,
      "fcm_dpo/q_t": 0.4046873450279236,
      "grad_norm": 23.905847549438477,
      "learning_rate": 4.737908228387656e-07,
      "logits/chosen": -0.37852075695991516,
      "logits/rejected": -0.37446096539497375,
      "logps/chosen": -155.4814453125,
      "logps/ref_chosen": -67.15288543701172,
      "logps/ref_rejected": -96.92537689208984,
      "logps/rejected": -256.0091552734375,
      "loss": 1.1205,
      "margin_dpo/margin_mean": 70.75523376464844,
      "margin_dpo/margin_std": 118.20479583740234,
      "step": 160
    },
    {
      "epoch": 0.2422907488986784,
      "fcm_dpo/beta": 0.005916006397455931,
      "fcm_dpo/delta": -0.016961436718702316,
      "fcm_dpo/margin": 70.23262023925781,
      "fcm_dpo/q_t": 0.4076644480228424,
      "grad_norm": 21.921010971069336,
      "learning_rate": 4.708572792802069e-07,
      "logits/chosen": -0.38327422738075256,
      "logits/rejected": -0.369274765253067,
      "logps/chosen": -138.60523986816406,
      "logps/ref_chosen": -57.40401077270508,
      "logps/ref_rejected": -80.31498718261719,
      "logps/rejected": -231.7488555908203,
      "loss": 1.1054,
      "margin_dpo/margin_mean": 70.23262023925781,
      "margin_dpo/margin_std": 105.99822998046875,
      "step": 165
    },
    {
      "epoch": 0.24963289280469897,
      "fcm_dpo/beta": 0.005466562695801258,
      "fcm_dpo/delta": -0.10773168504238129,
      "fcm_dpo/margin": 87.78046417236328,
      "fcm_dpo/q_t": 0.3945610821247101,
      "grad_norm": 17.49802589416504,
      "learning_rate": 4.6777824852166437e-07,
      "logits/chosen": -0.3227120041847229,
      "logits/rejected": -0.3182465136051178,
      "logps/chosen": -131.82830810546875,
      "logps/ref_chosen": -52.029144287109375,
      "logps/ref_rejected": -85.73944091796875,
      "logps/rejected": -253.31906127929688,
      "loss": 1.0613,
      "margin_dpo/margin_mean": 87.78046417236328,
      "margin_dpo/margin_std": 120.57357025146484,
      "step": 170
    },
    {
      "epoch": 0.25697503671071953,
      "fcm_dpo/beta": 0.0053649842739105225,
      "fcm_dpo/delta": -0.02835622988641262,
      "fcm_dpo/margin": 79.64360809326172,
      "fcm_dpo/q_t": 0.40541133284568787,
      "grad_norm": 29.300512313842773,
      "learning_rate": 4.645557588393406e-07,
      "logits/chosen": -0.3354695737361908,
      "logits/rejected": -0.3250008523464203,
      "logps/chosen": -156.86700439453125,
      "logps/ref_chosen": -62.996971130371094,
      "logps/ref_rejected": -92.98394012451172,
      "logps/rejected": -266.4975891113281,
      "loss": 1.0959,
      "margin_dpo/margin_mean": 79.64360046386719,
      "margin_dpo/margin_std": 116.92179107666016,
      "step": 175
    },
    {
      "epoch": 0.2643171806167401,
      "fcm_dpo/beta": 0.004915366414934397,
      "fcm_dpo/delta": -0.08844007551670074,
      "fcm_dpo/margin": 98.45795440673828,
      "fcm_dpo/q_t": 0.39298662543296814,
      "grad_norm": 18.73993492126465,
      "learning_rate": 4.611919330113591e-07,
      "logits/chosen": -0.2953912019729614,
      "logits/rejected": -0.29680582880973816,
      "logps/chosen": -155.4091796875,
      "logps/ref_chosen": -57.0670280456543,
      "logps/ref_rejected": -97.1115493774414,
      "logps/rejected": -293.9116516113281,
      "loss": 1.0541,
      "margin_dpo/margin_mean": 98.45795440673828,
      "margin_dpo/margin_std": 131.67587280273438,
      "step": 180
    },
    {
      "epoch": 0.27165932452276065,
      "fcm_dpo/beta": 0.00469121104106307,
      "fcm_dpo/delta": -0.007265460677444935,
      "fcm_dpo/margin": 72.3204574584961,
      "fcm_dpo/q_t": 0.423015296459198,
      "grad_norm": 19.846059799194336,
      "learning_rate": 4.5768898691940836e-07,
      "logits/chosen": -0.3211399018764496,
      "logits/rejected": -0.31003421545028687,
      "logps/chosen": -150.46461486816406,
      "logps/ref_chosen": -54.840736389160156,
      "logps/ref_rejected": -75.51002502441406,
      "logps/rejected": -243.45437622070312,
      "loss": 1.1531,
      "margin_dpo/margin_mean": 72.3204574584961,
      "margin_dpo/margin_std": 118.7593994140625,
      "step": 185
    },
    {
      "epoch": 0.2790014684287812,
      "fcm_dpo/beta": 0.004630076698958874,
      "fcm_dpo/delta": -0.02935381792485714,
      "fcm_dpo/margin": 92.37378692626953,
      "fcm_dpo/q_t": 0.4038674235343933,
      "grad_norm": 18.63943862915039,
      "learning_rate": 4.5404922808905543e-07,
      "logits/chosen": -0.3224960267543793,
      "logits/rejected": -0.31152957677841187,
      "logps/chosen": -155.59190368652344,
      "logps/ref_chosen": -57.72148895263672,
      "logps/ref_rejected": -86.85997009277344,
      "logps/rejected": -277.10418701171875,
      "loss": 1.0829,
      "margin_dpo/margin_mean": 92.37378692626953,
      "margin_dpo/margin_std": 125.8380126953125,
      "step": 190
    },
    {
      "epoch": 0.28634361233480177,
      "fcm_dpo/beta": 0.004372724797576666,
      "fcm_dpo/delta": -0.041835661977529526,
      "fcm_dpo/margin": 94.63658142089844,
      "fcm_dpo/q_t": 0.4060741066932678,
      "grad_norm": 19.838130950927734,
      "learning_rate": 4.5027505416968985e-07,
      "logits/chosen": -0.2853700518608093,
      "logits/rejected": -0.2791460156440735,
      "logps/chosen": -171.98052978515625,
      "logps/ref_chosen": -58.26164627075195,
      "logps/ref_rejected": -89.46485900878906,
      "logps/rejected": -297.82037353515625,
      "loss": 1.0971,
      "margin_dpo/margin_mean": 94.63658142089844,
      "margin_dpo/margin_std": 133.46847534179688,
      "step": 195
    },
    {
      "epoch": 0.2936857562408223,
      "fcm_dpo/beta": 0.0042917924001812935,
      "fcm_dpo/delta": -0.03731077164411545,
      "fcm_dpo/margin": 101.45069122314453,
      "fcm_dpo/q_t": 0.40244197845458984,
      "grad_norm": 21.282487869262695,
      "learning_rate": 4.4636895135509966e-07,
      "logits/chosen": -0.29983407258987427,
      "logits/rejected": -0.2885088324546814,
      "logps/chosen": -161.27999877929688,
      "logps/ref_chosen": -55.71953201293945,
      "logps/ref_rejected": -83.15235137939453,
      "logps/rejected": -290.16351318359375,
      "loss": 1.0801,
      "margin_dpo/margin_mean": 101.45067596435547,
      "margin_dpo/margin_std": 138.32688903808594,
      "step": 200
    },
    {
      "epoch": 0.2936857562408223,
      "eval_fcm_dpo/beta": 0.004899847786873579,
      "eval_fcm_dpo/delta": 0.03208634629845619,
      "eval_fcm_dpo/margin": 58.008628845214844,
      "eval_fcm_dpo/q_t": 0.4392388164997101,
      "eval_logits/chosen": -0.3472006916999817,
      "eval_logits/rejected": -0.3329962491989136,
      "eval_logps/chosen": -225.8389892578125,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -291.5945129394531,
      "eval_loss": 0.6239348649978638,
      "eval_margin_dpo/margin_mean": 58.008628845214844,
      "eval_margin_dpo/margin_std": 147.6545867919922,
      "eval_runtime": 39.6889,
      "eval_samples_per_second": 58.933,
      "eval_steps_per_second": 1.864,
      "step": 200
    },
    {
      "epoch": 0.3010279001468429,
      "fcm_dpo/beta": 0.005112385377287865,
      "fcm_dpo/delta": -0.07373825460672379,
      "fcm_dpo/margin": 91.88821411132812,
      "fcm_dpo/q_t": 0.39707884192466736,
      "grad_norm": 24.034622192382812,
      "learning_rate": 4.4233349274571974e-07,
      "logits/chosen": -0.3471163213253021,
      "logits/rejected": -0.3326929211616516,
      "logps/chosen": -181.0006103515625,
      "logps/ref_chosen": -65.13258361816406,
      "logps/ref_rejected": -92.10203552246094,
      "logps/rejected": -299.8582763671875,
      "loss": 1.0848,
      "margin_dpo/margin_mean": 91.88821411132812,
      "margin_dpo/margin_std": 137.34268188476562,
      "step": 205
    },
    {
      "epoch": 0.30837004405286345,
      "fcm_dpo/beta": 0.00456308713182807,
      "fcm_dpo/delta": -0.12305764853954315,
      "fcm_dpo/margin": 112.89747619628906,
      "fcm_dpo/q_t": 0.3846554160118103,
      "grad_norm": 25.71703338623047,
      "learning_rate": 4.381713366536311e-07,
      "logits/chosen": -0.30507659912109375,
      "logits/rejected": -0.2925286889076233,
      "logps/chosen": -175.38168334960938,
      "logps/ref_chosen": -54.52837371826172,
      "logps/ref_rejected": -87.06227111816406,
      "logps/rejected": -320.81304931640625,
      "loss": 1.0332,
      "margin_dpo/margin_mean": 112.89747619628906,
      "margin_dpo/margin_std": 143.22720336914062,
      "step": 210
    },
    {
      "epoch": 0.315712187958884,
      "fcm_dpo/beta": 0.004155799280852079,
      "fcm_dpo/delta": -0.09338673204183578,
      "fcm_dpo/margin": 117.60514068603516,
      "fcm_dpo/q_t": 0.39112934470176697,
      "grad_norm": 27.700122833251953,
      "learning_rate": 4.3388522485142885e-07,
      "logits/chosen": -0.2812441289424896,
      "logits/rejected": -0.265227735042572,
      "logps/chosen": -197.89137268066406,
      "logps/ref_chosen": -59.905250549316406,
      "logps/ref_rejected": -90.25511932373047,
      "logps/rejected": -345.84637451171875,
      "loss": 1.0502,
      "margin_dpo/margin_mean": 117.60514068603516,
      "margin_dpo/margin_std": 151.13796997070312,
      "step": 215
    },
    {
      "epoch": 0.32305433186490456,
      "fcm_dpo/beta": 0.0038633563090115786,
      "fcm_dpo/delta": -0.0188637413084507,
      "fcm_dpo/margin": 107.99397277832031,
      "fcm_dpo/q_t": 0.406075656414032,
      "grad_norm": 27.14980697631836,
      "learning_rate": 4.2947798076611047e-07,
      "logits/chosen": -0.2910076379776001,
      "logits/rejected": -0.27931129932403564,
      "logps/chosen": -209.11544799804688,
      "logps/ref_chosen": -57.68498611450195,
      "logps/ref_rejected": -87.72792053222656,
      "logps/rejected": -347.15234375,
      "loss": 1.0982,
      "margin_dpo/margin_mean": 107.99397277832031,
      "margin_dpo/margin_std": 155.56427001953125,
      "step": 220
    },
    {
      "epoch": 0.3303964757709251,
      "fcm_dpo/beta": 0.003635880770161748,
      "fcm_dpo/delta": -0.11107522249221802,
      "fcm_dpo/margin": 138.54534912109375,
      "fcm_dpo/q_t": 0.3866313397884369,
      "grad_norm": 21.774181365966797,
      "learning_rate": 4.249525076191759e-07,
      "logits/chosen": -0.26980072259902954,
      "logits/rejected": -0.2639961540699005,
      "logps/chosen": -216.45687866210938,
      "logps/ref_chosen": -54.47245407104492,
      "logps/ref_rejected": -93.26266479492188,
      "logps/rejected": -393.79248046875,
      "loss": 1.0374,
      "margin_dpo/margin_mean": 138.5453643798828,
      "margin_dpo/margin_std": 173.9648895263672,
      "step": 225
    },
    {
      "epoch": 0.3377386196769457,
      "fcm_dpo/beta": 0.003416923340409994,
      "fcm_dpo/delta": -0.05279744789004326,
      "fcm_dpo/margin": 131.7572021484375,
      "fcm_dpo/q_t": 0.3975502550601959,
      "grad_norm": 29.499502182006836,
      "learning_rate": 4.203117865141635e-07,
      "logits/chosen": -0.3394869565963745,
      "logits/rejected": -0.31970351934432983,
      "logps/chosen": -219.03005981445312,
      "logps/ref_chosen": -58.7701301574707,
      "logps/ref_rejected": -87.85963439941406,
      "logps/rejected": -379.87677001953125,
      "loss": 1.0656,
      "margin_dpo/margin_mean": 131.7572021484375,
      "margin_dpo/margin_std": 169.88063049316406,
      "step": 230
    },
    {
      "epoch": 0.34508076358296624,
      "fcm_dpo/beta": 0.003319287672638893,
      "fcm_dpo/delta": -0.0038651395589113235,
      "fcm_dpo/margin": 121.47023010253906,
      "fcm_dpo/q_t": 0.40905800461769104,
      "grad_norm": 27.062564849853516,
      "learning_rate": 4.1555887447288255e-07,
      "logits/chosen": -0.3577764332294464,
      "logits/rejected": -0.3489416241645813,
      "logps/chosen": -223.79031372070312,
      "logps/ref_chosen": -59.0481071472168,
      "logps/ref_rejected": -91.19654846191406,
      "logps/rejected": -377.40899658203125,
      "loss": 1.0973,
      "margin_dpo/margin_mean": 121.47023010253906,
      "margin_dpo/margin_std": 166.64047241210938,
      "step": 235
    },
    {
      "epoch": 0.3524229074889868,
      "fcm_dpo/beta": 0.0032308667432516813,
      "fcm_dpo/delta": -0.02104497328400612,
      "fcm_dpo/margin": 129.8544921875,
      "fcm_dpo/q_t": 0.4038158357143402,
      "grad_norm": 31.66817855834961,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -0.3540635108947754,
      "logits/rejected": -0.35047775506973267,
      "logps/chosen": -231.42440795898438,
      "logps/ref_chosen": -55.238983154296875,
      "logps/ref_rejected": -91.08428955078125,
      "logps/rejected": -397.12420654296875,
      "loss": 1.0799,
      "margin_dpo/margin_mean": 129.85450744628906,
      "margin_dpo/margin_std": 166.92564392089844,
      "step": 240
    },
    {
      "epoch": 0.35976505139500736,
      "fcm_dpo/beta": 0.003068047808483243,
      "fcm_dpo/delta": -0.07209217548370361,
      "fcm_dpo/margin": 152.28811645507812,
      "fcm_dpo/q_t": 0.3937745690345764,
      "grad_norm": 28.177356719970703,
      "learning_rate": 4.057290731287531e-07,
      "logits/chosen": -0.3967796266078949,
      "logits/rejected": -0.37566548585891724,
      "logps/chosen": -252.69760131835938,
      "logps/ref_chosen": -65.08844757080078,
      "logps/ref_rejected": -86.05777740478516,
      "logps/rejected": -425.95501708984375,
      "loss": 1.0455,
      "margin_dpo/margin_mean": 152.28811645507812,
      "margin_dpo/margin_std": 179.84130859375,
      "step": 245
    },
    {
      "epoch": 0.3671071953010279,
      "fcm_dpo/beta": 0.0029506587889045477,
      "fcm_dpo/delta": -0.029526233673095703,
      "fcm_dpo/margin": 144.90142822265625,
      "fcm_dpo/q_t": 0.4018786549568176,
      "grad_norm": 30.520957946777344,
      "learning_rate": 4.006586590948141e-07,
      "logits/chosen": -0.3914863169193268,
      "logits/rejected": -0.3723214566707611,
      "logps/chosen": -273.7362365722656,
      "logps/ref_chosen": -59.08491897583008,
      "logps/ref_rejected": -87.36727142333984,
      "logps/rejected": -446.9200134277344,
      "loss": 1.0748,
      "margin_dpo/margin_mean": 144.9014434814453,
      "margin_dpo/margin_std": 182.38096618652344,
      "step": 250
    },
    {
      "epoch": 0.3744493392070485,
      "fcm_dpo/beta": 0.0029482415411621332,
      "fcm_dpo/delta": 0.010621277615427971,
      "fcm_dpo/margin": 132.09812927246094,
      "fcm_dpo/q_t": 0.4101809859275818,
      "grad_norm": 21.41288185119629,
      "learning_rate": 3.954890003969163e-07,
      "logits/chosen": -0.38516297936439514,
      "logits/rejected": -0.36370107531547546,
      "logps/chosen": -264.5665588378906,
      "logps/ref_chosen": -61.85979461669922,
      "logps/ref_rejected": -88.32804107666016,
      "logps/rejected": -423.1329650878906,
      "loss": 1.1051,
      "margin_dpo/margin_mean": 132.09812927246094,
      "margin_dpo/margin_std": 182.64788818359375,
      "step": 255
    },
    {
      "epoch": 0.38179148311306904,
      "fcm_dpo/beta": 0.0029091238975524902,
      "fcm_dpo/delta": -0.0311284102499485,
      "fcm_dpo/margin": 147.6251678466797,
      "fcm_dpo/q_t": 0.4018276631832123,
      "grad_norm": 31.76065444946289,
      "learning_rate": 3.9022350248844246e-07,
      "logits/chosen": -0.3353942334651947,
      "logits/rejected": -0.33657923340797424,
      "logps/chosen": -256.07330322265625,
      "logps/ref_chosen": -52.843467712402344,
      "logps/ref_rejected": -90.4744873046875,
      "logps/rejected": -441.3294982910156,
      "loss": 1.0767,
      "margin_dpo/margin_mean": 147.6251678466797,
      "margin_dpo/margin_std": 192.7305908203125,
      "step": 260
    },
    {
      "epoch": 0.3891336270190896,
      "fcm_dpo/beta": 0.002804348012432456,
      "fcm_dpo/delta": -0.036855168640613556,
      "fcm_dpo/margin": 155.0548095703125,
      "fcm_dpo/q_t": 0.4009890556335449,
      "grad_norm": 26.897104263305664,
      "learning_rate": 3.848656339557562e-07,
      "logits/chosen": -0.3249055743217468,
      "logits/rejected": -0.3102591931819916,
      "logps/chosen": -295.2470703125,
      "logps/ref_chosen": -59.35320281982422,
      "logps/ref_rejected": -90.73350524902344,
      "logps/rejected": -481.68218994140625,
      "loss": 1.0752,
      "margin_dpo/margin_mean": 155.0548095703125,
      "margin_dpo/margin_std": 203.52064514160156,
      "step": 265
    },
    {
      "epoch": 0.3964757709251101,
      "fcm_dpo/beta": 0.002779749920591712,
      "fcm_dpo/delta": 0.0037582286167889833,
      "fcm_dpo/margin": 142.50425720214844,
      "fcm_dpo/q_t": 0.40962162613868713,
      "grad_norm": 36.368804931640625,
      "learning_rate": 3.794189242333106e-07,
      "logits/chosen": -0.3710918426513672,
      "logits/rejected": -0.3549976944923401,
      "logps/chosen": -305.8779602050781,
      "logps/ref_chosen": -66.30875396728516,
      "logps/ref_rejected": -95.4130630493164,
      "logps/rejected": -477.486572265625,
      "loss": 1.1067,
      "margin_dpo/margin_mean": 142.50425720214844,
      "margin_dpo/margin_std": 207.22396850585938,
      "step": 270
    },
    {
      "epoch": 0.40381791483113066,
      "fcm_dpo/beta": 0.002664657775312662,
      "fcm_dpo/delta": -0.05992863327264786,
      "fcm_dpo/margin": 171.36749267578125,
      "fcm_dpo/q_t": 0.3942517042160034,
      "grad_norm": 29.650026321411133,
      "learning_rate": 3.738869612786737e-07,
      "logits/chosen": -0.3030190169811249,
      "logits/rejected": -0.2977046072483063,
      "logps/chosen": -267.1299743652344,
      "logps/ref_chosen": -54.69990921020508,
      "logps/ref_rejected": -92.23838806152344,
      "logps/rejected": -476.03594970703125,
      "loss": 1.0428,
      "margin_dpo/margin_mean": 171.36749267578125,
      "margin_dpo/margin_std": 191.60519409179688,
      "step": 275
    },
    {
      "epoch": 0.4111600587371512,
      "fcm_dpo/beta": 0.002544180955737829,
      "fcm_dpo/delta": -0.059092938899993896,
      "fcm_dpo/margin": 179.50125122070312,
      "fcm_dpo/q_t": 0.395501971244812,
      "grad_norm": 27.062313079833984,
      "learning_rate": 3.6827338920900253e-07,
      "logits/chosen": -0.2739403247833252,
      "logits/rejected": -0.26840710639953613,
      "logps/chosen": -304.5663146972656,
      "logps/ref_chosen": -54.64586639404297,
      "logps/ref_rejected": -88.19416809082031,
      "logps/rejected": -517.6158447265625,
      "loss": 1.053,
      "margin_dpo/margin_mean": 179.5012664794922,
      "margin_dpo/margin_std": 217.075439453125,
      "step": 280
    },
    {
      "epoch": 0.4185022026431718,
      "fcm_dpo/beta": 0.002449630293995142,
      "fcm_dpo/delta": -0.01916874758899212,
      "fcm_dpo/margin": 170.6898956298828,
      "fcm_dpo/q_t": 0.40265077352523804,
      "grad_norm": 27.657169342041016,
      "learning_rate": 3.625819059005228e-07,
      "logits/chosen": -0.3483652174472809,
      "logits/rejected": -0.33659619092941284,
      "logps/chosen": -324.3221130371094,
      "logps/ref_chosen": -63.02496337890625,
      "logps/ref_rejected": -93.16323852539062,
      "logps/rejected": -525.1502685546875,
      "loss": 1.0686,
      "margin_dpo/margin_mean": 170.68988037109375,
      "margin_dpo/margin_std": 199.1321563720703,
      "step": 285
    },
    {
      "epoch": 0.42584434654919234,
      "fcm_dpo/beta": 0.0023679626174271107,
      "fcm_dpo/delta": -0.05114912986755371,
      "fcm_dpo/margin": 189.60928344726562,
      "fcm_dpo/q_t": 0.39646488428115845,
      "grad_norm": 24.63570785522461,
      "learning_rate": 3.568162605525952e-07,
      "logits/chosen": -0.30854520201683044,
      "logits/rejected": -0.30314746499061584,
      "logps/chosen": -322.7059631347656,
      "logps/ref_chosen": -58.37105178833008,
      "logps/ref_rejected": -91.59428405761719,
      "logps/rejected": -545.5384521484375,
      "loss": 1.0504,
      "margin_dpo/margin_mean": 189.60928344726562,
      "margin_dpo/margin_std": 216.8871307373047,
      "step": 290
    },
    {
      "epoch": 0.4331864904552129,
      "fcm_dpo/beta": 0.00226641446352005,
      "fcm_dpo/delta": -0.026808306574821472,
      "fcm_dpo/margin": 187.75306701660156,
      "fcm_dpo/q_t": 0.4028984606266022,
      "grad_norm": 37.57793045043945,
      "learning_rate": 3.509802512179737e-07,
      "logits/chosen": -0.3241910934448242,
      "logits/rejected": -0.31886667013168335,
      "logps/chosen": -314.74871826171875,
      "logps/ref_chosen": -55.113426208496094,
      "logps/ref_rejected": -85.29283905029297,
      "logps/rejected": -532.68115234375,
      "loss": 1.0814,
      "margin_dpo/margin_mean": 187.75306701660156,
      "margin_dpo/margin_std": 250.9693603515625,
      "step": 295
    },
    {
      "epoch": 0.44052863436123346,
      "fcm_dpo/beta": 0.0022166285198181868,
      "fcm_dpo/delta": -0.03155837580561638,
      "fcm_dpo/margin": 183.79110717773438,
      "fcm_dpo/q_t": 0.40807557106018066,
      "grad_norm": 29.412609100341797,
      "learning_rate": 3.4507772230088147e-07,
      "logits/chosen": -0.37595048546791077,
      "logits/rejected": -0.3784215748310089,
      "logps/chosen": -391.8227844238281,
      "logps/ref_chosen": -59.46582794189453,
      "logps/ref_rejected": -97.03690338134766,
      "logps/rejected": -613.1849975585938,
      "loss": 1.107,
      "margin_dpo/margin_mean": 183.79110717773438,
      "margin_dpo/margin_std": 271.849853515625,
      "step": 300
    },
    {
      "epoch": 0.447870778267254,
      "fcm_dpo/beta": 0.0021604837384074926,
      "fcm_dpo/delta": 0.016617389395833015,
      "fcm_dpo/margin": 177.69085693359375,
      "fcm_dpo/q_t": 0.41120848059654236,
      "grad_norm": 26.565410614013672,
      "learning_rate": 3.391125620245535e-07,
      "logits/chosen": -0.4044717252254486,
      "logits/rejected": -0.3968522846698761,
      "logps/chosen": -303.1658935546875,
      "logps/ref_chosen": -62.78144454956055,
      "logps/ref_rejected": -91.95039367675781,
      "logps/rejected": -510.02569580078125,
      "loss": 1.0938,
      "margin_dpo/margin_mean": 177.69085693359375,
      "margin_dpo/margin_std": 218.7647247314453,
      "step": 305
    },
    {
      "epoch": 0.4552129221732746,
      "fcm_dpo/beta": 0.0022380652371793985,
      "fcm_dpo/delta": 0.01699766516685486,
      "fcm_dpo/margin": 171.28904724121094,
      "fcm_dpo/q_t": 0.4098632335662842,
      "grad_norm": 25.57453155517578,
      "learning_rate": 3.3308869986991487e-07,
      "logits/chosen": -0.3871304988861084,
      "logits/rejected": -0.3712696135044098,
      "logps/chosen": -297.95452880859375,
      "logps/ref_chosen": -61.359039306640625,
      "logps/ref_rejected": -82.75496673583984,
      "logps/rejected": -490.6394958496094,
      "loss": 1.0861,
      "margin_dpo/margin_mean": 171.2890625,
      "margin_dpo/margin_std": 196.2567138671875,
      "step": 310
    },
    {
      "epoch": 0.46255506607929514,
      "fcm_dpo/beta": 0.0022226418368518353,
      "fcm_dpo/delta": -0.002875490579754114,
      "fcm_dpo/margin": 181.02659606933594,
      "fcm_dpo/q_t": 0.4087609350681305,
      "grad_norm": 23.077590942382812,
      "learning_rate": 3.270101039870797e-07,
      "logits/chosen": -0.3677603602409363,
      "logits/rejected": -0.36049968004226685,
      "logps/chosen": -327.0533447265625,
      "logps/ref_chosen": -51.77602005004883,
      "logps/ref_rejected": -84.58292388916016,
      "logps/rejected": -540.8868408203125,
      "loss": 1.0964,
      "margin_dpo/margin_mean": 181.02658081054688,
      "margin_dpo/margin_std": 249.53970336914062,
      "step": 315
    },
    {
      "epoch": 0.4698972099853157,
      "fcm_dpo/beta": 0.002118379110470414,
      "fcm_dpo/delta": -0.09106224775314331,
      "fcm_dpo/margin": 229.5352325439453,
      "fcm_dpo/q_t": 0.38847747445106506,
      "grad_norm": 26.46944236755371,
      "learning_rate": 3.208807785813777e-07,
      "logits/chosen": -0.40023693442344666,
      "logits/rejected": -0.3959748148918152,
      "logps/chosen": -333.487548828125,
      "logps/ref_chosen": -56.777862548828125,
      "logps/ref_rejected": -99.26368713378906,
      "logps/rejected": -605.5086059570312,
      "loss": 1.0268,
      "margin_dpo/margin_mean": 229.5352325439453,
      "margin_dpo/margin_std": 254.9644317626953,
      "step": 320
    },
    {
      "epoch": 0.47723935389133626,
      "fcm_dpo/beta": 0.002015223726630211,
      "fcm_dpo/delta": 0.0017573073273524642,
      "fcm_dpo/margin": 197.41522216796875,
      "fcm_dpo/q_t": 0.4082733690738678,
      "grad_norm": 27.207300186157227,
      "learning_rate": 3.147047612756302e-07,
      "logits/chosen": -0.4213026165962219,
      "logits/rejected": -0.4106159210205078,
      "logps/chosen": -381.36676025390625,
      "logps/ref_chosen": -58.28468704223633,
      "logps/ref_rejected": -83.80326843261719,
      "logps/rejected": -604.3004760742188,
      "loss": 1.0923,
      "margin_dpo/margin_mean": 197.41522216796875,
      "margin_dpo/margin_std": 251.11337280273438,
      "step": 325
    },
    {
      "epoch": 0.4845814977973568,
      "fcm_dpo/beta": 0.002033454831689596,
      "fcm_dpo/delta": -0.02132582850754261,
      "fcm_dpo/margin": 206.72314453125,
      "fcm_dpo/q_t": 0.40213847160339355,
      "grad_norm": 28.69426727294922,
      "learning_rate": 3.084861204504122e-07,
      "logits/chosen": -0.42850977182388306,
      "logits/rejected": -0.42162972688674927,
      "logps/chosen": -349.00677490234375,
      "logps/ref_chosen": -62.75822067260742,
      "logps/ref_rejected": -94.04203033447266,
      "logps/rejected": -587.0137939453125,
      "loss": 1.0667,
      "margin_dpo/margin_mean": 206.72314453125,
      "margin_dpo/margin_std": 237.6807403564453,
      "step": 330
    },
    {
      "epoch": 0.4919236417033774,
      "fcm_dpo/beta": 0.0019993516616523266,
      "fcm_dpo/delta": 0.017332084476947784,
      "fcm_dpo/margin": 191.62435913085938,
      "fcm_dpo/q_t": 0.41195163130760193,
      "grad_norm": 32.573978424072266,
      "learning_rate": 3.022289525640531e-07,
      "logits/chosen": -0.46553975343704224,
      "logits/rejected": -0.454815149307251,
      "logps/chosen": -367.4043273925781,
      "logps/ref_chosen": -58.59650421142578,
      "logps/ref_rejected": -88.69586944580078,
      "logps/rejected": -589.1280517578125,
      "loss": 1.1035,
      "margin_dpo/margin_mean": 191.62435913085938,
      "margin_dpo/margin_std": 254.1148223876953,
      "step": 335
    },
    {
      "epoch": 0.49926578560939794,
      "fcm_dpo/beta": 0.001957540400326252,
      "fcm_dpo/delta": -0.052527785301208496,
      "fcm_dpo/margin": 229.53335571289062,
      "fcm_dpo/q_t": 0.3968558609485626,
      "grad_norm": 28.016321182250977,
      "learning_rate": 2.959373794541426e-07,
      "logits/chosen": -0.45491886138916016,
      "logits/rejected": -0.44332224130630493,
      "logps/chosen": -372.17889404296875,
      "logps/ref_chosen": -58.18162155151367,
      "logps/ref_rejected": -94.44358825683594,
      "logps/rejected": -637.9742431640625,
      "loss": 1.0561,
      "margin_dpo/margin_mean": 229.53335571289062,
      "margin_dpo/margin_std": 272.4295959472656,
      "step": 340
    },
    {
      "epoch": 0.5066079295154186,
      "fcm_dpo/beta": 0.0019028617534786463,
      "fcm_dpo/delta": -0.032430313527584076,
      "fcm_dpo/margin": 226.35147094726562,
      "fcm_dpo/q_t": 0.4004201292991638,
      "grad_norm": 25.25508689880371,
      "learning_rate": 2.896155456223163e-07,
      "logits/chosen": -0.44945335388183594,
      "logits/rejected": -0.4429987072944641,
      "logps/chosen": -372.24151611328125,
      "logps/ref_chosen": -57.9904899597168,
      "logps/ref_rejected": -99.11092376708984,
      "logps/rejected": -639.7134399414062,
      "loss": 1.0594,
      "margin_dpo/margin_mean": 226.3514862060547,
      "margin_dpo/margin_std": 255.4004364013672,
      "step": 345
    },
    {
      "epoch": 0.5139500734214391,
      "fcm_dpo/beta": 0.0018661676440387964,
      "fcm_dpo/delta": 0.005433606915175915,
      "fcm_dpo/margin": 211.39328002929688,
      "fcm_dpo/q_t": 0.40922412276268005,
      "grad_norm": 30.192829132080078,
      "learning_rate": 2.8326761550411346e-07,
      "logits/chosen": -0.442319393157959,
      "logits/rejected": -0.4401112198829651,
      "logps/chosen": -384.77960205078125,
      "logps/ref_chosen": -58.29923629760742,
      "logps/ref_rejected": -89.25711822509766,
      "logps/rejected": -627.1307373046875,
      "loss": 1.0959,
      "margin_dpo/margin_mean": 211.3932647705078,
      "margin_dpo/margin_std": 280.6878967285156,
      "step": 350
    },
    {
      "epoch": 0.5212922173274597,
      "fcm_dpo/beta": 0.001845445716753602,
      "fcm_dpo/delta": -0.019822590053081512,
      "fcm_dpo/margin": 216.8295135498047,
      "fcm_dpo/q_t": 0.4076092839241028,
      "grad_norm": 37.005271911621094,
      "learning_rate": 2.7689777072570284e-07,
      "logits/chosen": -0.4728454649448395,
      "logits/rejected": -0.46268409490585327,
      "logps/chosen": -382.65460205078125,
      "logps/ref_chosen": -60.788482666015625,
      "logps/ref_rejected": -85.94129943847656,
      "logps/rejected": -624.6368408203125,
      "loss": 1.0868,
      "margin_dpo/margin_mean": 216.8295135498047,
      "margin_dpo/margin_std": 270.96405029296875,
      "step": 355
    },
    {
      "epoch": 0.5286343612334802,
      "fcm_dpo/beta": 0.0018649225821718574,
      "fcm_dpo/delta": 0.030804917216300964,
      "fcm_dpo/margin": 183.98890686035156,
      "fcm_dpo/q_t": 0.4203580319881439,
      "grad_norm": 36.66780471801758,
      "learning_rate": 2.7051020734928443e-07,
      "logits/chosen": -0.4746428430080414,
      "logits/rejected": -0.4617390036582947,
      "logps/chosen": -384.0989074707031,
      "logps/ref_chosen": -57.6871337890625,
      "logps/ref_rejected": -80.62527465820312,
      "logps/rejected": -591.0260009765625,
      "loss": 1.1282,
      "margin_dpo/margin_mean": 183.98892211914062,
      "margin_dpo/margin_std": 254.2589569091797,
      "step": 360
    },
    {
      "epoch": 0.5359765051395007,
      "fcm_dpo/beta": 0.0018712872406467795,
      "fcm_dpo/delta": -0.026755044236779213,
      "fcm_dpo/margin": 227.2862091064453,
      "fcm_dpo/q_t": 0.40148013830184937,
      "grad_norm": 26.54566764831543,
      "learning_rate": 2.641091331089811e-07,
      "logits/chosen": -0.473797470331192,
      "logits/rejected": -0.47837966680526733,
      "logps/chosen": -337.5777893066406,
      "logps/ref_chosen": -51.490867614746094,
      "logps/ref_rejected": -91.02871704101562,
      "logps/rejected": -604.4019165039062,
      "loss": 1.0628,
      "margin_dpo/margin_mean": 227.28622436523438,
      "margin_dpo/margin_std": 260.75958251953125,
      "step": 365
    },
    {
      "epoch": 0.5433186490455213,
      "fcm_dpo/beta": 0.0018645223462954164,
      "fcm_dpo/delta": 0.038537509739398956,
      "fcm_dpo/margin": 194.5872039794922,
      "fcm_dpo/q_t": 0.41541823744773865,
      "grad_norm": 38.50527572631836,
      "learning_rate": 2.5769876463904263e-07,
      "logits/chosen": -0.47979289293289185,
      "logits/rejected": -0.4759935438632965,
      "logps/chosen": -354.23822021484375,
      "logps/ref_chosen": -58.113502502441406,
      "logps/ref_rejected": -89.43451690673828,
      "logps/rejected": -580.1464233398438,
      "loss": 1.1143,
      "margin_dpo/margin_mean": 194.5872039794922,
      "margin_dpo/margin_std": 256.3455505371094,
      "step": 370
    },
    {
      "epoch": 0.5506607929515418,
      "fcm_dpo/beta": 0.0018788978923112154,
      "fcm_dpo/delta": -0.028660928830504417,
      "fcm_dpo/margin": 227.169677734375,
      "fcm_dpo/q_t": 0.403217077255249,
      "grad_norm": 42.09442901611328,
      "learning_rate": 2.512833246961859e-07,
      "logits/chosen": -0.5087894201278687,
      "logits/rejected": -0.494729608297348,
      "logps/chosen": -409.12677001953125,
      "logps/ref_chosen": -65.23600769042969,
      "logps/ref_rejected": -89.24995422363281,
      "logps/rejected": -660.3104248046875,
      "loss": 1.0839,
      "margin_dpo/margin_mean": 227.169677734375,
      "margin_dpo/margin_std": 307.71136474609375,
      "step": 375
    },
    {
      "epoch": 0.5580029368575624,
      "fcm_dpo/beta": 0.001791507238522172,
      "fcm_dpo/delta": -0.07822184264659882,
      "fcm_dpo/margin": 264.9973449707031,
      "fcm_dpo/q_t": 0.39022326469421387,
      "grad_norm": 27.727758407592773,
      "learning_rate": 2.4486703937790243e-07,
      "logits/chosen": -0.4783501625061035,
      "logits/rejected": -0.49099501967430115,
      "logps/chosen": -376.74932861328125,
      "logps/ref_chosen": -53.33893966674805,
      "logps/ref_rejected": -102.15375518798828,
      "logps/rejected": -690.5615234375,
      "loss": 1.034,
      "margin_dpo/margin_mean": 264.9973449707031,
      "margin_dpo/margin_std": 298.83721923828125,
      "step": 380
    },
    {
      "epoch": 0.5653450807635829,
      "fcm_dpo/beta": 0.0017382428050041199,
      "fcm_dpo/delta": 0.007922522723674774,
      "fcm_dpo/margin": 225.6304931640625,
      "fcm_dpo/q_t": 0.40880244970321655,
      "grad_norm": 39.414588928222656,
      "learning_rate": 2.3845413533856514e-07,
      "logits/chosen": -0.4913402497768402,
      "logits/rejected": -0.4843069911003113,
      "logps/chosen": -414.2498474121094,
      "logps/ref_chosen": -58.36262130737305,
      "logps/ref_rejected": -89.44685363769531,
      "logps/rejected": -670.964599609375,
      "loss": 1.0946,
      "margin_dpo/margin_mean": 225.6304931640625,
      "margin_dpo/margin_std": 290.14666748046875,
      "step": 385
    },
    {
      "epoch": 0.5726872246696035,
      "fcm_dpo/beta": 0.0017035829368978739,
      "fcm_dpo/delta": -0.029813114553689957,
      "fcm_dpo/margin": 251.324462890625,
      "fcm_dpo/q_t": 0.4024105966091156,
      "grad_norm": 26.369205474853516,
      "learning_rate": 2.320488370051681e-07,
      "logits/chosen": -0.486684650182724,
      "logits/rejected": -0.4810120165348053,
      "logps/chosen": -435.18115234375,
      "logps/ref_chosen": -56.380653381347656,
      "logps/ref_rejected": -90.47447204589844,
      "logps/rejected": -720.5994873046875,
      "loss": 1.0741,
      "margin_dpo/margin_mean": 251.324462890625,
      "margin_dpo/margin_std": 322.6702575683594,
      "step": 390
    },
    {
      "epoch": 0.580029368575624,
      "fcm_dpo/beta": 0.0017207658383995295,
      "fcm_dpo/delta": 0.012584102340042591,
      "fcm_dpo/margin": 225.0916748046875,
      "fcm_dpo/q_t": 0.41174229979515076,
      "grad_norm": 31.417821884155273,
      "learning_rate": 2.2565536379453404e-07,
      "logits/chosen": -0.47851499915122986,
      "logits/rejected": -0.4759146273136139,
      "logps/chosen": -402.8694763183594,
      "logps/ref_chosen": -55.95304489135742,
      "logps/ref_rejected": -87.13162994384766,
      "logps/rejected": -659.1397705078125,
      "loss": 1.1036,
      "margin_dpo/margin_mean": 225.0916748046875,
      "margin_dpo/margin_std": 308.83575439453125,
      "step": 395
    },
    {
      "epoch": 0.5873715124816447,
      "fcm_dpo/beta": 0.0016688309842720628,
      "fcm_dpo/delta": -0.05934460088610649,
      "fcm_dpo/margin": 273.747314453125,
      "fcm_dpo/q_t": 0.39453184604644775,
      "grad_norm": 22.06127166748047,
      "learning_rate": 2.192779273338215e-07,
      "logits/chosen": -0.48663657903671265,
      "logits/rejected": -0.4787791669368744,
      "logps/chosen": -424.73455810546875,
      "logps/ref_chosen": -64.59160614013672,
      "logps/ref_rejected": -96.700927734375,
      "logps/rejected": -730.5912475585938,
      "loss": 1.0433,
      "margin_dpo/margin_mean": 273.747314453125,
      "margin_dpo/margin_std": 306.27801513671875,
      "step": 400
    },
    {
      "epoch": 0.5873715124816447,
      "eval_fcm_dpo/beta": 0.0018981158500537276,
      "eval_fcm_dpo/delta": 0.03762635216116905,
      "eval_fcm_dpo/margin": 156.90065002441406,
      "eval_fcm_dpo/q_t": 0.43355923891067505,
      "eval_logits/chosen": -0.5229986310005188,
      "eval_logits/rejected": -0.5119041800498962,
      "eval_logps/chosen": -532.2332763671875,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -696.880859375,
      "eval_loss": 0.6033138036727905,
      "eval_margin_dpo/margin_mean": 156.90061950683594,
      "eval_margin_dpo/margin_std": 328.45404052734375,
      "eval_runtime": 39.7397,
      "eval_samples_per_second": 58.858,
      "eval_steps_per_second": 1.862,
      "step": 400
    },
    {
      "epoch": 0.5947136563876652,
      "fcm_dpo/beta": 0.001960620516911149,
      "fcm_dpo/delta": -0.1365078240633011,
      "fcm_dpo/margin": 260.3297119140625,
      "fcm_dpo/q_t": 0.3874695897102356,
      "grad_norm": 38.00107955932617,
      "learning_rate": 2.129207286861638e-07,
      "logits/chosen": -0.505179226398468,
      "logits/rejected": -0.5002223253250122,
      "logps/chosen": -406.5534973144531,
      "logps/ref_chosen": -53.61777877807617,
      "logps/ref_rejected": -81.28938293457031,
      "logps/rejected": -694.5548095703125,
      "loss": 1.0528,
      "margin_dpo/margin_mean": 260.3297119140625,
      "margin_dpo/margin_std": 351.9696044921875,
      "step": 405
    },
    {
      "epoch": 0.6020558002936858,
      "fcm_dpo/beta": 0.0018232803558930755,
      "fcm_dpo/delta": -0.01396404393017292,
      "fcm_dpo/margin": 216.00399780273438,
      "fcm_dpo/q_t": 0.4089049696922302,
      "grad_norm": 22.69110679626465,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -0.5198922157287598,
      "logits/rejected": -0.508955717086792,
      "logps/chosen": -396.2334899902344,
      "logps/ref_chosen": -58.9287223815918,
      "logps/ref_rejected": -85.55818176269531,
      "logps/rejected": -638.8670043945312,
      "loss": 1.096,
      "margin_dpo/margin_mean": 216.00399780273438,
      "margin_dpo/margin_std": 284.52081298828125,
      "step": 410
    },
    {
      "epoch": 0.6093979441997063,
      "fcm_dpo/beta": 0.0017951425397768617,
      "fcm_dpo/delta": -0.023637911304831505,
      "fcm_dpo/margin": 235.01473999023438,
      "fcm_dpo/q_t": 0.4024273753166199,
      "grad_norm": 26.751333236694336,
      "learning_rate": 2.002837796667909e-07,
      "logits/chosen": -0.4968322813510895,
      "logits/rejected": -0.5035668015480042,
      "logps/chosen": -417.4717712402344,
      "logps/ref_chosen": -58.45662307739258,
      "logps/ref_rejected": -93.67063903808594,
      "logps/rejected": -687.7005615234375,
      "loss": 1.0778,
      "margin_dpo/margin_mean": 235.01473999023438,
      "margin_dpo/margin_std": 297.6578063964844,
      "step": 415
    },
    {
      "epoch": 0.6167400881057269,
      "fcm_dpo/beta": 0.001714545302093029,
      "fcm_dpo/delta": -0.04269330948591232,
      "fcm_dpo/margin": 256.39959716796875,
      "fcm_dpo/q_t": 0.39849138259887695,
      "grad_norm": 58.22108459472656,
      "learning_rate": 1.9401235374032425e-07,
      "logits/chosen": -0.47421860694885254,
      "logits/rejected": -0.46059077978134155,
      "logps/chosen": -428.48443603515625,
      "logps/ref_chosen": -64.2349853515625,
      "logps/ref_rejected": -84.86299133300781,
      "logps/rejected": -705.5120849609375,
      "loss": 1.062,
      "margin_dpo/margin_mean": 256.39959716796875,
      "margin_dpo/margin_std": 306.3155822753906,
      "step": 420
    },
    {
      "epoch": 0.6240822320117474,
      "fcm_dpo/beta": 0.0017159763956442475,
      "fcm_dpo/delta": 0.010966436937451363,
      "fcm_dpo/margin": 226.81314086914062,
      "fcm_dpo/q_t": 0.4094250798225403,
      "grad_norm": 45.61664962768555,
      "learning_rate": 1.8777780903377732e-07,
      "logits/chosen": -0.5051223635673523,
      "logits/rejected": -0.4999346137046814,
      "logps/chosen": -442.1536560058594,
      "logps/ref_chosen": -56.054161071777344,
      "logps/ref_rejected": -85.17119598388672,
      "logps/rejected": -698.0838012695312,
      "loss": 1.1005,
      "margin_dpo/margin_mean": 226.81314086914062,
      "margin_dpo/margin_std": 304.298583984375,
      "step": 425
    },
    {
      "epoch": 0.631424375917768,
      "fcm_dpo/beta": 0.0017323114443570375,
      "fcm_dpo/delta": 0.009447330608963966,
      "fcm_dpo/margin": 213.61093139648438,
      "fcm_dpo/q_t": 0.414214551448822,
      "grad_norm": 40.23195266723633,
      "learning_rate": 1.8158425248197928e-07,
      "logits/chosen": -0.5632457733154297,
      "logits/rejected": -0.5552536249160767,
      "logps/chosen": -472.23919677734375,
      "logps/ref_chosen": -69.24568176269531,
      "logps/ref_rejected": -91.8664321899414,
      "logps/rejected": -708.4708862304688,
      "loss": 1.1182,
      "margin_dpo/margin_mean": 213.61093139648438,
      "margin_dpo/margin_std": 301.84185791015625,
      "step": 430
    },
    {
      "epoch": 0.6387665198237885,
      "fcm_dpo/beta": 0.0017033193726092577,
      "fcm_dpo/delta": -0.027609745040535927,
      "fcm_dpo/margin": 249.95333862304688,
      "fcm_dpo/q_t": 0.4024946093559265,
      "grad_norm": 34.37434768676758,
      "learning_rate": 1.7543576401928218e-07,
      "logits/chosen": -0.618207573890686,
      "logits/rejected": -0.6184796690940857,
      "logps/chosen": -466.94012451171875,
      "logps/ref_chosen": -60.03449630737305,
      "logps/ref_rejected": -90.6872329711914,
      "logps/rejected": -747.5462036132812,
      "loss": 1.0796,
      "margin_dpo/margin_mean": 249.95333862304688,
      "margin_dpo/margin_std": 326.78277587890625,
      "step": 435
    },
    {
      "epoch": 0.6461086637298091,
      "fcm_dpo/beta": 0.0016936672618612647,
      "fcm_dpo/delta": -0.009997454471886158,
      "fcm_dpo/margin": 227.880126953125,
      "fcm_dpo/q_t": 0.41129302978515625,
      "grad_norm": 52.6486930847168,
      "learning_rate": 1.6933639389195134e-07,
      "logits/chosen": -0.647782027721405,
      "logits/rejected": -0.6332479119300842,
      "logps/chosen": -471.6587829589844,
      "logps/ref_chosen": -65.50349426269531,
      "logps/ref_rejected": -85.66627502441406,
      "logps/rejected": -719.70166015625,
      "loss": 1.1022,
      "margin_dpo/margin_mean": 227.88015747070312,
      "margin_dpo/margin_std": 307.6328125,
      "step": 440
    },
    {
      "epoch": 0.6534508076358296,
      "fcm_dpo/beta": 0.001662442460656166,
      "fcm_dpo/delta": -0.03005291521549225,
      "fcm_dpo/margin": 245.63253784179688,
      "fcm_dpo/q_t": 0.4081791043281555,
      "grad_norm": 32.25020980834961,
      "learning_rate": 1.6329015999011182e-07,
      "logits/chosen": -0.6261739730834961,
      "logits/rejected": -0.6163330674171448,
      "logps/chosen": -485.8421936035156,
      "logps/ref_chosen": -60.72443389892578,
      "logps/ref_rejected": -89.9255142211914,
      "logps/rejected": -760.6757202148438,
      "loss": 1.1036,
      "margin_dpo/margin_mean": 245.63253784179688,
      "margin_dpo/margin_std": 356.2879943847656,
      "step": 445
    },
    {
      "epoch": 0.6607929515418502,
      "fcm_dpo/beta": 0.0015721891541033983,
      "fcm_dpo/delta": -0.07007580995559692,
      "fcm_dpo/margin": 297.0084533691406,
      "fcm_dpo/q_t": 0.3933987319469452,
      "grad_norm": 37.675498962402344,
      "learning_rate": 1.573010452010098e-07,
      "logits/chosen": -0.674242377281189,
      "logits/rejected": -0.6845596432685852,
      "logps/chosen": -466.7557067871094,
      "logps/ref_chosen": -59.96248245239258,
      "logps/ref_rejected": -98.71509552001953,
      "logps/rejected": -802.5167846679688,
      "loss": 1.0453,
      "margin_dpo/margin_mean": 297.0084533691406,
      "margin_dpo/margin_std": 355.6662292480469,
      "step": 450
    },
    {
      "epoch": 0.6681350954478708,
      "fcm_dpo/beta": 0.0015043766470625997,
      "fcm_dpo/delta": -0.04557330161333084,
      "fcm_dpo/margin": 295.05352783203125,
      "fcm_dpo/q_t": 0.3984234929084778,
      "grad_norm": 46.07245635986328,
      "learning_rate": 1.5137299478533064e-07,
      "logits/chosen": -0.7228642702102661,
      "logits/rejected": -0.7254277467727661,
      "logps/chosen": -525.2667846679688,
      "logps/ref_chosen": -54.48131561279297,
      "logps/ref_rejected": -90.6321029663086,
      "logps/rejected": -856.4710693359375,
      "loss": 1.0728,
      "margin_dpo/margin_mean": 295.05352783203125,
      "margin_dpo/margin_std": 389.12469482421875,
      "step": 455
    },
    {
      "epoch": 0.6754772393538914,
      "fcm_dpo/beta": 0.0013938735937699676,
      "fcm_dpo/delta": -0.0647268071770668,
      "fcm_dpo/margin": 331.0070495605469,
      "fcm_dpo/q_t": 0.3938430845737457,
      "grad_norm": 38.70931625366211,
      "learning_rate": 1.4550991377830423e-07,
      "logits/chosen": -0.6967940926551819,
      "logits/rejected": -0.7114230394363403,
      "logps/chosen": -526.4164428710938,
      "logps/ref_chosen": -52.97611618041992,
      "logps/ref_rejected": -95.65971374511719,
      "logps/rejected": -900.1070556640625,
      "loss": 1.0402,
      "margin_dpo/margin_mean": 331.0070495605469,
      "margin_dpo/margin_std": 373.8094177246094,
      "step": 460
    },
    {
      "epoch": 0.6828193832599119,
      "fcm_dpo/beta": 0.0013582499232143164,
      "fcm_dpo/delta": -0.040646448731422424,
      "fcm_dpo/margin": 323.32489013671875,
      "fcm_dpo/q_t": 0.3995057940483093,
      "grad_norm": 29.232101440429688,
      "learning_rate": 1.3971566441730714e-07,
      "logits/chosen": -0.7399168014526367,
      "logits/rejected": -0.7570127248764038,
      "logps/chosen": -553.7732543945312,
      "logps/ref_chosen": -58.2827033996582,
      "logps/ref_rejected": -94.15567779541016,
      "logps/rejected": -912.9710693359375,
      "loss": 1.0657,
      "margin_dpo/margin_mean": 323.32489013671875,
      "margin_dpo/margin_std": 396.70892333984375,
      "step": 465
    },
    {
      "epoch": 0.6901615271659325,
      "fcm_dpo/beta": 0.0012759026139974594,
      "fcm_dpo/delta": -0.050003357231616974,
      "fcm_dpo/margin": 350.7779235839844,
      "fcm_dpo/q_t": 0.3979727625846863,
      "grad_norm": 27.062070846557617,
      "learning_rate": 1.339940635976592e-07,
      "logits/chosen": -0.7901474833488464,
      "logits/rejected": -0.7927638292312622,
      "logps/chosen": -599.5657348632812,
      "logps/ref_chosen": -62.69774627685547,
      "logps/ref_rejected": -96.33873748779297,
      "logps/rejected": -983.9846801757812,
      "loss": 1.0702,
      "margin_dpo/margin_mean": 350.7779846191406,
      "margin_dpo/margin_std": 457.7704162597656,
      "step": 470
    },
    {
      "epoch": 0.697503671071953,
      "fcm_dpo/beta": 0.0012454693205654621,
      "fcm_dpo/delta": -0.008181336335837841,
      "fcm_dpo/margin": 326.77447509765625,
      "fcm_dpo/q_t": 0.41024675965309143,
      "grad_norm": 34.630069732666016,
      "learning_rate": 1.2834888035828596e-07,
      "logits/chosen": -0.9200963973999023,
      "logits/rejected": -0.9176486730575562,
      "logps/chosen": -705.766845703125,
      "logps/ref_chosen": -61.12194061279297,
      "logps/ref_rejected": -92.42192077636719,
      "logps/rejected": -1063.84130859375,
      "loss": 1.1387,
      "margin_dpo/margin_mean": 326.7744140625,
      "margin_dpo/margin_std": 567.3615112304688,
      "step": 475
    },
    {
      "epoch": 0.7048458149779736,
      "fcm_dpo/beta": 0.0012077607680112123,
      "fcm_dpo/delta": -0.04966871812939644,
      "fcm_dpo/margin": 349.90106201171875,
      "fcm_dpo/q_t": 0.4062713086605072,
      "grad_norm": 62.070411682128906,
      "learning_rate": 1.227838333989088e-07,
      "logits/chosen": -0.9689761996269226,
      "logits/rejected": -0.9785006642341614,
      "logps/chosen": -730.4820556640625,
      "logps/ref_chosen": -53.550628662109375,
      "logps/ref_rejected": -82.65167999267578,
      "logps/rejected": -1109.484130859375,
      "loss": 1.111,
      "margin_dpo/margin_mean": 349.90106201171875,
      "margin_dpo/margin_std": 551.5509033203125,
      "step": 480
    },
    {
      "epoch": 0.7121879588839941,
      "fcm_dpo/beta": 0.001141466898843646,
      "fcm_dpo/delta": -0.044795047491788864,
      "fcm_dpo/margin": 370.09796142578125,
      "fcm_dpo/q_t": 0.404446542263031,
      "grad_norm": 33.84178924560547,
      "learning_rate": 1.1730258863039347e-07,
      "logits/chosen": -0.9285829663276672,
      "logits/rejected": -0.9435798525810242,
      "logps/chosen": -689.6338500976562,
      "logps/ref_chosen": -60.76704788208008,
      "logps/ref_rejected": -92.70649719238281,
      "logps/rejected": -1091.671142578125,
      "loss": 1.0989,
      "margin_dpo/margin_mean": 370.09796142578125,
      "margin_dpo/margin_std": 542.5574951171875,
      "step": 485
    },
    {
      "epoch": 0.7195301027900147,
      "fcm_dpo/beta": 0.0011192201636731625,
      "fcm_dpo/delta": 0.0058375876396894455,
      "fcm_dpo/margin": 352.23004150390625,
      "fcm_dpo/q_t": 0.4108501374721527,
      "grad_norm": 50.5614128112793,
      "learning_rate": 1.1190875675987355e-07,
      "logits/chosen": -0.9131494760513306,
      "logits/rejected": -0.9286991953849792,
      "logps/chosen": -628.578369140625,
      "logps/ref_chosen": -53.78142547607422,
      "logps/ref_rejected": -91.67438507080078,
      "logps/rejected": -1018.7013549804688,
      "loss": 1.1102,
      "margin_dpo/margin_mean": 352.2300109863281,
      "margin_dpo/margin_std": 520.4669189453125,
      "step": 490
    },
    {
      "epoch": 0.7268722466960352,
      "fcm_dpo/beta": 0.0011620967416092753,
      "fcm_dpo/delta": 0.019140174612402916,
      "fcm_dpo/margin": 309.65850830078125,
      "fcm_dpo/q_t": 0.418142706155777,
      "grad_norm": 35.54931640625,
      "learning_rate": 1.0660589091223854e-07,
      "logits/chosen": -0.9482976794242859,
      "logits/rejected": -0.9485372304916382,
      "logps/chosen": -638.1903076171875,
      "logps/ref_chosen": -58.9004020690918,
      "logps/ref_rejected": -82.15424346923828,
      "logps/rejected": -971.1026611328125,
      "loss": 1.1356,
      "margin_dpo/margin_mean": 309.6585388183594,
      "margin_dpo/margin_std": 488.75909423828125,
      "step": 495
    },
    {
      "epoch": 0.7342143906020558,
      "fcm_dpo/beta": 0.0011855836492031813,
      "fcm_dpo/delta": 0.024493711069226265,
      "fcm_dpo/margin": 291.8643798828125,
      "fcm_dpo/q_t": 0.42091307044029236,
      "grad_norm": 30.435544967651367,
      "learning_rate": 1.0139748428955333e-07,
      "logits/chosen": -0.9160359501838684,
      "logits/rejected": -0.91932213306427,
      "logps/chosen": -699.1871948242188,
      "logps/ref_chosen": -62.13483810424805,
      "logps/ref_rejected": -87.91773223876953,
      "logps/rejected": -1016.83447265625,
      "loss": 1.1519,
      "margin_dpo/margin_mean": 291.8643798828125,
      "margin_dpo/margin_std": 487.0228576660156,
      "step": 500
    },
    {
      "epoch": 0.7415565345080763,
      "fcm_dpo/beta": 0.0011691536055877805,
      "fcm_dpo/delta": 0.006966643966734409,
      "fcm_dpo/margin": 336.22186279296875,
      "fcm_dpo/q_t": 0.41082048416137695,
      "grad_norm": 31.930696487426758,
      "learning_rate": 9.628696786995188e-08,
      "logits/chosen": -0.8171493411064148,
      "logits/rejected": -0.8128056526184082,
      "logps/chosen": -598.7544555664062,
      "logps/ref_chosen": -62.631813049316406,
      "logps/ref_rejected": -87.59168243408203,
      "logps/rejected": -959.9362182617188,
      "loss": 1.1017,
      "margin_dpo/margin_mean": 336.2218322753906,
      "margin_dpo/margin_std": 463.255615234375,
      "step": 505
    },
    {
      "epoch": 0.748898678414097,
      "fcm_dpo/beta": 0.0012091896496713161,
      "fcm_dpo/delta": 0.028515011072158813,
      "fcm_dpo/margin": 307.8777770996094,
      "fcm_dpo/q_t": 0.4152832627296448,
      "grad_norm": 37.425514221191406,
      "learning_rate": 9.127770814751932e-08,
      "logits/chosen": -0.7757030129432678,
      "logits/rejected": -0.7864035367965698,
      "logps/chosen": -574.5384521484375,
      "logps/ref_chosen": -60.552574157714844,
      "logps/ref_rejected": -91.0874252319336,
      "logps/rejected": -912.951171875,
      "loss": 1.1211,
      "margin_dpo/margin_mean": 307.8777770996094,
      "margin_dpo/margin_std": 457.5035705566406,
      "step": 510
    },
    {
      "epoch": 0.7562408223201175,
      "fcm_dpo/beta": 0.0012351378099992871,
      "fcm_dpo/delta": 0.04284387826919556,
      "fcm_dpo/margin": 290.5696105957031,
      "fcm_dpo/q_t": 0.41743749380111694,
      "grad_norm": 42.793312072753906,
      "learning_rate": 8.637300491465272e-08,
      "logits/chosen": -0.7973334789276123,
      "logits/rejected": -0.8029496073722839,
      "logps/chosen": -538.7898559570312,
      "logps/ref_chosen": -60.9382438659668,
      "logps/ref_rejected": -83.36767578125,
      "logps/rejected": -851.7888793945312,
      "loss": 1.1273,
      "margin_dpo/margin_mean": 290.5696105957031,
      "margin_dpo/margin_std": 429.46759033203125,
      "step": 515
    },
    {
      "epoch": 0.7635829662261381,
      "fcm_dpo/beta": 0.0012580296024680138,
      "fcm_dpo/delta": 0.001387386815622449,
      "fcm_dpo/margin": 316.6277770996094,
      "fcm_dpo/q_t": 0.4091448187828064,
      "grad_norm": 41.04801559448242,
      "learning_rate": 8.15760890883607e-08,
      "logits/chosen": -0.8406698107719421,
      "logits/rejected": -0.8473473787307739,
      "logps/chosen": -598.2073974609375,
      "logps/ref_chosen": -65.47642517089844,
      "logps/ref_rejected": -98.10872650146484,
      "logps/rejected": -947.4674072265625,
      "loss": 1.1012,
      "margin_dpo/margin_mean": 316.6277770996094,
      "margin_dpo/margin_std": 440.81640625,
      "step": 520
    },
    {
      "epoch": 0.7709251101321586,
      "fcm_dpo/beta": 0.0012629171833395958,
      "fcm_dpo/delta": -0.03817005455493927,
      "fcm_dpo/margin": 345.6103820800781,
      "fcm_dpo/q_t": 0.4016591012477875,
      "grad_norm": 33.16600799560547,
      "learning_rate": 7.689012058193384e-08,
      "logits/chosen": -0.9183955192565918,
      "logits/rejected": -0.9257911443710327,
      "logps/chosen": -636.746337890625,
      "logps/ref_chosen": -59.072021484375,
      "logps/ref_rejected": -82.8821792602539,
      "logps/rejected": -1006.1668701171875,
      "loss": 1.0892,
      "margin_dpo/margin_mean": 345.6103515625,
      "margin_dpo/margin_std": 494.670654296875,
      "step": 525
    },
    {
      "epoch": 0.7782672540381792,
      "fcm_dpo/beta": 0.001211377326399088,
      "fcm_dpo/delta": -0.020752830430865288,
      "fcm_dpo/margin": 330.8208923339844,
      "fcm_dpo/q_t": 0.4099728465080261,
      "grad_norm": 49.311241149902344,
      "learning_rate": 7.231818622338822e-08,
      "logits/chosen": -0.9991754293441772,
      "logits/rejected": -0.9904956817626953,
      "logps/chosen": -721.9043579101562,
      "logps/ref_chosen": -61.11234664916992,
      "logps/ref_rejected": -87.01112365722656,
      "logps/rejected": -1078.6241455078125,
      "loss": 1.1306,
      "margin_dpo/margin_mean": 330.8208923339844,
      "margin_dpo/margin_std": 549.4489135742188,
      "step": 530
    },
    {
      "epoch": 0.7856093979441997,
      "fcm_dpo/beta": 0.001194678363390267,
      "fcm_dpo/delta": -0.01992225833237171,
      "fcm_dpo/margin": 334.68017578125,
      "fcm_dpo/q_t": 0.4102866053581238,
      "grad_norm": 28.87725067138672,
      "learning_rate": 6.786329772205246e-08,
      "logits/chosen": -0.9631765484809875,
      "logits/rejected": -0.9504965543746948,
      "logps/chosen": -703.7914428710938,
      "logps/ref_chosen": -60.96736526489258,
      "logps/ref_rejected": -81.727294921875,
      "logps/rejected": -1059.231689453125,
      "loss": 1.1238,
      "margin_dpo/margin_mean": 334.68017578125,
      "margin_dpo/margin_std": 534.3411865234375,
      "step": 535
    },
    {
      "epoch": 0.7929515418502202,
      "fcm_dpo/beta": 0.001148082548752427,
      "fcm_dpo/delta": -0.046564746648073196,
      "fcm_dpo/margin": 386.79718017578125,
      "fcm_dpo/q_t": 0.4005100131034851,
      "grad_norm": 40.99845886230469,
      "learning_rate": 6.352838968463919e-08,
      "logits/chosen": -0.8846408128738403,
      "logits/rejected": -0.8896158337593079,
      "logps/chosen": -599.884765625,
      "logps/ref_chosen": -58.64385986328125,
      "logps/ref_rejected": -90.40965270996094,
      "logps/rejected": -1018.4476318359375,
      "loss": 1.0741,
      "margin_dpo/margin_mean": 386.7971496582031,
      "margin_dpo/margin_std": 522.71826171875,
      "step": 540
    },
    {
      "epoch": 0.8002936857562408,
      "fcm_dpo/beta": 0.0011082093697041273,
      "fcm_dpo/delta": -0.027572477236390114,
      "fcm_dpo/margin": 361.2355651855469,
      "fcm_dpo/q_t": 0.4120791554450989,
      "grad_norm": 31.77776527404785,
      "learning_rate": 5.9316317682106294e-08,
      "logits/chosen": -0.9308038949966431,
      "logits/rejected": -0.9519888758659363,
      "logps/chosen": -702.5291137695312,
      "logps/ref_chosen": -64.73474884033203,
      "logps/ref_rejected": -100.44208526611328,
      "logps/rejected": -1099.471923828125,
      "loss": 1.1263,
      "margin_dpo/margin_mean": 361.2355651855469,
      "margin_dpo/margin_std": 592.0833740234375,
      "step": 545
    },
    {
      "epoch": 0.8076358296622613,
      "fcm_dpo/beta": 0.0011139644775539637,
      "fcm_dpo/delta": 0.027384355664253235,
      "fcm_dpo/margin": 318.26593017578125,
      "fcm_dpo/q_t": 0.42001286149024963,
      "grad_norm": 41.989044189453125,
      "learning_rate": 5.5229856368582376e-08,
      "logits/chosen": -0.9026668667793274,
      "logits/rejected": -0.9054571390151978,
      "logps/chosen": -672.8770751953125,
      "logps/ref_chosen": -59.13951873779297,
      "logps/ref_rejected": -87.15635681152344,
      "logps/rejected": -1019.1598510742188,
      "loss": 1.1374,
      "margin_dpo/margin_mean": 318.26593017578125,
      "margin_dpo/margin_std": 509.5707092285156,
      "step": 550
    },
    {
      "epoch": 0.8149779735682819,
      "fcm_dpo/beta": 0.0010551275918260217,
      "fcm_dpo/delta": -0.08985067903995514,
      "fcm_dpo/margin": 458.0889587402344,
      "fcm_dpo/q_t": 0.39341944456100464,
      "grad_norm": 37.768455505371094,
      "learning_rate": 5.127169765359515e-08,
      "logits/chosen": -0.9196065664291382,
      "logits/rejected": -0.9442132115364075,
      "logps/chosen": -678.6885986328125,
      "logps/ref_chosen": -62.1995849609375,
      "logps/ref_rejected": -102.51883697509766,
      "logps/rejected": -1177.096923828125,
      "loss": 1.0596,
      "margin_dpo/margin_mean": 458.0889587402344,
      "margin_dpo/margin_std": 621.8514404296875,
      "step": 555
    },
    {
      "epoch": 0.8223201174743024,
      "fcm_dpo/beta": 0.001013588160276413,
      "fcm_dpo/delta": -0.03401713818311691,
      "fcm_dpo/margin": 426.43377685546875,
      "fcm_dpo/q_t": 0.4025075435638428,
      "grad_norm": 32.85615158081055,
      "learning_rate": 4.7444448928806615e-08,
      "logits/chosen": -0.9791582226753235,
      "logits/rejected": -0.987125039100647,
      "logps/chosen": -729.31787109375,
      "logps/ref_chosen": -61.541908264160156,
      "logps/ref_rejected": -97.37491607666016,
      "logps/rejected": -1191.584716796875,
      "loss": 1.0859,
      "margin_dpo/margin_mean": 426.43377685546875,
      "margin_dpo/margin_std": 586.039794921875,
      "step": 560
    },
    {
      "epoch": 0.8296622613803231,
      "fcm_dpo/beta": 0.0010072619188576937,
      "fcm_dpo/delta": 0.004320817068219185,
      "fcm_dpo/margin": 365.18194580078125,
      "fcm_dpo/q_t": 0.41634687781333923,
      "grad_norm": 38.367191314697266,
      "learning_rate": 4.375063135042445e-08,
      "logits/chosen": -1.0174771547317505,
      "logits/rejected": -1.0257703065872192,
      "logps/chosen": -805.9225463867188,
      "logps/ref_chosen": -62.85475540161133,
      "logps/ref_rejected": -93.8392105102539,
      "logps/rejected": -1202.0889892578125,
      "loss": 1.1532,
      "margin_dpo/margin_mean": 365.18194580078125,
      "margin_dpo/margin_std": 646.3736572265625,
      "step": 565
    },
    {
      "epoch": 0.8370044052863436,
      "fcm_dpo/beta": 0.000991632230579853,
      "fcm_dpo/delta": -0.003437476698309183,
      "fcm_dpo/margin": 406.2334899902344,
      "fcm_dpo/q_t": 0.40820297598838806,
      "grad_norm": 32.0954475402832,
      "learning_rate": 4.019267817841834e-08,
      "logits/chosen": -1.057476282119751,
      "logits/rejected": -1.0744601488113403,
      "logps/chosen": -756.536865234375,
      "logps/ref_chosen": -57.98622512817383,
      "logps/ref_rejected": -87.91555786132812,
      "logps/rejected": -1192.6998291015625,
      "loss": 1.108,
      "margin_dpo/margin_mean": 406.2334899902344,
      "margin_dpo/margin_std": 592.2410278320312,
      "step": 570
    },
    {
      "epoch": 0.8443465491923642,
      "fcm_dpo/beta": 0.000984189915470779,
      "fcm_dpo/delta": -0.0074859424494206905,
      "fcm_dpo/margin": 413.28948974609375,
      "fcm_dpo/q_t": 0.4089413583278656,
      "grad_norm": 39.946693420410156,
      "learning_rate": 3.677293317363864e-08,
      "logits/chosen": -1.012502908706665,
      "logits/rejected": -1.0302174091339111,
      "logps/chosen": -763.88623046875,
      "logps/ref_chosen": -55.194114685058594,
      "logps/ref_rejected": -89.68229675292969,
      "logps/rejected": -1211.663818359375,
      "loss": 1.1112,
      "margin_dpo/margin_mean": 413.28948974609375,
      "margin_dpo/margin_std": 633.7578735351562,
      "step": 575
    },
    {
      "epoch": 0.8516886930983847,
      "fcm_dpo/beta": 0.0010015666484832764,
      "fcm_dpo/delta": 0.005687057971954346,
      "fcm_dpo/margin": 373.99749755859375,
      "fcm_dpo/q_t": 0.41545066237449646,
      "grad_norm": 36.99066925048828,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -0.9414408802986145,
      "logits/rejected": -0.9566031694412231,
      "logps/chosen": -681.4942626953125,
      "logps/ref_chosen": -54.605796813964844,
      "logps/ref_rejected": -88.9614486694336,
      "logps/rejected": -1089.847412109375,
      "loss": 1.1256,
      "margin_dpo/margin_mean": 373.99749755859375,
      "margin_dpo/margin_std": 568.8309326171875,
      "step": 580
    },
    {
      "epoch": 0.8590308370044053,
      "fcm_dpo/beta": 0.0009860583813861012,
      "fcm_dpo/delta": -0.024760950356721878,
      "fcm_dpo/margin": 374.35772705078125,
      "fcm_dpo/q_t": 0.4164944589138031,
      "grad_norm": 30.979644775390625,
      "learning_rate": 3.035698600998121e-08,
      "logits/chosen": -1.005885362625122,
      "logits/rejected": -1.0213685035705566,
      "logps/chosen": -722.11083984375,
      "logps/ref_chosen": -59.03770065307617,
      "logps/ref_rejected": -94.4029541015625,
      "logps/rejected": -1131.833984375,
      "loss": 1.1365,
      "margin_dpo/margin_mean": 374.35772705078125,
      "margin_dpo/margin_std": 598.1573486328125,
      "step": 585
    },
    {
      "epoch": 0.8663729809104258,
      "fcm_dpo/beta": 0.0009923167526721954,
      "fcm_dpo/delta": 0.020330144092440605,
      "fcm_dpo/margin": 382.85650634765625,
      "fcm_dpo/q_t": 0.41268691420555115,
      "grad_norm": 26.06329917907715,
      "learning_rate": 2.736501028272095e-08,
      "logits/chosen": -1.001636266708374,
      "logits/rejected": -1.0243651866912842,
      "logps/chosen": -699.3173828125,
      "logps/ref_chosen": -53.5163688659668,
      "logps/ref_rejected": -99.90290832519531,
      "logps/rejected": -1128.5604248046875,
      "loss": 1.1114,
      "margin_dpo/margin_mean": 382.85650634765625,
      "margin_dpo/margin_std": 544.3261108398438,
      "step": 590
    },
    {
      "epoch": 0.8737151248164464,
      "fcm_dpo/beta": 0.0009959937306120992,
      "fcm_dpo/delta": 0.008491059765219688,
      "fcm_dpo/margin": 356.68133544921875,
      "fcm_dpo/q_t": 0.4196888506412506,
      "grad_norm": 51.35660934448242,
      "learning_rate": 2.451969280180849e-08,
      "logits/chosen": -0.9872266054153442,
      "logits/rejected": -0.9868671298027039,
      "logps/chosen": -701.3692626953125,
      "logps/ref_chosen": -51.44538497924805,
      "logps/ref_rejected": -77.43083190917969,
      "logps/rejected": -1084.035888671875,
      "loss": 1.1503,
      "margin_dpo/margin_mean": 356.68133544921875,
      "margin_dpo/margin_std": 606.1689453125,
      "step": 595
    },
    {
      "epoch": 0.8810572687224669,
      "fcm_dpo/beta": 0.0010021307971328497,
      "fcm_dpo/delta": 0.01705138385295868,
      "fcm_dpo/margin": 358.8981018066406,
      "fcm_dpo/q_t": 0.418224960565567,
      "grad_norm": 36.674869537353516,
      "learning_rate": 2.1822907887504932e-08,
      "logits/chosen": -0.9832857251167297,
      "logits/rejected": -0.9899972677230835,
      "logps/chosen": -721.6881103515625,
      "logps/ref_chosen": -57.161705017089844,
      "logps/ref_rejected": -87.73274230957031,
      "logps/rejected": -1111.1573486328125,
      "loss": 1.1393,
      "margin_dpo/margin_mean": 358.89813232421875,
      "margin_dpo/margin_std": 566.36669921875,
      "step": 600
    },
    {
      "epoch": 0.8810572687224669,
      "eval_fcm_dpo/beta": 0.0011259522289037704,
      "eval_fcm_dpo/delta": 0.023851953446865082,
      "eval_fcm_dpo/margin": 270.7278747558594,
      "eval_fcm_dpo/q_t": 0.4331645667552948,
      "eval_logits/chosen": -1.0197498798370361,
      "eval_logits/rejected": -1.0176793336868286,
      "eval_logps/chosen": -823.232421875,
      "eval_logps/ref_chosen": -79.05104064941406,
      "eval_logps/ref_rejected": -86.79793548583984,
      "eval_logps/rejected": -1101.70703125,
      "eval_loss": 0.6074439883232117,
      "eval_margin_dpo/margin_mean": 270.7278747558594,
      "eval_margin_dpo/margin_std": 601.7490234375,
      "eval_runtime": 39.7407,
      "eval_samples_per_second": 58.856,
      "eval_steps_per_second": 1.862,
      "step": 600
    },
    {
      "epoch": 0.8883994126284875,
      "fcm_dpo/beta": 0.0011384881800040603,
      "fcm_dpo/delta": -0.09710332006216049,
      "fcm_dpo/margin": 432.0416564941406,
      "fcm_dpo/q_t": 0.39160969853401184,
      "grad_norm": 36.93719482421875,
      "learning_rate": 1.9276432015946446e-08,
      "logits/chosen": -0.9972618222236633,
      "logits/rejected": -1.01743483543396,
      "logps/chosen": -690.4810791015625,
      "logps/ref_chosen": -58.169830322265625,
      "logps/ref_rejected": -95.36891174316406,
      "logps/rejected": -1159.7216796875,
      "loss": 1.0561,
      "margin_dpo/margin_mean": 432.0416564941406,
      "margin_dpo/margin_std": 594.4193725585938,
      "step": 605
    },
    {
      "epoch": 0.895741556534508,
      "fcm_dpo/beta": 0.0010870896512642503,
      "fcm_dpo/delta": -0.01724116876721382,
      "fcm_dpo/margin": 382.8441467285156,
      "fcm_dpo/q_t": 0.4077000617980957,
      "grad_norm": 48.549068450927734,
      "learning_rate": 1.6881942648911074e-08,
      "logits/chosen": -0.9739276766777039,
      "logits/rejected": -0.978052020072937,
      "logps/chosen": -703.338623046875,
      "logps/ref_chosen": -58.97087860107422,
      "logps/ref_rejected": -89.0286865234375,
      "logps/rejected": -1116.2406005859375,
      "loss": 1.102,
      "margin_dpo/margin_mean": 382.8440856933594,
      "margin_dpo/margin_std": 576.8626708984375,
      "step": 610
    },
    {
      "epoch": 0.9030837004405287,
      "fcm_dpo/beta": 0.001023916294798255,
      "fcm_dpo/delta": -0.07400115579366684,
      "fcm_dpo/margin": 458.7464294433594,
      "fcm_dpo/q_t": 0.39545583724975586,
      "grad_norm": 26.998958587646484,
      "learning_rate": 1.4641017128809801e-08,
      "logits/chosen": -0.9687691926956177,
      "logits/rejected": -1.0111548900604248,
      "logps/chosen": -723.0494384765625,
      "logps/ref_chosen": -58.081878662109375,
      "logps/ref_rejected": -95.92155456542969,
      "logps/rejected": -1219.635498046875,
      "loss": 1.0677,
      "margin_dpo/margin_mean": 458.7464904785156,
      "margin_dpo/margin_std": 639.427734375,
      "step": 615
    },
    {
      "epoch": 0.9104258443465492,
      "fcm_dpo/beta": 0.0010129540460184216,
      "fcm_dpo/delta": 0.012466437183320522,
      "fcm_dpo/margin": 362.10418701171875,
      "fcm_dpo/q_t": 0.4198682904243469,
      "grad_norm": 31.997928619384766,
      "learning_rate": 1.2555131639630567e-08,
      "logits/chosen": -0.9928318858146667,
      "logits/rejected": -0.9981027841567993,
      "logps/chosen": -801.2032470703125,
      "logps/ref_chosen": -62.203094482421875,
      "logps/ref_rejected": -80.53683471679688,
      "logps/rejected": -1181.641357421875,
      "loss": 1.1594,
      "margin_dpo/margin_mean": 362.10418701171875,
      "margin_dpo/margin_std": 658.1754150390625,
      "step": 620
    },
    {
      "epoch": 0.9177679882525698,
      "fcm_dpo/beta": 0.0009900578297674656,
      "fcm_dpo/delta": -0.04029404744505882,
      "fcm_dpo/margin": 442.16802978515625,
      "fcm_dpo/q_t": 0.4008522033691406,
      "grad_norm": 67.80354309082031,
      "learning_rate": 1.0625660234518913e-08,
      "logits/chosen": -0.9926285743713379,
      "logits/rejected": -1.0111424922943115,
      "logps/chosen": -743.4299926757812,
      "logps/ref_chosen": -61.727455139160156,
      "logps/ref_rejected": -88.4387435913086,
      "logps/rejected": -1212.309326171875,
      "loss": 1.0786,
      "margin_dpo/margin_mean": 442.1680603027344,
      "margin_dpo/margin_std": 596.7000732421875,
      "step": 625
    },
    {
      "epoch": 0.9251101321585903,
      "fcm_dpo/beta": 0.0009705528500489891,
      "fcm_dpo/delta": -0.0021920218132436275,
      "fcm_dpo/margin": 385.10406494140625,
      "fcm_dpo/q_t": 0.41799044609069824,
      "grad_norm": 57.19618225097656,
      "learning_rate": 8.85387393063622e-09,
      "logits/chosen": -0.981094479560852,
      "logits/rejected": -0.999189019203186,
      "logps/chosen": -764.3931884765625,
      "logps/ref_chosen": -61.30865478515625,
      "logps/ref_rejected": -96.54997253417969,
      "logps/rejected": -1184.7386474609375,
      "loss": 1.1426,
      "margin_dpo/margin_mean": 385.10406494140625,
      "margin_dpo/margin_std": 646.9403076171875,
      "step": 630
    },
    {
      "epoch": 0.9324522760646109,
      "fcm_dpo/beta": 0.000991217908449471,
      "fcm_dpo/delta": 0.04439081624150276,
      "fcm_dpo/margin": 321.2152099609375,
      "fcm_dpo/q_t": 0.42787250876426697,
      "grad_norm": 41.87347412109375,
      "learning_rate": 7.240939871891699e-09,
      "logits/chosen": -1.0177139043807983,
      "logits/rejected": -1.0183391571044922,
      "logps/chosen": -819.9927978515625,
      "logps/ref_chosen": -63.7315673828125,
      "logps/ref_rejected": -89.66435241699219,
      "logps/rejected": -1167.1407470703125,
      "loss": 1.1768,
      "margin_dpo/margin_mean": 321.2152404785156,
      "margin_dpo/margin_std": 599.1561889648438,
      "step": 635
    },
    {
      "epoch": 0.9397944199706314,
      "fcm_dpo/beta": 0.0009926257189363241,
      "fcm_dpo/delta": -0.05684474855661392,
      "fcm_dpo/margin": 380.9329528808594,
      "fcm_dpo/q_t": 0.41502299904823303,
      "grad_norm": 40.166175842285156,
      "learning_rate": 5.7879205600998296e-09,
      "logits/chosen": -0.9638550877571106,
      "logits/rejected": -0.9688556790351868,
      "logps/chosen": -797.8259887695312,
      "logps/ref_chosen": -59.17915725708008,
      "logps/ref_rejected": -88.51210021972656,
      "logps/rejected": -1208.0919189453125,
      "loss": 1.1439,
      "margin_dpo/margin_mean": 380.9329528808594,
      "margin_dpo/margin_std": 651.566162109375,
      "step": 640
    },
    {
      "epoch": 0.947136563876652,
      "fcm_dpo/beta": 0.0009744317503646016,
      "fcm_dpo/delta": 0.0341024175286293,
      "fcm_dpo/margin": 321.48162841796875,
      "fcm_dpo/q_t": 0.43180274963378906,
      "grad_norm": 79.06712341308594,
      "learning_rate": 4.495773155069299e-09,
      "logits/chosen": -0.9938424229621887,
      "logits/rejected": -1.0045819282531738,
      "logps/chosen": -791.4851684570312,
      "logps/ref_chosen": -59.50596237182617,
      "logps/ref_rejected": -93.92404174804688,
      "logps/rejected": -1147.3848876953125,
      "loss": 1.1994,
      "margin_dpo/margin_mean": 321.48162841796875,
      "margin_dpo/margin_std": 670.0167846679688,
      "step": 645
    },
    {
      "epoch": 0.9544787077826725,
      "fcm_dpo/beta": 0.0009998297318816185,
      "fcm_dpo/delta": 0.0030700929928570986,
      "fcm_dpo/margin": 377.16595458984375,
      "fcm_dpo/q_t": 0.41648992896080017,
      "grad_norm": 39.774078369140625,
      "learning_rate": 3.3653488440851253e-09,
      "logits/chosen": -0.994001567363739,
      "logits/rejected": -1.0127675533294678,
      "logps/chosen": -719.2337036132812,
      "logps/ref_chosen": -57.774566650390625,
      "logps/ref_rejected": -89.61600494384766,
      "logps/rejected": -1128.2410888671875,
      "loss": 1.1354,
      "margin_dpo/margin_mean": 377.16595458984375,
      "margin_dpo/margin_std": 618.4581298828125,
      "step": 650
    },
    {
      "epoch": 0.9618208516886931,
      "fcm_dpo/beta": 0.0009819023543968797,
      "fcm_dpo/delta": 0.001074020517989993,
      "fcm_dpo/margin": 406.14300537109375,
      "fcm_dpo/q_t": 0.41048040986061096,
      "grad_norm": 35.741294860839844,
      "learning_rate": 2.397392281198729e-09,
      "logits/chosen": -1.0151567459106445,
      "logits/rejected": -1.0480637550354004,
      "logps/chosen": -682.8621826171875,
      "logps/ref_chosen": -55.68403244018555,
      "logps/ref_rejected": -102.4081802368164,
      "logps/rejected": -1135.7293701171875,
      "loss": 1.1068,
      "margin_dpo/margin_mean": 406.1430358886719,
      "margin_dpo/margin_std": 598.8343505859375,
      "step": 655
    },
    {
      "epoch": 0.9691629955947136,
      "fcm_dpo/beta": 0.0009497703285887837,
      "fcm_dpo/delta": -0.056020140647888184,
      "fcm_dpo/margin": 475.952880859375,
      "fcm_dpo/q_t": 0.39766284823417664,
      "grad_norm": 36.36381149291992,
      "learning_rate": 1.592541096695571e-09,
      "logits/chosen": -0.988078773021698,
      "logits/rejected": -1.017830491065979,
      "logps/chosen": -716.3291625976562,
      "logps/ref_chosen": -59.19981002807617,
      "logps/ref_rejected": -94.19200134277344,
      "logps/rejected": -1227.274169921875,
      "loss": 1.0617,
      "margin_dpo/margin_mean": 475.95281982421875,
      "margin_dpo/margin_std": 605.3031005859375,
      "step": 660
    },
    {
      "epoch": 0.9765051395007343,
      "fcm_dpo/beta": 0.0009469041833654046,
      "fcm_dpo/delta": 0.0041985055431723595,
      "fcm_dpo/margin": 417.9197692871094,
      "fcm_dpo/q_t": 0.41134023666381836,
      "grad_norm": 40.71209716796875,
      "learning_rate": 9.513254770636137e-10,
      "logits/chosen": -1.0182468891143799,
      "logits/rejected": -1.0391613245010376,
      "logps/chosen": -698.357421875,
      "logps/ref_chosen": -61.2533073425293,
      "logps/ref_rejected": -95.86351013183594,
      "logps/rejected": -1150.887451171875,
      "loss": 1.1093,
      "margin_dpo/margin_mean": 417.9197692871094,
      "margin_dpo/margin_std": 626.8549194335938,
      "step": 665
    },
    {
      "epoch": 0.9838472834067548,
      "fcm_dpo/beta": 0.0009450761717744172,
      "fcm_dpo/delta": -0.012223686091601849,
      "fcm_dpo/margin": 435.3887634277344,
      "fcm_dpo/q_t": 0.40688472986221313,
      "grad_norm": 29.223600387573242,
      "learning_rate": 4.741678157389739e-10,
      "logits/chosen": -0.9618457555770874,
      "logits/rejected": -0.9741120338439941,
      "logps/chosen": -724.4750366210938,
      "logps/ref_chosen": -62.95263671875,
      "logps/ref_rejected": -92.4662094116211,
      "logps/rejected": -1189.3773193359375,
      "loss": 1.1,
      "margin_dpo/margin_mean": 435.3887634277344,
      "margin_dpo/margin_std": 627.5145263671875,
      "step": 670
    },
    {
      "epoch": 0.9911894273127754,
      "fcm_dpo/beta": 0.000940955535043031,
      "fcm_dpo/delta": -0.006233499385416508,
      "fcm_dpo/margin": 431.116455078125,
      "fcm_dpo/q_t": 0.4094417095184326,
      "grad_norm": 23.441814422607422,
      "learning_rate": 1.6138243485910863e-10,
      "logits/chosen": -1.0078109502792358,
      "logits/rejected": -1.02387535572052,
      "logps/chosen": -741.0542602539062,
      "logps/ref_chosen": -48.5856819152832,
      "logps/ref_rejected": -81.27871704101562,
      "logps/rejected": -1204.8636474609375,
      "loss": 1.1069,
      "margin_dpo/margin_mean": 431.116455078125,
      "margin_dpo/margin_std": 644.87255859375,
      "step": 675
    },
    {
      "epoch": 0.9985315712187959,
      "fcm_dpo/beta": 0.0009158365428447723,
      "fcm_dpo/delta": -0.01913156732916832,
      "fcm_dpo/margin": 456.44354248046875,
      "fcm_dpo/q_t": 0.40588778257369995,
      "grad_norm": 31.682743072509766,
      "learning_rate": 1.31753782067201e-11,
      "logits/chosen": -0.9820996522903442,
      "logits/rejected": -0.9966181516647339,
      "logps/chosen": -744.6751708984375,
      "logps/ref_chosen": -60.25421905517578,
      "logps/ref_rejected": -87.23457336425781,
      "logps/rejected": -1228.09912109375,
      "loss": 1.0916,
      "margin_dpo/margin_mean": 456.44354248046875,
      "margin_dpo/margin_std": 645.1212768554688,
      "step": 680
    },
    {
      "epoch": 1.0,
      "step": 681,
      "total_flos": 0.0,
      "train_loss": 1.1011297496229893,
      "train_runtime": 1861.3017,
      "train_samples_per_second": 23.423,
      "train_steps_per_second": 0.366
    }
  ],
  "logging_steps": 5,
  "max_steps": 681,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}