{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": -0.05443763732910156,
      "KL/mean": -0.04046645760536194,
      "KL/rejected_KL_mean": -0.026498794555664062,
      "KL/std": 0.47091516852378845,
      "epoch": 0.0020942408376963353,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.02793481945991516,
      "fcm_dpo/q_t": 0.500069797039032,
      "grad_norm": 28.590707778930664,
      "learning_rate": 0.0,
      "logits/chosen": -0.5898098945617676,
      "logits/rejected": -0.604260265827179,
      "logps/chosen": -275.28570556640625,
      "logps/ref_chosen": -275.2312927246094,
      "logps/ref_rejected": -222.9380340576172,
      "logps/rejected": -222.96453857421875,
      "loss": 5.5463,
      "margin_dpo/margin_mean": -0.02793477475643158,
      "margin_dpo/margin_std": 0.5724214911460876,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": 0.04458808898925781,
      "KL/mean": 0.0374295711517334,
      "KL/rejected_KL_mean": 0.030269622802734375,
      "KL/std": 0.49036121368408203,
      "epoch": 0.004188481675392671,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.014312177896499634,
      "fcm_dpo/q_t": 0.4999642074108124,
      "grad_norm": 27.878870010375977,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -0.6574729681015015,
      "logits/rejected": -0.6464410424232483,
      "logps/chosen": -264.7165222167969,
      "logps/ref_chosen": -264.7611083984375,
      "logps/ref_rejected": -242.5597686767578,
      "logps/rejected": -242.52951049804688,
      "loss": 5.5446,
      "margin_dpo/margin_mean": 0.014312252402305603,
      "margin_dpo/margin_std": 0.6423971652984619,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": 0.022775650024414062,
      "KL/mean": 0.007140308618545532,
      "KL/rejected_KL_mean": -0.008495330810546875,
      "KL/std": 0.5243284702301025,
      "epoch": 0.0062827225130890054,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.031269073486328125,
      "fcm_dpo/q_t": 0.49992185831069946,
      "grad_norm": 25.815317153930664,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": -0.6839790344238281,
      "logits/rejected": -0.7351304888725281,
      "logps/chosen": -274.0791320800781,
      "logps/ref_chosen": -274.1018981933594,
      "logps/ref_rejected": -286.5882568359375,
      "logps/rejected": -286.5967712402344,
      "loss": 5.544,
      "margin_dpo/margin_mean": 0.031267568469047546,
      "margin_dpo/margin_std": 0.804020881652832,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": 0.002529144287109375,
      "KL/mean": -0.004495471715927124,
      "KL/rejected_KL_mean": -0.011518478393554688,
      "KL/std": 0.5827552080154419,
      "epoch": 0.008376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.014056563377380371,
      "fcm_dpo/q_t": 0.4999648630619049,
      "grad_norm": 31.692033767700195,
      "learning_rate": 3.125e-08,
      "logits/chosen": -0.6173326373100281,
      "logits/rejected": -0.611765444278717,
      "logps/chosen": -329.83575439453125,
      "logps/ref_chosen": -329.8382568359375,
      "logps/ref_rejected": -303.2850646972656,
      "logps/rejected": -303.29656982421875,
      "loss": 5.5447,
      "margin_dpo/margin_mean": 0.014057323336601257,
      "margin_dpo/margin_std": 0.7402433156967163,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": -0.012723922729492188,
      "KL/mean": 0.03565850853919983,
      "KL/rejected_KL_mean": 0.08404731750488281,
      "KL/std": 0.5450270771980286,
      "epoch": 0.010471204188481676,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.09676727652549744,
      "fcm_dpo/q_t": 0.5002419352531433,
      "grad_norm": 29.52903938293457,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": -0.575000524520874,
      "logits/rejected": -0.5917554497718811,
      "logps/chosen": -301.7516784667969,
      "logps/ref_chosen": -301.7389221191406,
      "logps/ref_rejected": -274.7654724121094,
      "logps/rejected": -274.681396484375,
      "loss": 5.5491,
      "margin_dpo/margin_mean": -0.09676751494407654,
      "margin_dpo/margin_std": 0.7769181728363037,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": 0.10225868225097656,
      "KL/mean": 0.023305565118789673,
      "KL/rejected_KL_mean": -0.055644989013671875,
      "KL/std": 0.5169264078140259,
      "epoch": 0.012565445026178011,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15789711475372314,
      "fcm_dpo/q_t": 0.499605268239975,
      "grad_norm": 28.266115188598633,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -0.6809132099151611,
      "logits/rejected": -0.6438777446746826,
      "logps/chosen": -285.5924072265625,
      "logps/ref_chosen": -285.6946716308594,
      "logps/ref_rejected": -245.8200225830078,
      "logps/rejected": -245.87567138671875,
      "loss": 5.5389,
      "margin_dpo/margin_mean": 0.15789739787578583,
      "margin_dpo/margin_std": 0.675003707408905,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": -0.058353424072265625,
      "KL/mean": -0.034923337399959564,
      "KL/rejected_KL_mean": -0.011510848999023438,
      "KL/std": 0.49605488777160645,
      "epoch": 0.014659685863874346,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.046851709485054016,
      "fcm_dpo/q_t": 0.5001171231269836,
      "grad_norm": 28.5643253326416,
      "learning_rate": 6.25e-08,
      "logits/chosen": -0.5685729384422302,
      "logits/rejected": -0.600430965423584,
      "logps/chosen": -264.7137756347656,
      "logps/ref_chosen": -264.65545654296875,
      "logps/ref_rejected": -253.10305786132812,
      "logps/rejected": -253.11456298828125,
      "loss": 5.5471,
      "margin_dpo/margin_mean": -0.04685226082801819,
      "margin_dpo/margin_std": 0.6954786777496338,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": 0.0595550537109375,
      "KL/mean": 0.026862621307373047,
      "KL/rejected_KL_mean": -0.0058193206787109375,
      "KL/std": 0.5154087543487549,
      "epoch": 0.016753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06536287069320679,
      "fcm_dpo/q_t": 0.49983665347099304,
      "grad_norm": 30.72178840637207,
      "learning_rate": 7.291666666666667e-08,
      "logits/chosen": -0.6795619130134583,
      "logits/rejected": -0.6867237687110901,
      "logps/chosen": -354.1291809082031,
      "logps/ref_chosen": -354.1887512207031,
      "logps/ref_rejected": -282.9112243652344,
      "logps/rejected": -282.9170227050781,
      "loss": 5.5426,
      "margin_dpo/margin_mean": 0.06536369025707245,
      "margin_dpo/margin_std": 0.7256952524185181,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": 0.023357391357421875,
      "KL/mean": 0.004101134836673737,
      "KL/rejected_KL_mean": -0.015165328979492188,
      "KL/std": 0.48046308755874634,
      "epoch": 0.018848167539267015,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.03852246701717377,
      "fcm_dpo/q_t": 0.49990367889404297,
      "grad_norm": 27.868152618408203,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": -0.6396810412406921,
      "logits/rejected": -0.6580004692077637,
      "logps/chosen": -285.52685546875,
      "logps/ref_chosen": -285.5502014160156,
      "logps/ref_rejected": -267.99664306640625,
      "logps/rejected": -268.0118408203125,
      "loss": 5.5437,
      "margin_dpo/margin_mean": 0.038522377610206604,
      "margin_dpo/margin_std": 0.6642946600914001,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": 0.013101577758789062,
      "KL/mean": 0.0008476004004478455,
      "KL/rejected_KL_mean": -0.011411666870117188,
      "KL/std": 0.5422469973564148,
      "epoch": 0.020942408376963352,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.024518415331840515,
      "fcm_dpo/q_t": 0.4999386966228485,
      "grad_norm": 26.496253967285156,
      "learning_rate": 9.375e-08,
      "logits/chosen": -0.6832457780838013,
      "logits/rejected": -0.6776775121688843,
      "logps/chosen": -251.8992919921875,
      "logps/ref_chosen": -251.91238403320312,
      "logps/ref_rejected": -226.45260620117188,
      "logps/rejected": -226.46400451660156,
      "loss": 5.5442,
      "margin_dpo/margin_mean": 0.024518877267837524,
      "margin_dpo/margin_std": 0.676828145980835,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": 0.044345855712890625,
      "KL/mean": 0.0831230953335762,
      "KL/rejected_KL_mean": 0.1219024658203125,
      "KL/std": 0.5263944268226624,
      "epoch": 0.023036649214659685,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.07755695283412933,
      "fcm_dpo/q_t": 0.5001938939094543,
      "grad_norm": 28.984025955200195,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -0.5924692153930664,
      "logits/rejected": -0.6473174095153809,
      "logps/chosen": -301.03912353515625,
      "logps/ref_chosen": -301.08343505859375,
      "logps/ref_rejected": -259.546630859375,
      "logps/rejected": -259.42474365234375,
      "loss": 5.5483,
      "margin_dpo/margin_mean": -0.07755695283412933,
      "margin_dpo/margin_std": 0.7186371088027954,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": -0.011810302734375,
      "KL/mean": -0.01581135392189026,
      "KL/rejected_KL_mean": -0.019805908203125,
      "KL/std": 0.5478135347366333,
      "epoch": 0.025130890052356022,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.007990360260009766,
      "fcm_dpo/q_t": 0.49997997283935547,
      "grad_norm": 30.30181121826172,
      "learning_rate": 1.1458333333333332e-07,
      "logits/chosen": -0.5725276470184326,
      "logits/rejected": -0.5322108268737793,
      "logps/chosen": -287.5599060058594,
      "logps/ref_chosen": -287.548095703125,
      "logps/ref_rejected": -277.37945556640625,
      "logps/rejected": -277.3992614746094,
      "loss": 5.5449,
      "margin_dpo/margin_mean": 0.007990524172782898,
      "margin_dpo/margin_std": 0.7634983658790588,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": -0.01092529296875,
      "KL/mean": -0.049455322325229645,
      "KL/rejected_KL_mean": -0.08800125122070312,
      "KL/std": 0.5138577818870544,
      "epoch": 0.027225130890052355,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07708151638507843,
      "fcm_dpo/q_t": 0.49980732798576355,
      "grad_norm": 27.30363655090332,
      "learning_rate": 1.25e-07,
      "logits/chosen": -0.6579503417015076,
      "logits/rejected": -0.6656107902526855,
      "logps/chosen": -270.6773376464844,
      "logps/ref_chosen": -270.6664123535156,
      "logps/ref_rejected": -274.6546936035156,
      "logps/rejected": -274.74267578125,
      "loss": 5.5421,
      "margin_dpo/margin_mean": 0.07708097994327545,
      "margin_dpo/margin_std": 0.6744092702865601,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": 0.017566680908203125,
      "KL/mean": -0.023801714181900024,
      "KL/rejected_KL_mean": -0.0651702880859375,
      "KL/std": 0.4970622658729553,
      "epoch": 0.02931937172774869,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08273828029632568,
      "fcm_dpo/q_t": 0.4997932016849518,
      "grad_norm": 28.225872039794922,
      "learning_rate": 1.3541666666666666e-07,
      "logits/chosen": -0.6232421398162842,
      "logits/rejected": -0.6520561575889587,
      "logps/chosen": -281.5756530761719,
      "logps/ref_chosen": -281.59320068359375,
      "logps/ref_rejected": -263.52215576171875,
      "logps/rejected": -263.58734130859375,
      "loss": 5.5419,
      "margin_dpo/margin_mean": 0.08273807168006897,
      "margin_dpo/margin_std": 0.6712805032730103,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": 0.017675399780273438,
      "KL/mean": 0.029014699161052704,
      "KL/rejected_KL_mean": 0.04035186767578125,
      "KL/std": 0.5099201202392578,
      "epoch": 0.031413612565445025,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.022673889994621277,
      "fcm_dpo/q_t": 0.5000566840171814,
      "grad_norm": 30.290714263916016,
      "learning_rate": 1.4583333333333335e-07,
      "logits/chosen": -0.6323933601379395,
      "logits/rejected": -0.6433793902397156,
      "logps/chosen": -298.4357604980469,
      "logps/ref_chosen": -298.45343017578125,
      "logps/ref_rejected": -227.17832946777344,
      "logps/rejected": -227.13800048828125,
      "loss": 5.5461,
      "margin_dpo/margin_mean": -0.022674620151519775,
      "margin_dpo/margin_std": 0.7184179425239563,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": 0.10569190979003906,
      "KL/mean": 0.043116308748722076,
      "KL/rejected_KL_mean": -0.0194549560546875,
      "KL/std": 0.5487975478172302,
      "epoch": 0.033507853403141365,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1251499503850937,
      "fcm_dpo/q_t": 0.499687135219574,
      "grad_norm": 30.669429779052734,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -0.6036809682846069,
      "logits/rejected": -0.6009221076965332,
      "logps/chosen": -293.8609313964844,
      "logps/ref_chosen": -293.96661376953125,
      "logps/ref_rejected": -250.78443908691406,
      "logps/rejected": -250.80389404296875,
      "loss": 5.5402,
      "margin_dpo/margin_mean": 0.12514957785606384,
      "margin_dpo/margin_std": 0.7813780903816223,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.08317947387695312,
      "KL/mean": 0.04663021117448807,
      "KL/rejected_KL_mean": 0.010082244873046875,
      "KL/std": 0.5347276926040649,
      "epoch": 0.0356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07308481633663177,
      "fcm_dpo/q_t": 0.4998173117637634,
      "grad_norm": 27.673967361450195,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -0.5772124528884888,
      "logits/rejected": -0.6039091944694519,
      "logps/chosen": -262.3108215332031,
      "logps/ref_chosen": -262.39398193359375,
      "logps/ref_rejected": -248.500244140625,
      "logps/rejected": -248.49014282226562,
      "loss": 5.5423,
      "margin_dpo/margin_mean": 0.07308512926101685,
      "margin_dpo/margin_std": 0.6998055577278137,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": 0.022989273071289062,
      "KL/mean": -0.0186566561460495,
      "KL/rejected_KL_mean": -0.06031227111816406,
      "KL/std": 0.5273064970970154,
      "epoch": 0.03769633507853403,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.08329591155052185,
      "fcm_dpo/q_t": 0.4997917711734772,
      "grad_norm": 29.680051803588867,
      "learning_rate": 1.7708333333333334e-07,
      "logits/chosen": -0.6185211539268494,
      "logits/rejected": -0.6248131394386292,
      "logps/chosen": -293.6862487792969,
      "logps/ref_chosen": -293.709228515625,
      "logps/ref_rejected": -274.5875244140625,
      "logps/rejected": -274.6478576660156,
      "loss": 5.5419,
      "margin_dpo/margin_mean": 0.08329518139362335,
      "margin_dpo/margin_std": 0.7160223126411438,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": 0.0862274169921875,
      "KL/mean": 0.06030166149139404,
      "KL/rejected_KL_mean": 0.03438377380371094,
      "KL/std": 0.4625430703163147,
      "epoch": 0.039790575916230364,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.051838815212249756,
      "fcm_dpo/q_t": 0.4998703896999359,
      "grad_norm": 28.14927864074707,
      "learning_rate": 1.875e-07,
      "logits/chosen": -0.6315677165985107,
      "logits/rejected": -0.6282655000686646,
      "logps/chosen": -280.179443359375,
      "logps/ref_chosen": -280.26568603515625,
      "logps/ref_rejected": -259.9742736816406,
      "logps/rejected": -259.93988037109375,
      "loss": 5.5431,
      "margin_dpo/margin_mean": 0.05183883011341095,
      "margin_dpo/margin_std": 0.6121694445610046,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.1454639434814453,
      "KL/mean": 0.07818175107240677,
      "KL/rejected_KL_mean": 0.010904312133789062,
      "KL/std": 0.5232239961624146,
      "epoch": 0.041884816753926704,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1345617026090622,
      "fcm_dpo/q_t": 0.4996635913848877,
      "grad_norm": 29.66081428527832,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": -0.6154824495315552,
      "logits/rejected": -0.646511435508728,
      "logps/chosen": -303.74993896484375,
      "logps/ref_chosen": -303.8954162597656,
      "logps/ref_rejected": -260.214599609375,
      "logps/rejected": -260.2037048339844,
      "loss": 5.5399,
      "margin_dpo/margin_mean": 0.13456162810325623,
      "margin_dpo/margin_std": 0.776418924331665,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.16033554077148438,
      "KL/mean": 0.059254035353660583,
      "KL/rejected_KL_mean": -0.04183387756347656,
      "KL/std": 0.5349810123443604,
      "epoch": 0.04397905759162304,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2021702229976654,
      "fcm_dpo/q_t": 0.4994945824146271,
      "grad_norm": 35.1745719909668,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -0.642871618270874,
      "logits/rejected": -0.6705335974693298,
      "logps/chosen": -301.3731384277344,
      "logps/ref_chosen": -301.5334777832031,
      "logps/ref_rejected": -280.28900146484375,
      "logps/rejected": -280.330810546875,
      "loss": 5.5372,
      "margin_dpo/margin_mean": 0.2021702378988266,
      "margin_dpo/margin_std": 0.7195119261741638,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.11140251159667969,
      "KL/mean": 0.09679543972015381,
      "KL/rejected_KL_mean": 0.082183837890625,
      "KL/std": 0.5265232920646667,
      "epoch": 0.04607329842931937,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.029221147298812866,
      "fcm_dpo/q_t": 0.49992698431015015,
      "grad_norm": 25.29145622253418,
      "learning_rate": 2.1875e-07,
      "logits/chosen": -0.6614061594009399,
      "logits/rejected": -0.6633257269859314,
      "logps/chosen": -259.88372802734375,
      "logps/ref_chosen": -259.9951477050781,
      "logps/ref_rejected": -243.0721435546875,
      "logps/rejected": -242.98997497558594,
      "loss": 5.5441,
      "margin_dpo/margin_mean": 0.02922128140926361,
      "margin_dpo/margin_std": 0.7248610258102417,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.11682510375976562,
      "KL/mean": 0.033845022320747375,
      "KL/rejected_KL_mean": -0.04913902282714844,
      "KL/std": 0.5686609148979187,
      "epoch": 0.048167539267015703,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.16596052050590515,
      "fcm_dpo/q_t": 0.4995850622653961,
      "grad_norm": 27.988231658935547,
      "learning_rate": 2.2916666666666663e-07,
      "logits/chosen": -0.6159858703613281,
      "logits/rejected": -0.6495825052261353,
      "logps/chosen": -282.0639343261719,
      "logps/ref_chosen": -282.1807556152344,
      "logps/ref_rejected": -265.0758056640625,
      "logps/rejected": -265.12493896484375,
      "loss": 5.5386,
      "margin_dpo/margin_mean": 0.16596047580242157,
      "margin_dpo/margin_std": 0.8245087265968323,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": 0.206268310546875,
      "KL/mean": 0.05522076040506363,
      "KL/rejected_KL_mean": -0.09583663940429688,
      "KL/std": 0.6240689754486084,
      "epoch": 0.050261780104712044,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.30209439992904663,
      "fcm_dpo/q_t": 0.4992447793483734,
      "grad_norm": 29.82961654663086,
      "learning_rate": 2.3958333333333335e-07,
      "logits/chosen": -0.6571820378303528,
      "logits/rejected": -0.5694364309310913,
      "logps/chosen": -300.9733581542969,
      "logps/ref_chosen": -301.17962646484375,
      "logps/ref_rejected": -302.12786865234375,
      "logps/rejected": -302.22369384765625,
      "loss": 5.5332,
      "margin_dpo/margin_mean": 0.3020938038825989,
      "margin_dpo/margin_std": 0.8368014097213745,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.1793060302734375,
      "KL/mean": 0.13233965635299683,
      "KL/rejected_KL_mean": 0.08537006378173828,
      "KL/std": 0.5242065191268921,
      "epoch": 0.05235602094240838,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09393073618412018,
      "fcm_dpo/q_t": 0.49976521730422974,
      "grad_norm": 26.477157592773438,
      "learning_rate": 2.5e-07,
      "logits/chosen": -0.596278965473175,
      "logits/rejected": -0.6043489575386047,
      "logps/chosen": -246.56716918945312,
      "logps/ref_chosen": -246.74649047851562,
      "logps/ref_rejected": -235.55638122558594,
      "logps/rejected": -235.4709930419922,
      "loss": 5.5415,
      "margin_dpo/margin_mean": 0.09393027424812317,
      "margin_dpo/margin_std": 0.6793190240859985,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": 0.28119850158691406,
      "KL/mean": 0.2227058708667755,
      "KL/rejected_KL_mean": 0.1642169952392578,
      "KL/std": 0.5824305415153503,
      "epoch": 0.05445026178010471,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.11699268221855164,
      "fcm_dpo/q_t": 0.49970752000808716,
      "grad_norm": 28.63658905029297,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -0.6545581817626953,
      "logits/rejected": -0.6691527962684631,
      "logps/chosen": -281.9143981933594,
      "logps/ref_chosen": -282.1955871582031,
      "logps/ref_rejected": -235.3135528564453,
      "logps/rejected": -235.1493377685547,
      "loss": 5.5406,
      "margin_dpo/margin_mean": 0.11699323356151581,
      "margin_dpo/margin_std": 0.747821033000946,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.26987648010253906,
      "KL/mean": 0.17068162560462952,
      "KL/rejected_KL_mean": 0.07148551940917969,
      "KL/std": 0.6263948082923889,
      "epoch": 0.05654450261780105,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.19838984310626984,
      "fcm_dpo/q_t": 0.499504029750824,
      "grad_norm": 27.719825744628906,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": -0.6554375290870667,
      "logits/rejected": -0.6753584146499634,
      "logps/chosen": -323.5864562988281,
      "logps/ref_chosen": -323.8563537597656,
      "logps/ref_rejected": -245.968017578125,
      "logps/rejected": -245.8965301513672,
      "loss": 5.5373,
      "margin_dpo/margin_mean": 0.1983897089958191,
      "margin_dpo/margin_std": 0.8603197932243347,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.2645702362060547,
      "KL/mean": 0.1998385488986969,
      "KL/rejected_KL_mean": 0.1351032257080078,
      "KL/std": 0.6069234609603882,
      "epoch": 0.05863874345549738,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1294664591550827,
      "fcm_dpo/q_t": 0.49967634677886963,
      "grad_norm": 26.354848861694336,
      "learning_rate": 2.8125e-07,
      "logits/chosen": -0.6198784112930298,
      "logits/rejected": -0.6290311217308044,
      "logps/chosen": -247.982177734375,
      "logps/ref_chosen": -248.24673461914062,
      "logps/ref_rejected": -240.0382080078125,
      "logps/rejected": -239.9031219482422,
      "loss": 5.5401,
      "margin_dpo/margin_mean": 0.12946686148643494,
      "margin_dpo/margin_std": 0.8481366038322449,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": 0.32039451599121094,
      "KL/mean": 0.20665234327316284,
      "KL/rejected_KL_mean": 0.09290885925292969,
      "KL/std": 0.6451480388641357,
      "epoch": 0.060732984293193716,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.22749897837638855,
      "fcm_dpo/q_t": 0.4994312524795532,
      "grad_norm": 29.818029403686523,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": -0.5938171148300171,
      "logits/rejected": -0.6162937879562378,
      "logps/chosen": -317.93603515625,
      "logps/ref_chosen": -318.2564392089844,
      "logps/ref_rejected": -286.75848388671875,
      "logps/rejected": -286.6656188964844,
      "loss": 5.5362,
      "margin_dpo/margin_mean": 0.22749751806259155,
      "margin_dpo/margin_std": 0.8251089453697205,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.4100780487060547,
      "KL/mean": 0.2640945613384247,
      "KL/rejected_KL_mean": 0.11810874938964844,
      "KL/std": 0.6603554487228394,
      "epoch": 0.06282722513089005,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.29195964336395264,
      "fcm_dpo/q_t": 0.49927011132240295,
      "grad_norm": 28.872472763061523,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": -0.5968848466873169,
      "logits/rejected": -0.6139403581619263,
      "logps/chosen": -252.6390380859375,
      "logps/ref_chosen": -253.0491485595703,
      "logps/ref_rejected": -261.30029296875,
      "logps/rejected": -261.1821594238281,
      "loss": 5.5336,
      "margin_dpo/margin_mean": 0.291959673166275,
      "margin_dpo/margin_std": 0.9232319593429565,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": 0.4171581268310547,
      "KL/mean": 0.28819912672042847,
      "KL/rejected_KL_mean": 0.15924644470214844,
      "KL/std": 0.6864354610443115,
      "epoch": 0.06492146596858639,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2579154670238495,
      "fcm_dpo/q_t": 0.4993552565574646,
      "grad_norm": 25.03595733642578,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -0.654579758644104,
      "logits/rejected": -0.6890774965286255,
      "logps/chosen": -247.73585510253906,
      "logps/ref_chosen": -248.15301513671875,
      "logps/ref_rejected": -203.17703247070312,
      "logps/rejected": -203.01779174804688,
      "loss": 5.535,
      "margin_dpo/margin_mean": 0.25791555643081665,
      "margin_dpo/margin_std": 0.9564522504806519,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.48644065856933594,
      "KL/mean": 0.3326733708381653,
      "KL/rejected_KL_mean": 0.17890357971191406,
      "KL/std": 0.7589449286460876,
      "epoch": 0.06701570680628273,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.30753594636917114,
      "fcm_dpo/q_t": 0.4992311894893646,
      "grad_norm": 29.670412063598633,
      "learning_rate": 3.2291666666666666e-07,
      "logits/chosen": -0.6130271553993225,
      "logits/rejected": -0.6176277995109558,
      "logps/chosen": -305.0534973144531,
      "logps/ref_chosen": -305.5399475097656,
      "logps/ref_rejected": -267.6527099609375,
      "logps/rejected": -267.4737854003906,
      "loss": 5.533,
      "margin_dpo/margin_mean": 0.3075369596481323,
      "margin_dpo/margin_std": 1.0005149841308594,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.6479530334472656,
      "KL/mean": 0.44665971398353577,
      "KL/rejected_KL_mean": 0.24537277221679688,
      "KL/std": 0.8759632110595703,
      "epoch": 0.06910994764397906,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.40256696939468384,
      "fcm_dpo/q_t": 0.4989936053752899,
      "grad_norm": 28.556854248046875,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -0.6414982080459595,
      "logits/rejected": -0.6540948748588562,
      "logps/chosen": -285.5855712890625,
      "logps/ref_chosen": -286.2335205078125,
      "logps/ref_rejected": -255.38748168945312,
      "logps/rejected": -255.14210510253906,
      "loss": 5.5292,
      "margin_dpo/margin_mean": 0.4025673568248749,
      "margin_dpo/margin_std": 1.043798804283142,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.7430992126464844,
      "KL/mean": 0.38929152488708496,
      "KL/rejected_KL_mean": 0.035488128662109375,
      "KL/std": 0.9840909242630005,
      "epoch": 0.0712041884816754,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7076153755187988,
      "fcm_dpo/q_t": 0.49823105335235596,
      "grad_norm": 31.13680648803711,
      "learning_rate": 3.4375e-07,
      "logits/chosen": -0.6229280233383179,
      "logits/rejected": -0.6322309374809265,
      "logps/chosen": -340.8489074707031,
      "logps/ref_chosen": -341.5920104980469,
      "logps/ref_rejected": -278.8866882324219,
      "logps/rejected": -278.8511962890625,
      "loss": 5.5171,
      "margin_dpo/margin_mean": 0.7076154947280884,
      "margin_dpo/margin_std": 1.1253160238265991,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": 0.6277313232421875,
      "KL/mean": 0.44873669743537903,
      "KL/rejected_KL_mean": 0.2697429656982422,
      "KL/std": 0.9546669125556946,
      "epoch": 0.07329842931937172,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.35799384117126465,
      "fcm_dpo/q_t": 0.4991050064563751,
      "grad_norm": 26.445619583129883,
      "learning_rate": 3.541666666666667e-07,
      "logits/chosen": -0.6277632117271423,
      "logits/rejected": -0.6477382183074951,
      "logps/chosen": -264.4518127441406,
      "logps/ref_chosen": -265.0795593261719,
      "logps/ref_rejected": -264.4876708984375,
      "logps/rejected": -264.2179260253906,
      "loss": 5.531,
      "margin_dpo/margin_mean": 0.35799336433410645,
      "margin_dpo/margin_std": 1.2602258920669556,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.8565578460693359,
      "KL/mean": 0.5326486229896545,
      "KL/rejected_KL_mean": 0.208740234375,
      "KL/std": 1.0565747022628784,
      "epoch": 0.07539267015706806,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6478173732757568,
      "fcm_dpo/q_t": 0.498380571603775,
      "grad_norm": 31.941911697387695,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -0.6065114736557007,
      "logits/rejected": -0.624266505241394,
      "logps/chosen": -296.4695739746094,
      "logps/ref_chosen": -297.3261413574219,
      "logps/ref_rejected": -282.09515380859375,
      "logps/rejected": -281.8864440917969,
      "loss": 5.5195,
      "margin_dpo/margin_mean": 0.6478171348571777,
      "margin_dpo/margin_std": 1.3030225038528442,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": 0.7484455108642578,
      "KL/mean": 0.4896807074546814,
      "KL/rejected_KL_mean": 0.2309284210205078,
      "KL/std": 1.159031867980957,
      "epoch": 0.0774869109947644,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.517516016960144,
      "fcm_dpo/q_t": 0.4987063407897949,
      "grad_norm": 30.880260467529297,
      "learning_rate": 3.75e-07,
      "logits/chosen": -0.5905128121376038,
      "logits/rejected": -0.6039587259292603,
      "logps/chosen": -313.28558349609375,
      "logps/ref_chosen": -314.0340270996094,
      "logps/ref_rejected": -299.3437805175781,
      "logps/rejected": -299.11285400390625,
      "loss": 5.5248,
      "margin_dpo/margin_mean": 0.5175141096115112,
      "margin_dpo/margin_std": 1.6650607585906982,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 1.009002685546875,
      "KL/mean": 0.7270980477333069,
      "KL/rejected_KL_mean": 0.44519996643066406,
      "KL/std": 1.2026073932647705,
      "epoch": 0.07958115183246073,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5638107061386108,
      "fcm_dpo/q_t": 0.49859052896499634,
      "grad_norm": 28.458763122558594,
      "learning_rate": 3.8541666666666665e-07,
      "logits/chosen": -0.6414623260498047,
      "logits/rejected": -0.6538353562355042,
      "logps/chosen": -281.5321960449219,
      "logps/ref_chosen": -282.54119873046875,
      "logps/ref_rejected": -269.7773132324219,
      "logps/rejected": -269.33209228515625,
      "loss": 5.5229,
      "margin_dpo/margin_mean": 0.5638105273246765,
      "margin_dpo/margin_std": 1.4827549457550049,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": 1.387380599975586,
      "KL/mean": 0.7378964424133301,
      "KL/rejected_KL_mean": 0.08840751647949219,
      "KL/std": 1.5625250339508057,
      "epoch": 0.08167539267015707,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2989695072174072,
      "fcm_dpo/q_t": 0.496753066778183,
      "grad_norm": 29.53527069091797,
      "learning_rate": 3.958333333333333e-07,
      "logits/chosen": -0.615884006023407,
      "logits/rejected": -0.6298059225082397,
      "logps/chosen": -275.38555908203125,
      "logps/ref_chosen": -276.7729187011719,
      "logps/ref_rejected": -249.95889282226562,
      "logps/rejected": -249.87049865722656,
      "loss": 5.4938,
      "margin_dpo/margin_mean": 1.2989693880081177,
      "margin_dpo/margin_std": 1.8492337465286255,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": 1.0653915405273438,
      "KL/mean": 0.6882903575897217,
      "KL/rejected_KL_mean": 0.3111915588378906,
      "KL/std": 1.3857722282409668,
      "epoch": 0.08376963350785341,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7541953921318054,
      "fcm_dpo/q_t": 0.4981148838996887,
      "grad_norm": 27.370119094848633,
      "learning_rate": 4.0625e-07,
      "logits/chosen": -0.6240854263305664,
      "logits/rejected": -0.6599107980728149,
      "logps/chosen": -283.24169921875,
      "logps/ref_chosen": -284.30706787109375,
      "logps/ref_rejected": -244.4459991455078,
      "logps/rejected": -244.13479614257812,
      "loss": 5.5155,
      "margin_dpo/margin_mean": 0.7541957497596741,
      "margin_dpo/margin_std": 1.8628623485565186,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": 1.1275959014892578,
      "KL/mean": 0.7282809615135193,
      "KL/rejected_KL_mean": 0.3289756774902344,
      "KL/std": 1.5008491277694702,
      "epoch": 0.08586387434554973,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7986243367195129,
      "fcm_dpo/q_t": 0.49800363183021545,
      "grad_norm": 30.310182571411133,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -0.6330433487892151,
      "logits/rejected": -0.6593391299247742,
      "logps/chosen": -292.68756103515625,
      "logps/ref_chosen": -293.8151550292969,
      "logps/ref_rejected": -252.16815185546875,
      "logps/rejected": -251.83917236328125,
      "loss": 5.5136,
      "margin_dpo/margin_mean": 0.798625111579895,
      "margin_dpo/margin_std": 1.8047460317611694,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 1.3023014068603516,
      "KL/mean": 0.7951821088790894,
      "KL/rejected_KL_mean": 0.28806304931640625,
      "KL/std": 1.7551039457321167,
      "epoch": 0.08795811518324607,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0142276287078857,
      "fcm_dpo/q_t": 0.4974648356437683,
      "grad_norm": 27.438058853149414,
      "learning_rate": 4.270833333333333e-07,
      "logits/chosen": -0.63153475522995,
      "logits/rejected": -0.6492542624473572,
      "logps/chosen": -251.45797729492188,
      "logps/ref_chosen": -252.76023864746094,
      "logps/ref_rejected": -261.0414733886719,
      "logps/rejected": -260.75341796875,
      "loss": 5.5053,
      "margin_dpo/margin_mean": 1.014228105545044,
      "margin_dpo/margin_std": 2.2831621170043945,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 1.2909984588623047,
      "KL/mean": 0.7054203748703003,
      "KL/rejected_KL_mean": 0.11982917785644531,
      "KL/std": 1.8495666980743408,
      "epoch": 0.09005235602094241,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1711581945419312,
      "fcm_dpo/q_t": 0.4970724582672119,
      "grad_norm": 29.84286880493164,
      "learning_rate": 4.375e-07,
      "logits/chosen": -0.5870351791381836,
      "logits/rejected": -0.6016489863395691,
      "logps/chosen": -315.5437316894531,
      "logps/ref_chosen": -316.8347473144531,
      "logps/ref_rejected": -273.7649230957031,
      "logps/rejected": -273.64508056640625,
      "loss": 5.499,
      "margin_dpo/margin_mean": 1.1711583137512207,
      "margin_dpo/margin_std": 2.2551350593566895,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": 1.5232257843017578,
      "KL/mean": 0.7169191837310791,
      "KL/rejected_KL_mean": -0.08938789367675781,
      "KL/std": 2.1754655838012695,
      "epoch": 0.09214659685863874,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6126174926757812,
      "fcm_dpo/q_t": 0.4959692060947418,
      "grad_norm": 30.84475326538086,
      "learning_rate": 4.479166666666667e-07,
      "logits/chosen": -0.6050872206687927,
      "logits/rejected": -0.6044933795928955,
      "logps/chosen": -285.3524475097656,
      "logps/ref_chosen": -286.8757019042969,
      "logps/ref_rejected": -282.4681396484375,
      "logps/rejected": -282.55755615234375,
      "loss": 5.4819,
      "margin_dpo/margin_mean": 1.6126176118850708,
      "margin_dpo/margin_std": 2.980741024017334,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": 1.7031822204589844,
      "KL/mean": 1.0522090196609497,
      "KL/rejected_KL_mean": 0.40123748779296875,
      "KL/std": 2.238448143005371,
      "epoch": 0.09424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.3019399642944336,
      "fcm_dpo/q_t": 0.4967461824417114,
      "grad_norm": 28.910964965820312,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": -0.6813381314277649,
      "logits/rejected": -0.7038441896438599,
      "logps/chosen": -322.5601806640625,
      "logps/ref_chosen": -324.2633972167969,
      "logps/ref_rejected": -293.09466552734375,
      "logps/rejected": -292.69342041015625,
      "loss": 5.4941,
      "margin_dpo/margin_mean": 1.3019402027130127,
      "margin_dpo/margin_std": 2.779609441757202,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": 1.7381591796875,
      "KL/mean": 0.9746726751327515,
      "KL/rejected_KL_mean": 0.21118736267089844,
      "KL/std": 2.4180023670196533,
      "epoch": 0.09633507853403141,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.526973009109497,
      "fcm_dpo/q_t": 0.4961833655834198,
      "grad_norm": 30.442108154296875,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -0.6318519711494446,
      "logits/rejected": -0.6458595991134644,
      "logps/chosen": -296.5975646972656,
      "logps/ref_chosen": -298.3357238769531,
      "logps/ref_rejected": -267.66204833984375,
      "logps/rejected": -267.4508361816406,
      "loss": 5.485,
      "margin_dpo/margin_mean": 1.5269722938537598,
      "margin_dpo/margin_std": 2.5452775955200195,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": 1.5774993896484375,
      "KL/mean": 1.0336077213287354,
      "KL/rejected_KL_mean": 0.48970985412597656,
      "KL/std": 2.3391621112823486,
      "epoch": 0.09842931937172775,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0877865552902222,
      "fcm_dpo/q_t": 0.49728119373321533,
      "grad_norm": 26.332962036132812,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": -0.5959615111351013,
      "logits/rejected": -0.61611008644104,
      "logps/chosen": -260.98944091796875,
      "logps/ref_chosen": -262.5669250488281,
      "logps/ref_rejected": -258.70989990234375,
      "logps/rejected": -258.2201843261719,
      "loss": 5.5031,
      "margin_dpo/margin_mean": 1.0877865552902222,
      "margin_dpo/margin_std": 3.3929736614227295,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": 1.772359848022461,
      "KL/mean": 1.0279061794281006,
      "KL/rejected_KL_mean": 0.2834606170654297,
      "KL/std": 2.5309736728668213,
      "epoch": 0.10052356020942409,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.488900899887085,
      "fcm_dpo/q_t": 0.496279776096344,
      "grad_norm": 27.56769371032715,
      "learning_rate": 4.895833333333333e-07,
      "logits/chosen": -0.6129103899002075,
      "logits/rejected": -0.6380035877227783,
      "logps/chosen": -267.72088623046875,
      "logps/ref_chosen": -269.4932556152344,
      "logps/ref_rejected": -241.888916015625,
      "logps/rejected": -241.60546875,
      "loss": 5.4868,
      "margin_dpo/margin_mean": 1.4889018535614014,
      "margin_dpo/margin_std": 2.91670823097229,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 2.256755828857422,
      "KL/mean": 1.3878839015960693,
      "KL/rejected_KL_mean": 0.5190181732177734,
      "KL/std": 2.6580569744110107,
      "epoch": 0.10261780104712041,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7377341985702515,
      "fcm_dpo/q_t": 0.4956571161746979,
      "grad_norm": 27.611753463745117,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6712480187416077,
      "logits/rejected": -0.6555536985397339,
      "logps/chosen": -255.62774658203125,
      "logps/ref_chosen": -257.8844909667969,
      "logps/ref_rejected": -256.8912048339844,
      "logps/rejected": -256.3721923828125,
      "loss": 5.4772,
      "margin_dpo/margin_mean": 1.7377345561981201,
      "margin_dpo/margin_std": 3.369967222213745,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": 1.9994640350341797,
      "KL/mean": 1.4609719514846802,
      "KL/rejected_KL_mean": 0.9224815368652344,
      "KL/std": 3.039249897003174,
      "epoch": 0.10471204188481675,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0769891738891602,
      "fcm_dpo/q_t": 0.4973089396953583,
      "grad_norm": 27.952144622802734,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": -0.6236312985420227,
      "logits/rejected": -0.6451153755187988,
      "logps/chosen": -299.62939453125,
      "logps/ref_chosen": -301.62884521484375,
      "logps/ref_rejected": -298.2716064453125,
      "logps/rejected": -297.34912109375,
      "loss": 5.5039,
      "margin_dpo/margin_mean": 1.0769885778427124,
      "margin_dpo/margin_std": 3.7813327312469482,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": 2.007976531982422,
      "KL/mean": 1.0874520540237427,
      "KL/rejected_KL_mean": 0.16692543029785156,
      "KL/std": 3.5725419521331787,
      "epoch": 0.1068062827225131,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.8410534858703613,
      "fcm_dpo/q_t": 0.4954001307487488,
      "grad_norm": 29.133390426635742,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -0.615746796131134,
      "logits/rejected": -0.6096806526184082,
      "logps/chosen": -267.36444091796875,
      "logps/ref_chosen": -269.37237548828125,
      "logps/ref_rejected": -297.0167541503906,
      "logps/rejected": -296.84979248046875,
      "loss": 5.4743,
      "margin_dpo/margin_mean": 1.8410539627075195,
      "margin_dpo/margin_std": 4.548240661621094,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": 2.8379573822021484,
      "KL/mean": 1.2705695629119873,
      "KL/rejected_KL_mean": -0.2968254089355469,
      "KL/std": 3.7937066555023193,
      "epoch": 0.10890052356020942,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.134784698486328,
      "fcm_dpo/q_t": 0.4921700358390808,
      "grad_norm": 30.25187873840332,
      "learning_rate": 4.99939671821067e-07,
      "logits/chosen": -0.6572005748748779,
      "logits/rejected": -0.663905680179596,
      "logps/chosen": -304.0648193359375,
      "logps/ref_chosen": -306.9028015136719,
      "logps/ref_rejected": -281.24737548828125,
      "logps/rejected": -281.544189453125,
      "loss": 5.4233,
      "margin_dpo/margin_mean": 3.134784460067749,
      "margin_dpo/margin_std": 4.76301383972168,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 2.8564796447753906,
      "KL/mean": 1.7158511877059937,
      "KL/rejected_KL_mean": 0.5752315521240234,
      "KL/std": 4.075250148773193,
      "epoch": 0.11099476439790576,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.2812445163726807,
      "fcm_dpo/q_t": 0.49430161714553833,
      "grad_norm": 29.959592819213867,
      "learning_rate": 4.998927532591591e-07,
      "logits/chosen": -0.6579018831253052,
      "logits/rejected": -0.6985629200935364,
      "logps/chosen": -283.1194763183594,
      "logps/ref_chosen": -285.9759521484375,
      "logps/ref_rejected": -273.9073486328125,
      "logps/rejected": -273.33209228515625,
      "loss": 5.4575,
      "margin_dpo/margin_mean": 2.2812438011169434,
      "margin_dpo/margin_std": 5.07851505279541,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": 2.6852340698242188,
      "KL/mean": 1.7344377040863037,
      "KL/rejected_KL_mean": 0.7836399078369141,
      "KL/std": 4.301624774932861,
      "epoch": 0.1130890052356021,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.901583194732666,
      "fcm_dpo/q_t": 0.49525216221809387,
      "grad_norm": 26.382888793945312,
      "learning_rate": 4.998324337072792e-07,
      "logits/chosen": -0.69151771068573,
      "logits/rejected": -0.6999548077583313,
      "logps/chosen": -303.8193664550781,
      "logps/ref_chosen": -306.504638671875,
      "logps/ref_rejected": -272.67431640625,
      "logps/rejected": -271.89068603515625,
      "loss": 5.4729,
      "margin_dpo/margin_mean": 1.9015839099884033,
      "margin_dpo/margin_std": 5.501827716827393,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 1.8460063934326172,
      "KL/mean": 0.6938503384590149,
      "KL/rejected_KL_mean": -0.45831298828125,
      "KL/std": 3.678321123123169,
      "epoch": 0.11518324607329843,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.304323196411133,
      "fcm_dpo/q_t": 0.49424460530281067,
      "grad_norm": 24.83275604248047,
      "learning_rate": 4.997587164001815e-07,
      "logits/chosen": -0.6331048011779785,
      "logits/rejected": -0.6366676092147827,
      "logps/chosen": -220.48414611816406,
      "logps/ref_chosen": -222.33013916015625,
      "logps/ref_rejected": -206.59571838378906,
      "logps/rejected": -207.05401611328125,
      "loss": 5.456,
      "margin_dpo/margin_mean": 2.304323673248291,
      "margin_dpo/margin_std": 4.8657379150390625,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": 2.875276565551758,
      "KL/mean": 1.307525873184204,
      "KL/rejected_KL_mean": -0.260223388671875,
      "KL/std": 4.787164688110352,
      "epoch": 0.11727748691099477,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.13550066947937,
      "fcm_dpo/q_t": 0.49217385053634644,
      "grad_norm": 27.477989196777344,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -0.6051826477050781,
      "logits/rejected": -0.6193740963935852,
      "logps/chosen": -247.60287475585938,
      "logps/ref_chosen": -250.47816467285156,
      "logps/ref_rejected": -228.25848388671875,
      "logps/rejected": -228.51869201660156,
      "loss": 5.425,
      "margin_dpo/margin_mean": 3.135500907897949,
      "margin_dpo/margin_std": 5.9617414474487305,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": 3.8332347869873047,
      "KL/mean": 1.7499638795852661,
      "KL/rejected_KL_mean": -0.33330535888671875,
      "KL/std": 4.810323238372803,
      "epoch": 0.1193717277486911,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.166540145874023,
      "fcm_dpo/q_t": 0.4895956218242645,
      "grad_norm": 30.821842193603516,
      "learning_rate": 4.99571105051544e-07,
      "logits/chosen": -0.7008582353591919,
      "logits/rejected": -0.6718518137931824,
      "logps/chosen": -311.2862854003906,
      "logps/ref_chosen": -315.1195373535156,
      "logps/ref_rejected": -272.755615234375,
      "logps/rejected": -273.08892822265625,
      "loss": 5.3837,
      "margin_dpo/margin_mean": 4.166539669036865,
      "margin_dpo/margin_std": 5.651122570037842,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 2.5054054260253906,
      "KL/mean": 1.3090157508850098,
      "KL/rejected_KL_mean": 0.11263465881347656,
      "KL/std": 4.480978012084961,
      "epoch": 0.12146596858638743,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 2.3927712440490723,
      "fcm_dpo/q_t": 0.4940217137336731,
      "grad_norm": 28.39775848388672,
      "learning_rate": 4.994572210710314e-07,
      "logits/chosen": -0.6188403367996216,
      "logits/rejected": -0.6432065963745117,
      "logps/chosen": -262.6762390136719,
      "logps/ref_chosen": -265.1816711425781,
      "logps/ref_rejected": -268.2203369140625,
      "logps/rejected": -268.1076965332031,
      "loss": 5.4536,
      "margin_dpo/margin_mean": 2.3927717208862305,
      "margin_dpo/margin_std": 5.734703540802002,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 2.1373958587646484,
      "KL/mean": 1.3352781534194946,
      "KL/rejected_KL_mean": 0.5331630706787109,
      "KL/std": 5.218527317047119,
      "epoch": 0.12356020942408377,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.6042373180389404,
      "fcm_dpo/q_t": 0.4959961175918579,
      "grad_norm": 29.90195655822754,
      "learning_rate": 4.993299594568162e-07,
      "logits/chosen": -0.607314944267273,
      "logits/rejected": -0.6005712151527405,
      "logps/chosen": -284.2165832519531,
      "logps/ref_chosen": -286.35394287109375,
      "logps/ref_rejected": -260.6757507324219,
      "logps/rejected": -260.142578125,
      "loss": 5.4869,
      "margin_dpo/margin_mean": 1.6042369604110718,
      "margin_dpo/margin_std": 6.974043846130371,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": 2.8796634674072266,
      "KL/mean": 1.1987619400024414,
      "KL/rejected_KL_mean": -0.48214149475097656,
      "KL/std": 5.123387813568115,
      "epoch": 0.1256544502617801,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.361804485321045,
      "fcm_dpo/q_t": 0.4916055202484131,
      "grad_norm": 27.98661231994629,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": -0.6914751529693604,
      "logits/rejected": -0.7169291973114014,
      "logps/chosen": -255.86895751953125,
      "logps/ref_chosen": -258.74859619140625,
      "logps/ref_rejected": -255.04893493652344,
      "logps/rejected": -255.53106689453125,
      "loss": 5.4171,
      "margin_dpo/margin_mean": 3.361804246902466,
      "margin_dpo/margin_std": 6.976766109466553,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": 3.104330062866211,
      "KL/mean": 1.4834375381469727,
      "KL/rejected_KL_mean": -0.13745689392089844,
      "KL/std": 5.609597206115723,
      "epoch": 0.12774869109947645,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.241795539855957,
      "fcm_dpo/q_t": 0.49191004037857056,
      "grad_norm": 29.877731323242188,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -0.6498665809631348,
      "logits/rejected": -0.6669278740882874,
      "logps/chosen": -275.36358642578125,
      "logps/ref_chosen": -278.4678955078125,
      "logps/ref_rejected": -252.02720642089844,
      "logps/rejected": -252.16464233398438,
      "loss": 5.4233,
      "margin_dpo/margin_mean": 3.241795063018799,
      "margin_dpo/margin_std": 7.654033660888672,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": 4.051607131958008,
      "KL/mean": 2.3158695697784424,
      "KL/rejected_KL_mean": 0.5801334381103516,
      "KL/std": 5.725602626800537,
      "epoch": 0.12984293193717278,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.471477508544922,
      "fcm_dpo/q_t": 0.4913414418697357,
      "grad_norm": 26.62920570373535,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": -0.6097573041915894,
      "logits/rejected": -0.6515059471130371,
      "logps/chosen": -268.8726806640625,
      "logps/ref_chosen": -272.92431640625,
      "logps/ref_rejected": -260.7935485839844,
      "logps/rejected": -260.2134094238281,
      "loss": 5.4134,
      "margin_dpo/margin_mean": 3.471477746963501,
      "margin_dpo/margin_std": 7.446447372436523,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": 3.4179611206054688,
      "KL/mean": 1.6585086584091187,
      "KL/rejected_KL_mean": -0.1009521484375,
      "KL/std": 5.611577987670898,
      "epoch": 0.1319371727748691,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.518895387649536,
      "fcm_dpo/q_t": 0.4912171959877014,
      "grad_norm": 28.223430633544922,
      "learning_rate": 4.986872839090852e-07,
      "logits/chosen": -0.6668751239776611,
      "logits/rejected": -0.6736673712730408,
      "logps/chosen": -273.6710510253906,
      "logps/ref_chosen": -277.0889892578125,
      "logps/ref_rejected": -273.3413391113281,
      "logps/rejected": -273.4422607421875,
      "loss": 5.4113,
      "margin_dpo/margin_mean": 3.5188941955566406,
      "margin_dpo/margin_std": 7.246774673461914,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": 4.1474609375,
      "KL/mean": 2.040192127227783,
      "KL/rejected_KL_mean": -0.06707000732421875,
      "KL/std": 6.2975850105285645,
      "epoch": 0.13403141361256546,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.214529514312744,
      "fcm_dpo/q_t": 0.48948606848716736,
      "grad_norm": 28.362247467041016,
      "learning_rate": 4.9849325083059e-07,
      "logits/chosen": -0.6360031962394714,
      "logits/rejected": -0.6315053701400757,
      "logps/chosen": -279.6770324707031,
      "logps/ref_chosen": -283.8244934082031,
      "logps/ref_rejected": -263.29351806640625,
      "logps/rejected": -263.360595703125,
      "loss": 5.3859,
      "margin_dpo/margin_mean": 4.214529037475586,
      "margin_dpo/margin_std": 8.358125686645508,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": 3.4035091400146484,
      "KL/mean": 1.7503830194473267,
      "KL/rejected_KL_mean": 0.09726905822753906,
      "KL/std": 5.768850326538086,
      "epoch": 0.13612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.306236982345581,
      "fcm_dpo/q_t": 0.4917478561401367,
      "grad_norm": 27.84429931640625,
      "learning_rate": 4.982858918131906e-07,
      "logits/chosen": -0.7009012699127197,
      "logits/rejected": -0.6689469218254089,
      "logps/chosen": -261.4664306640625,
      "logps/ref_chosen": -264.8699645996094,
      "logps/ref_rejected": -268.5076904296875,
      "logps/rejected": -268.4104309082031,
      "loss": 5.4199,
      "margin_dpo/margin_mean": 3.3062353134155273,
      "margin_dpo/margin_std": 7.423593997955322,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": 3.1753273010253906,
      "KL/mean": 1.074022650718689,
      "KL/rejected_KL_mean": -1.0272808074951172,
      "KL/std": 7.453335762023926,
      "epoch": 0.1382198952879581,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.20259952545166,
      "fcm_dpo/q_t": 0.48952922224998474,
      "grad_norm": 27.885974884033203,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -0.6832341551780701,
      "logits/rejected": -0.7035021185874939,
      "logps/chosen": -269.75299072265625,
      "logps/ref_chosen": -272.9283142089844,
      "logps/ref_rejected": -280.94696044921875,
      "logps/rejected": -281.9742431640625,
      "loss": 5.3896,
      "margin_dpo/margin_mean": 4.20259952545166,
      "margin_dpo/margin_std": 10.182978630065918,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": 4.06806755065918,
      "KL/mean": 2.2403817176818848,
      "KL/rejected_KL_mean": 0.41269874572753906,
      "KL/std": 7.132499694824219,
      "epoch": 0.14031413612565444,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 3.6553730964660645,
      "fcm_dpo/q_t": 0.49088141322135925,
      "grad_norm": 25.494956970214844,
      "learning_rate": 4.978312411558517e-07,
      "logits/chosen": -0.6895359754562378,
      "logits/rejected": -0.7218597531318665,
      "logps/chosen": -262.118896484375,
      "logps/ref_chosen": -266.18695068359375,
      "logps/ref_rejected": -250.17405700683594,
      "logps/rejected": -249.76138305664062,
      "loss": 5.4086,
      "margin_dpo/margin_mean": 3.655371904373169,
      "margin_dpo/margin_std": 8.813667297363281,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": 3.047025680541992,
      "KL/mean": 0.7095094323158264,
      "KL/rejected_KL_mean": -1.6280040740966797,
      "KL/std": 8.07433795928955,
      "epoch": 0.1424083769633508,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.675025463104248,
      "fcm_dpo/q_t": 0.48837074637413025,
      "grad_norm": 28.20166778564453,
      "learning_rate": 4.975839738974473e-07,
      "logits/chosen": -0.6875774264335632,
      "logits/rejected": -0.701158881187439,
      "logps/chosen": -294.8915710449219,
      "logps/ref_chosen": -297.9385986328125,
      "logps/ref_rejected": -261.5141296386719,
      "logps/rejected": -263.1421203613281,
      "loss": 5.3723,
      "margin_dpo/margin_mean": 4.6750264167785645,
      "margin_dpo/margin_std": 10.54362964630127,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": 3.8675155639648438,
      "KL/mean": 0.8734449148178101,
      "KL/rejected_KL_mean": -2.1206188201904297,
      "KL/std": 7.855175018310547,
      "epoch": 0.14450261780104712,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.988137722015381,
      "fcm_dpo/q_t": 0.48507678508758545,
      "grad_norm": 28.532976150512695,
      "learning_rate": 4.97323429461901e-07,
      "logits/chosen": -0.6833586096763611,
      "logits/rejected": -0.7138805985450745,
      "logps/chosen": -261.75,
      "logps/ref_chosen": -265.6175231933594,
      "logps/ref_rejected": -236.8287353515625,
      "logps/rejected": -238.9493408203125,
      "loss": 5.3193,
      "margin_dpo/margin_mean": 5.988138675689697,
      "margin_dpo/margin_std": 9.709370613098145,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": 4.31059455871582,
      "KL/mean": 1.392843246459961,
      "KL/rejected_KL_mean": -1.5249042510986328,
      "KL/std": 8.663267135620117,
      "epoch": 0.14659685863874344,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.8354997634887695,
      "fcm_dpo/q_t": 0.48546600341796875,
      "grad_norm": 28.64533042907715,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": -0.673731803894043,
      "logits/rejected": -0.7068012356758118,
      "logps/chosen": -291.91534423828125,
      "logps/ref_chosen": -296.2259216308594,
      "logps/ref_rejected": -254.68496704101562,
      "logps/rejected": -256.2098388671875,
      "loss": 5.328,
      "margin_dpo/margin_mean": 5.835498332977295,
      "margin_dpo/margin_std": 10.900845527648926,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": 5.133369445800781,
      "KL/mean": 2.5695688724517822,
      "KL/rejected_KL_mean": 0.0057697296142578125,
      "KL/std": 9.295702934265137,
      "epoch": 0.1486910994764398,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.12759256362915,
      "fcm_dpo/q_t": 0.48721927404403687,
      "grad_norm": 28.448589324951172,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -0.6521552205085754,
      "logits/rejected": -0.641081690788269,
      "logps/chosen": -283.7938537597656,
      "logps/ref_chosen": -288.92724609375,
      "logps/ref_rejected": -278.6405334472656,
      "logps/rejected": -278.6347351074219,
      "loss": 5.3592,
      "margin_dpo/margin_mean": 5.12759256362915,
      "margin_dpo/margin_std": 12.25449275970459,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": 3.2922000885009766,
      "KL/mean": 0.7612596750259399,
      "KL/rejected_KL_mean": -1.7696723937988281,
      "KL/std": 8.423717498779297,
      "epoch": 0.15078534031413612,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.061861038208008,
      "fcm_dpo/q_t": 0.4873760938644409,
      "grad_norm": 28.14560317993164,
      "learning_rate": 4.964622763700252e-07,
      "logits/chosen": -0.6828498244285583,
      "logits/rejected": -0.6933290362358093,
      "logps/chosen": -233.75311279296875,
      "logps/ref_chosen": -237.0452880859375,
      "logps/ref_rejected": -252.7946319580078,
      "logps/rejected": -254.56431579589844,
      "loss": 5.3579,
      "margin_dpo/margin_mean": 5.061861515045166,
      "margin_dpo/margin_std": 10.47833251953125,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": 4.2953033447265625,
      "KL/mean": 1.9382250308990479,
      "KL/rejected_KL_mean": -0.418853759765625,
      "KL/std": 8.961085319519043,
      "epoch": 0.15287958115183245,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.714160442352295,
      "fcm_dpo/q_t": 0.48824095726013184,
      "grad_norm": 27.88671875,
      "learning_rate": 4.961487700566646e-07,
      "logits/chosen": -0.650427520275116,
      "logits/rejected": -0.6673690676689148,
      "logps/chosen": -268.7577819824219,
      "logps/ref_chosen": -273.0531005859375,
      "logps/ref_rejected": -246.8330841064453,
      "logps/rejected": -247.251953125,
      "loss": 5.3754,
      "margin_dpo/margin_mean": 4.714159965515137,
      "margin_dpo/margin_std": 12.311352729797363,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": 3.3842735290527344,
      "KL/mean": 1.139523983001709,
      "KL/rejected_KL_mean": -1.1052322387695312,
      "KL/std": 9.33742904663086,
      "epoch": 0.1549738219895288,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 4.489516735076904,
      "fcm_dpo/q_t": 0.4888220429420471,
      "grad_norm": 30.374713897705078,
      "learning_rate": 4.958220635317885e-07,
      "logits/chosen": -0.727591872215271,
      "logits/rejected": -0.705908477306366,
      "logps/chosen": -338.89764404296875,
      "logps/ref_chosen": -342.2818908691406,
      "logps/ref_rejected": -330.0293884277344,
      "logps/rejected": -331.13458251953125,
      "loss": 5.3825,
      "margin_dpo/margin_mean": 4.489516258239746,
      "margin_dpo/margin_std": 11.6991548538208,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": 4.628074645996094,
      "KL/mean": 1.2845404148101807,
      "KL/rejected_KL_mean": -2.0590038299560547,
      "KL/std": 9.426990509033203,
      "epoch": 0.15706806282722513,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.687081336975098,
      "fcm_dpo/q_t": 0.48334014415740967,
      "grad_norm": 29.525991439819336,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": -0.6397058367729187,
      "logits/rejected": -0.6411034464836121,
      "logps/chosen": -262.236083984375,
      "logps/ref_chosen": -266.8641662597656,
      "logps/ref_rejected": -276.8699951171875,
      "logps/rejected": -278.92901611328125,
      "loss": 5.2956,
      "margin_dpo/margin_mean": 6.687081336975098,
      "margin_dpo/margin_std": 11.006551742553711,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": 4.128786087036133,
      "KL/mean": 0.5162752866744995,
      "KL/rejected_KL_mean": -3.096240997314453,
      "KL/std": 9.9516019821167,
      "epoch": 0.15916230366492146,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.225029945373535,
      "fcm_dpo/q_t": 0.4820476174354553,
      "grad_norm": 29.189165115356445,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -0.7249865531921387,
      "logits/rejected": -0.7330608367919922,
      "logps/chosen": -277.0457763671875,
      "logps/ref_chosen": -281.174560546875,
      "logps/ref_rejected": -263.6067199707031,
      "logps/rejected": -266.70294189453125,
      "loss": 5.2791,
      "margin_dpo/margin_mean": 7.225028038024902,
      "margin_dpo/margin_std": 12.833976745605469,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": 3.7848587036132812,
      "KL/mean": 0.8554364442825317,
      "KL/rejected_KL_mean": -2.0739879608154297,
      "KL/std": 11.057303428649902,
      "epoch": 0.1612565445026178,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.858854293823242,
      "fcm_dpo/q_t": 0.48545295000076294,
      "grad_norm": 33.1968994140625,
      "learning_rate": 4.947629214246236e-07,
      "logits/chosen": -0.5619158148765564,
      "logits/rejected": -0.5707319974899292,
      "logps/chosen": -302.3104248046875,
      "logps/ref_chosen": -306.09527587890625,
      "logps/ref_rejected": -253.49569702148438,
      "logps/rejected": -255.56968688964844,
      "loss": 5.3353,
      "margin_dpo/margin_mean": 5.858853816986084,
      "margin_dpo/margin_std": 13.917139053344727,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": 3.8920040130615234,
      "KL/mean": -0.4653443694114685,
      "KL/rejected_KL_mean": -4.822689056396484,
      "KL/std": 11.011449813842773,
      "epoch": 0.16335078534031414,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.714692115783691,
      "fcm_dpo/q_t": 0.478377103805542,
      "grad_norm": 29.64669418334961,
      "learning_rate": 4.943835963210323e-07,
      "logits/chosen": -0.6889611482620239,
      "logits/rejected": -0.6843315362930298,
      "logps/chosen": -253.0103302001953,
      "logps/ref_chosen": -256.90234375,
      "logps/ref_rejected": -211.57154846191406,
      "logps/rejected": -216.39425659179688,
      "loss": 5.2258,
      "margin_dpo/margin_mean": 8.714691162109375,
      "margin_dpo/margin_std": 14.447361946105957,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": 3.1355762481689453,
      "KL/mean": -1.0290173292160034,
      "KL/rejected_KL_mean": -5.193607330322266,
      "KL/std": 11.810872077941895,
      "epoch": 0.16544502617801046,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.329183578491211,
      "fcm_dpo/q_t": 0.47933313250541687,
      "grad_norm": 29.96116828918457,
      "learning_rate": 4.939911656668361e-07,
      "logits/chosen": -0.6535371541976929,
      "logits/rejected": -0.6741061806678772,
      "logps/chosen": -263.13800048828125,
      "logps/ref_chosen": -266.2735595703125,
      "logps/ref_rejected": -251.57257080078125,
      "logps/rejected": -256.76617431640625,
      "loss": 5.2431,
      "margin_dpo/margin_mean": 8.329182624816895,
      "margin_dpo/margin_std": 15.062546730041504,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": 1.823995590209961,
      "KL/mean": -1.4612393379211426,
      "KL/rejected_KL_mean": -4.746469497680664,
      "KL/std": 11.360502243041992,
      "epoch": 0.16753926701570682,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 6.570463180541992,
      "fcm_dpo/q_t": 0.48369115591049194,
      "grad_norm": 28.999744415283203,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": -0.67144775390625,
      "logits/rejected": -0.7016305923461914,
      "logps/chosen": -286.0270080566406,
      "logps/ref_chosen": -287.8509826660156,
      "logps/ref_rejected": -256.0766296386719,
      "logps/rejected": -260.8230895996094,
      "loss": 5.3064,
      "margin_dpo/margin_mean": 6.570462703704834,
      "margin_dpo/margin_std": 13.166526794433594,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": 2.382204055786133,
      "KL/mean": -1.1927471160888672,
      "KL/rejected_KL_mean": -4.767694473266602,
      "KL/std": 12.2095365524292,
      "epoch": 0.16963350785340314,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.14989709854126,
      "fcm_dpo/q_t": 0.48227113485336304,
      "grad_norm": 28.133319854736328,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -0.6487826108932495,
      "logits/rejected": -0.6483340263366699,
      "logps/chosen": -266.1409912109375,
      "logps/ref_chosen": -268.5232238769531,
      "logps/ref_rejected": -237.81137084960938,
      "logps/rejected": -242.57904052734375,
      "loss": 5.294,
      "margin_dpo/margin_mean": 7.149898052215576,
      "margin_dpo/margin_std": 16.77553367614746,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": 2.3929805755615234,
      "KL/mean": -1.5555365085601807,
      "KL/rejected_KL_mean": -5.504051208496094,
      "KL/std": 12.21084213256836,
      "epoch": 0.17172774869109947,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.897035121917725,
      "fcm_dpo/q_t": 0.4804663062095642,
      "grad_norm": 27.87925148010254,
      "learning_rate": 4.92735454356513e-07,
      "logits/chosen": -0.7268998622894287,
      "logits/rejected": -0.734237015247345,
      "logps/chosen": -276.970947265625,
      "logps/ref_chosen": -279.36395263671875,
      "logps/ref_rejected": -236.51365661621094,
      "logps/rejected": -242.01771545410156,
      "loss": 5.264,
      "margin_dpo/margin_mean": 7.897035121917725,
      "margin_dpo/margin_std": 15.801900863647461,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": 0.21791458129882812,
      "KL/mean": -4.0900774002075195,
      "KL/rejected_KL_mean": -8.398073196411133,
      "KL/std": 13.566779136657715,
      "epoch": 0.17382198952879582,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.61598014831543,
      "fcm_dpo/q_t": 0.4786100387573242,
      "grad_norm": 30.80162239074707,
      "learning_rate": 4.922908189595017e-07,
      "logits/chosen": -0.6965982913970947,
      "logits/rejected": -0.68050217628479,
      "logps/chosen": -274.0013122558594,
      "logps/ref_chosen": -274.21923828125,
      "logps/ref_rejected": -276.2212219238281,
      "logps/rejected": -284.6192626953125,
      "loss": 5.2445,
      "margin_dpo/margin_mean": 8.615979194641113,
      "margin_dpo/margin_std": 18.128047943115234,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": 0.579315185546875,
      "KL/mean": -3.078718423843384,
      "KL/rejected_KL_mean": -6.736743927001953,
      "KL/std": 13.271068572998047,
      "epoch": 0.17591623036649215,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 7.316063404083252,
      "fcm_dpo/q_t": 0.48185282945632935,
      "grad_norm": 29.842124938964844,
      "learning_rate": 4.918331902411841e-07,
      "logits/chosen": -0.731010377407074,
      "logits/rejected": -0.745148241519928,
      "logps/chosen": -293.8182373046875,
      "logps/ref_chosen": -294.3975524902344,
      "logps/ref_rejected": -279.81884765625,
      "logps/rejected": -286.5555725097656,
      "loss": 5.2887,
      "margin_dpo/margin_mean": 7.31606388092041,
      "margin_dpo/margin_std": 16.805049896240234,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": -1.5383739471435547,
      "KL/mean": -4.415766716003418,
      "KL/rejected_KL_mean": -7.293155670166016,
      "KL/std": 12.30350399017334,
      "epoch": 0.17801047120418848,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 5.7547736167907715,
      "fcm_dpo/q_t": 0.48571985960006714,
      "grad_norm": 29.31752586364746,
      "learning_rate": 4.913625927427995e-07,
      "logits/chosen": -0.6745836734771729,
      "logits/rejected": -0.6828963756561279,
      "logps/chosen": -245.2005615234375,
      "logps/ref_chosen": -243.66220092773438,
      "logps/ref_rejected": -263.9421691894531,
      "logps/rejected": -271.2353210449219,
      "loss": 5.3453,
      "margin_dpo/margin_mean": 5.7547736167907715,
      "margin_dpo/margin_std": 15.48192310333252,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": 1.201507568359375,
      "KL/mean": -3.2460994720458984,
      "KL/rejected_KL_mean": -7.6937103271484375,
      "KL/std": 13.281003952026367,
      "epoch": 0.18010471204188483,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.895225524902344,
      "fcm_dpo/q_t": 0.477919340133667,
      "grad_norm": 35.50041198730469,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -0.687268078327179,
      "logits/rejected": -0.6804224848747253,
      "logps/chosen": -308.22918701171875,
      "logps/ref_chosen": -309.4306945800781,
      "logps/ref_rejected": -290.91278076171875,
      "logps/rejected": -298.6064758300781,
      "loss": 5.2291,
      "margin_dpo/margin_mean": 8.895223617553711,
      "margin_dpo/margin_std": 17.361343383789062,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": -0.17951202392578125,
      "KL/mean": -5.308200836181641,
      "KL/rejected_KL_mean": -10.436893463134766,
      "KL/std": 15.234745979309082,
      "epoch": 0.18219895287958116,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.25738525390625,
      "fcm_dpo/q_t": 0.4746708869934082,
      "grad_norm": 29.576007843017578,
      "learning_rate": 4.903825930468148e-07,
      "logits/chosen": -0.7614724040031433,
      "logits/rejected": -0.7564012408256531,
      "logps/chosen": -278.2072448730469,
      "logps/ref_chosen": -278.0277099609375,
      "logps/ref_rejected": -245.70123291015625,
      "logps/rejected": -256.13812255859375,
      "loss": 5.1838,
      "margin_dpo/margin_mean": 10.25738525390625,
      "margin_dpo/margin_std": 18.901840209960938,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": -2.139179229736328,
      "KL/mean": -6.588398456573486,
      "KL/rejected_KL_mean": -11.037609100341797,
      "KL/std": 15.035432815551758,
      "epoch": 0.18429319371727748,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.898429870605469,
      "fcm_dpo/q_t": 0.47800421714782715,
      "grad_norm": 28.77937126159668,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": -0.7642254829406738,
      "logits/rejected": -0.7801080346107483,
      "logps/chosen": -268.65399169921875,
      "logps/ref_chosen": -266.5148010253906,
      "logps/ref_rejected": -265.90081787109375,
      "logps/rejected": -276.93841552734375,
      "loss": 5.2354,
      "margin_dpo/margin_mean": 8.898430824279785,
      "margin_dpo/margin_std": 19.121912002563477,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": -0.026453018188476562,
      "KL/mean": -4.663508415222168,
      "KL/rejected_KL_mean": -9.300565719604492,
      "KL/std": 14.203524589538574,
      "epoch": 0.18638743455497384,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.274109840393066,
      "fcm_dpo/q_t": 0.47702857851982117,
      "grad_norm": 30.51325798034668,
      "learning_rate": 4.893510300863676e-07,
      "logits/chosen": -0.7511242628097534,
      "logits/rejected": -0.7423003315925598,
      "logps/chosen": -265.71575927734375,
      "logps/ref_chosen": -265.6893005371094,
      "logps/ref_rejected": -251.49314880371094,
      "logps/rejected": -260.793701171875,
      "loss": 5.2202,
      "margin_dpo/margin_mean": 9.27410888671875,
      "margin_dpo/margin_std": 18.239276885986328,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": -1.1146183013916016,
      "KL/mean": -5.223875999450684,
      "KL/rejected_KL_mean": -9.333135604858398,
      "KL/std": 14.293412208557129,
      "epoch": 0.18848167539267016,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 8.218518257141113,
      "fcm_dpo/q_t": 0.47962281107902527,
      "grad_norm": 29.832656860351562,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": -0.7413437366485596,
      "logits/rejected": -0.750033438205719,
      "logps/chosen": -308.5396423339844,
      "logps/ref_chosen": -307.4250183105469,
      "logps/ref_rejected": -265.7172546386719,
      "logps/rejected": -275.0503845214844,
      "loss": 5.2571,
      "margin_dpo/margin_mean": 8.21851921081543,
      "margin_dpo/margin_std": 17.65616226196289,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": -2.107004165649414,
      "KL/mean": -6.873067855834961,
      "KL/rejected_KL_mean": -11.639135360717773,
      "KL/std": 14.819557189941406,
      "epoch": 0.1905759162303665,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.532129287719727,
      "fcm_dpo/q_t": 0.47649314999580383,
      "grad_norm": 33.01511001586914,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -0.6739709377288818,
      "logits/rejected": -0.6908794641494751,
      "logps/chosen": -237.84799194335938,
      "logps/ref_chosen": -235.74098205566406,
      "logps/ref_rejected": -226.6428985595703,
      "logps/rejected": -238.28204345703125,
      "loss": 5.2151,
      "margin_dpo/margin_mean": 9.53212833404541,
      "margin_dpo/margin_std": 19.21533966064453,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": -3.0767154693603516,
      "KL/mean": -8.113126754760742,
      "KL/rejected_KL_mean": -13.149515151977539,
      "KL/std": 17.398942947387695,
      "epoch": 0.19267015706806281,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.072810173034668,
      "fcm_dpo/q_t": 0.47510626912117004,
      "grad_norm": 33.4132194519043,
      "learning_rate": 4.877074915775048e-07,
      "logits/chosen": -0.7294397354125977,
      "logits/rejected": -0.7123967409133911,
      "logps/chosen": -286.5242614746094,
      "logps/ref_chosen": -283.4475402832031,
      "logps/ref_rejected": -273.134033203125,
      "logps/rejected": -286.2835693359375,
      "loss": 5.1999,
      "margin_dpo/margin_mean": 10.072809219360352,
      "margin_dpo/margin_std": 21.200801849365234,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": -2.370962142944336,
      "KL/mean": -7.201323509216309,
      "KL/rejected_KL_mean": -12.031681060791016,
      "KL/std": 15.408292770385742,
      "epoch": 0.19476439790575917,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.660724639892578,
      "fcm_dpo/q_t": 0.4761650562286377,
      "grad_norm": 28.56575584411621,
      "learning_rate": 4.871341104867864e-07,
      "logits/chosen": -0.7372090220451355,
      "logits/rejected": -0.7614083290100098,
      "logps/chosen": -235.70811462402344,
      "logps/ref_chosen": -233.33714294433594,
      "logps/ref_rejected": -230.54273986816406,
      "logps/rejected": -242.57440185546875,
      "loss": 5.2089,
      "margin_dpo/margin_mean": 9.660725593566895,
      "margin_dpo/margin_std": 19.43204689025879,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": -2.3014163970947266,
      "KL/mean": -6.997286796569824,
      "KL/rejected_KL_mean": -11.693157196044922,
      "KL/std": 16.432268142700195,
      "epoch": 0.1968586387434555,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 9.39173698425293,
      "fcm_dpo/q_t": 0.4767668843269348,
      "grad_norm": 31.365861892700195,
      "learning_rate": 4.865480126133871e-07,
      "logits/chosen": -0.6905912756919861,
      "logits/rejected": -0.7123920321464539,
      "logps/chosen": -296.9541931152344,
      "logps/ref_chosen": -294.6528015136719,
      "logps/ref_rejected": -283.657958984375,
      "logps/rejected": -295.35113525390625,
      "loss": 5.2295,
      "margin_dpo/margin_mean": 9.39173698425293,
      "margin_dpo/margin_std": 21.868799209594727,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": -3.325338363647461,
      "KL/mean": -8.352673530578613,
      "KL/rejected_KL_mean": -13.379995346069336,
      "KL/std": 18.014055252075195,
      "epoch": 0.19895287958115182,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.054659843444824,
      "fcm_dpo/q_t": 0.47524651885032654,
      "grad_norm": 33.56502914428711,
      "learning_rate": 4.859492293879573e-07,
      "logits/chosen": -0.7306184768676758,
      "logits/rejected": -0.7523177266120911,
      "logps/chosen": -314.9950256347656,
      "logps/ref_chosen": -311.6697082519531,
      "logps/ref_rejected": -262.7471923828125,
      "logps/rejected": -276.1271667480469,
      "loss": 5.2104,
      "margin_dpo/margin_mean": 10.054659843444824,
      "margin_dpo/margin_std": 22.503978729248047,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": -4.69744873046875,
      "KL/mean": -9.833925247192383,
      "KL/rejected_KL_mean": -14.970409393310547,
      "KL/std": 19.27062225341797,
      "epoch": 0.20104712041884817,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 10.272958755493164,
      "fcm_dpo/q_t": 0.47473055124282837,
      "grad_norm": 34.23600387573242,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -0.7016977667808533,
      "logits/rejected": -0.7141472697257996,
      "logps/chosen": -287.2534484863281,
      "logps/ref_chosen": -282.55596923828125,
      "logps/ref_rejected": -242.71588134765625,
      "logps/rejected": -257.686279296875,
      "loss": 5.2031,
      "margin_dpo/margin_mean": 10.272958755493164,
      "margin_dpo/margin_std": 23.312923431396484,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": -4.325815200805664,
      "KL/mean": -10.312779426574707,
      "KL/rejected_KL_mean": -16.299734115600586,
      "KL/std": 17.278697967529297,
      "epoch": 0.2031413612565445,
      "fcm_dpo/beta": 0.009999998845160007,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 11.97391128540039,
      "fcm_dpo/q_t": 0.470502108335495,
      "grad_norm": 32.034175872802734,
      "learning_rate": 4.847137360032699e-07,
      "logits/chosen": -0.7554161548614502,
      "logits/rejected": -0.7431137561798096,
      "logps/chosen": -307.90362548828125,
      "logps/ref_chosen": -303.57781982421875,
      "logps/ref_rejected": -264.22491455078125,
      "logps/rejected": -280.5246887207031,
      "loss": 5.1332,
      "margin_dpo/margin_mean": 11.973912239074707,
      "margin_dpo/margin_std": 22.30205535888672,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": -4.808895111083984,
      "KL/mean": -10.913530349731445,
      "KL/rejected_KL_mean": -17.018169403076172,
      "KL/std": 18.934410095214844,
      "epoch": 0.20523560209424083,
      "fcm_dpo/beta": 0.010225581005215645,
      "fcm_dpo/delta": 0.050084613263607025,
      "fcm_dpo/margin": 12.209274291992188,
      "fcm_dpo/q_t": 0.4698373079299927,
      "grad_norm": 35.21879577636719,
      "learning_rate": 4.84077092099773e-07,
      "logits/chosen": -0.7706345915794373,
      "logits/rejected": -0.7827906608581543,
      "logps/chosen": -291.63922119140625,
      "logps/ref_chosen": -286.8303527832031,
      "logps/ref_rejected": -278.08331298828125,
      "logps/rejected": -295.10150146484375,
      "loss": 5.1305,
      "margin_dpo/margin_mean": 12.209275245666504,
      "margin_dpo/margin_std": 22.742431640625,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": -5.250001907348633,
      "KL/mean": -11.552210807800293,
      "KL/rejected_KL_mean": -17.854415893554688,
      "KL/std": 19.800201416015625,
      "epoch": 0.20732984293193718,
      "fcm_dpo/beta": 0.010584528557956219,
      "fcm_dpo/delta": 0.057685159146785736,
      "fcm_dpo/margin": 12.604412078857422,
      "fcm_dpo/q_t": 0.4675399959087372,
      "grad_norm": 34.393184661865234,
      "learning_rate": 4.834278953522137e-07,
      "logits/chosen": -0.7348243594169617,
      "logits/rejected": -0.7475413084030151,
      "logps/chosen": -285.17120361328125,
      "logps/ref_chosen": -279.92120361328125,
      "logps/ref_rejected": -250.3365478515625,
      "logps/rejected": -268.1909484863281,
      "loss": 5.1162,
      "margin_dpo/margin_mean": 12.604412078857422,
      "margin_dpo/margin_std": 27.16425323486328,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": -7.852087020874023,
      "KL/mean": -14.079845428466797,
      "KL/rejected_KL_mean": -20.307607650756836,
      "KL/std": 21.994434356689453,
      "epoch": 0.2094240837696335,
      "fcm_dpo/beta": 0.01122270617634058,
      "fcm_dpo/delta": 0.04895498603582382,
      "fcm_dpo/margin": 12.455517768859863,
      "fcm_dpo/q_t": 0.46626055240631104,
      "grad_norm": 39.31376647949219,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": -0.7788305878639221,
      "logits/rejected": -0.7915370464324951,
      "logps/chosen": -304.6798095703125,
      "logps/ref_chosen": -296.8276672363281,
      "logps/ref_rejected": -275.56146240234375,
      "logps/rejected": -295.8690490722656,
      "loss": 5.0871,
      "margin_dpo/margin_mean": 12.455516815185547,
      "margin_dpo/margin_std": 24.149505615234375,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": -5.106649398803711,
      "KL/mean": -12.583588600158691,
      "KL/rejected_KL_mean": -20.060527801513672,
      "KL/std": 22.745441436767578,
      "epoch": 0.21151832460732983,
      "fcm_dpo/beta": 0.011516120284795761,
      "fcm_dpo/delta": 0.04284465312957764,
      "fcm_dpo/margin": 14.953871726989746,
      "fcm_dpo/q_t": 0.4584512412548065,
      "grad_norm": 36.80229949951172,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -0.7645897269248962,
      "logits/rejected": -0.7782154679298401,
      "logps/chosen": -257.8486633300781,
      "logps/ref_chosen": -252.74203491210938,
      "logps/ref_rejected": -276.4185485839844,
      "logps/rejected": -296.4790954589844,
      "loss": 4.9851,
      "margin_dpo/margin_mean": 14.95387077331543,
      "margin_dpo/margin_std": 26.34838104248047,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": -5.091787338256836,
      "KL/mean": -12.538023948669434,
      "KL/rejected_KL_mean": -19.98426055908203,
      "KL/std": 19.70044708251953,
      "epoch": 0.2136125654450262,
      "fcm_dpo/beta": 0.012297187000513077,
      "fcm_dpo/delta": 0.09727917611598969,
      "fcm_dpo/margin": 14.892480850219727,
      "fcm_dpo/q_t": 0.4556141197681427,
      "grad_norm": 38.596126556396484,
      "learning_rate": 4.814053395442932e-07,
      "logits/chosen": -0.7544700503349304,
      "logits/rejected": -0.7506892681121826,
      "logps/chosen": -224.6455078125,
      "logps/ref_chosen": -219.5537109375,
      "logps/ref_rejected": -231.90853881835938,
      "logps/rejected": -251.89280700683594,
      "loss": 4.9461,
      "margin_dpo/margin_mean": 14.89248275756836,
      "margin_dpo/margin_std": 24.8420352935791,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": -4.679975509643555,
      "KL/mean": -11.467307090759277,
      "KL/rejected_KL_mean": -18.25464630126953,
      "KL/std": 21.565065383911133,
      "epoch": 0.2157068062827225,
      "fcm_dpo/beta": 0.013260327279567719,
      "fcm_dpo/delta": 0.07738283276557922,
      "fcm_dpo/margin": 13.574670791625977,
      "fcm_dpo/q_t": 0.45749619603157043,
      "grad_norm": 42.808231353759766,
      "learning_rate": 4.807062862684873e-07,
      "logits/chosen": -0.7717380523681641,
      "logits/rejected": -0.7687013149261475,
      "logps/chosen": -264.35504150390625,
      "logps/ref_chosen": -259.6750793457031,
      "logps/ref_rejected": -278.7400817871094,
      "logps/rejected": -296.9947509765625,
      "loss": 4.9815,
      "margin_dpo/margin_mean": 13.574668884277344,
      "margin_dpo/margin_std": 25.648746490478516,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": -9.099061965942383,
      "KL/mean": -14.080096244812012,
      "KL/rejected_KL_mean": -19.06113052368164,
      "KL/std": 22.42926597595215,
      "epoch": 0.21780104712041884,
      "fcm_dpo/beta": 0.013728268444538116,
      "fcm_dpo/delta": 0.056247130036354065,
      "fcm_dpo/margin": 9.962065696716309,
      "fcm_dpo/q_t": 0.467223584651947,
      "grad_norm": 45.04121780395508,
      "learning_rate": 4.799948609147061e-07,
      "logits/chosen": -0.7725397348403931,
      "logits/rejected": -0.779254138469696,
      "logps/chosen": -277.0732421875,
      "logps/ref_chosen": -267.9741516113281,
      "logps/ref_rejected": -230.5306396484375,
      "logps/rejected": -249.59176635742188,
      "loss": 5.1481,
      "margin_dpo/margin_mean": 9.962067604064941,
      "margin_dpo/margin_std": 26.271961212158203,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": -5.214313507080078,
      "KL/mean": -15.498095512390137,
      "KL/rejected_KL_mean": -25.781888961791992,
      "KL/std": 25.159330368041992,
      "epoch": 0.2198952879581152,
      "fcm_dpo/beta": 0.014470679685473442,
      "fcm_dpo/delta": 0.07279935479164124,
      "fcm_dpo/margin": 20.567564010620117,
      "fcm_dpo/q_t": 0.4290893077850342,
      "grad_norm": 47.909767150878906,
      "learning_rate": 4.792711016345321e-07,
      "logits/chosen": -0.7564911842346191,
      "logits/rejected": -0.7663524150848389,
      "logps/chosen": -327.4691467285156,
      "logps/ref_chosen": -322.25482177734375,
      "logps/ref_rejected": -279.02978515625,
      "logps/rejected": -304.8116455078125,
      "loss": 4.6112,
      "margin_dpo/margin_mean": 20.56756591796875,
      "margin_dpo/margin_std": 26.976749420166016,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": -12.383544921875,
      "KL/mean": -18.636343002319336,
      "KL/rejected_KL_mean": -24.889148712158203,
      "KL/std": 23.448848724365234,
      "epoch": 0.22198952879581152,
      "fcm_dpo/beta": 0.015400283969938755,
      "fcm_dpo/delta": 0.05613970384001732,
      "fcm_dpo/margin": 12.505603790283203,
      "fcm_dpo/q_t": 0.4544346332550049,
      "grad_norm": 55.89374542236328,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -0.7401987314224243,
      "logits/rejected": -0.7758114337921143,
      "logps/chosen": -308.5413513183594,
      "logps/ref_chosen": -296.15777587890625,
      "logps/ref_rejected": -266.2691650390625,
      "logps/rejected": -291.1582946777344,
      "loss": 5.0229,
      "margin_dpo/margin_mean": 12.505602836608887,
      "margin_dpo/margin_std": 29.30100440979004,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": -4.1842041015625,
      "KL/mean": -13.946722984313965,
      "KL/rejected_KL_mean": -23.709245681762695,
      "KL/std": 21.948148727416992,
      "epoch": 0.22408376963350785,
      "fcm_dpo/beta": 0.016665179282426834,
      "fcm_dpo/delta": 0.12827080488204956,
      "fcm_dpo/margin": 19.525043487548828,
      "fcm_dpo/q_t": 0.42414212226867676,
      "grad_norm": 58.357696533203125,
      "learning_rate": 4.777867372064105e-07,
      "logits/chosen": -0.7807031869888306,
      "logits/rejected": -0.7739313840866089,
      "logps/chosen": -311.1805114746094,
      "logps/ref_chosen": -306.996337890625,
      "logps/ref_rejected": -296.79412841796875,
      "logps/rejected": -320.5033264160156,
      "loss": 4.5624,
      "margin_dpo/margin_mean": 19.525041580200195,
      "margin_dpo/margin_std": 27.53119659423828,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": -5.245018005371094,
      "KL/mean": -14.298295021057129,
      "KL/rejected_KL_mean": -23.351581573486328,
      "KL/std": 24.845993041992188,
      "epoch": 0.2261780104712042,
      "fcm_dpo/beta": 0.017718296498060226,
      "fcm_dpo/delta": 0.058396052569150925,
      "fcm_dpo/margin": 18.1065731048584,
      "fcm_dpo/q_t": 0.4270344376564026,
      "grad_norm": 186.7511444091797,
      "learning_rate": 4.770262116604223e-07,
      "logits/chosen": -0.753963053226471,
      "logits/rejected": -0.7640179991722107,
      "logps/chosen": -300.397705078125,
      "logps/ref_chosen": -295.1526794433594,
      "logps/ref_rejected": -235.974853515625,
      "logps/rejected": -259.3264465332031,
      "loss": 4.667,
      "margin_dpo/margin_mean": 18.1065731048584,
      "margin_dpo/margin_std": 29.95061492919922,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": -7.844930648803711,
      "KL/mean": -17.878192901611328,
      "KL/rejected_KL_mean": -27.911449432373047,
      "KL/std": 24.97487449645996,
      "epoch": 0.22827225130890053,
      "fcm_dpo/beta": 0.01844417303800583,
      "fcm_dpo/delta": 0.03585211560130119,
      "fcm_dpo/margin": 20.066516876220703,
      "fcm_dpo/q_t": 0.41676247119903564,
      "grad_norm": 64.53350067138672,
      "learning_rate": 4.7625351138769166e-07,
      "logits/chosen": -0.7871335744857788,
      "logits/rejected": -0.7832871079444885,
      "logps/chosen": -333.76971435546875,
      "logps/ref_chosen": -325.9248046875,
      "logps/ref_rejected": -279.15423583984375,
      "logps/rejected": -307.065673828125,
      "loss": 4.5215,
      "margin_dpo/margin_mean": 20.06651496887207,
      "margin_dpo/margin_std": 31.141969680786133,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": -8.234062194824219,
      "KL/mean": -17.65524673461914,
      "KL/rejected_KL_mean": -27.076433181762695,
      "KL/std": 26.86785888671875,
      "epoch": 0.23036649214659685,
      "fcm_dpo/beta": 0.019586404785513878,
      "fcm_dpo/delta": 0.08295932412147522,
      "fcm_dpo/margin": 18.842374801635742,
      "fcm_dpo/q_t": 0.41768479347229004,
      "grad_norm": 61.55100631713867,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": -0.8020541667938232,
      "logits/rejected": -0.7908164262771606,
      "logps/chosen": -282.67327880859375,
      "logps/ref_chosen": -274.439208984375,
      "logps/ref_rejected": -260.0552062988281,
      "logps/rejected": -287.13165283203125,
      "loss": 4.6105,
      "margin_dpo/margin_mean": 18.84237289428711,
      "margin_dpo/margin_std": 33.24864959716797,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": -8.768362045288086,
      "KL/mean": -18.7258243560791,
      "KL/rejected_KL_mean": -28.683292388916016,
      "KL/std": 27.40424346923828,
      "epoch": 0.2324607329842932,
      "fcm_dpo/beta": 0.020320452749729156,
      "fcm_dpo/delta": 0.009053957648575306,
      "fcm_dpo/margin": 19.9149227142334,
      "fcm_dpo/q_t": 0.4109950065612793,
      "grad_norm": 68.26365661621094,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -0.8383268117904663,
      "logits/rejected": -0.8181830048561096,
      "logps/chosen": -338.0045166015625,
      "logps/ref_chosen": -329.2361755371094,
      "logps/ref_rejected": -287.82830810546875,
      "logps/rejected": -316.5115661621094,
      "loss": 4.5523,
      "margin_dpo/margin_mean": 19.914920806884766,
      "margin_dpo/margin_std": 33.638423919677734,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": -14.008644104003906,
      "KL/mean": -20.38642120361328,
      "KL/rejected_KL_mean": -26.76420021057129,
      "KL/std": 26.108625411987305,
      "epoch": 0.23455497382198953,
      "fcm_dpo/beta": 0.019938142970204353,
      "fcm_dpo/delta": -0.0065420642495155334,
      "fcm_dpo/margin": 12.755571365356445,
      "fcm_dpo/q_t": 0.44417494535446167,
      "grad_norm": 79.93477630615234,
      "learning_rate": 4.7386277983585053e-07,
      "logits/chosen": -0.7363457679748535,
      "logits/rejected": -0.7664005756378174,
      "logps/chosen": -271.0680236816406,
      "logps/ref_chosen": -257.0593566894531,
      "logps/ref_rejected": -272.9595031738281,
      "logps/rejected": -299.72369384765625,
      "loss": 5.1157,
      "margin_dpo/margin_mean": 12.755572319030762,
      "margin_dpo/margin_std": 34.953208923339844,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": -11.02505874633789,
      "KL/mean": -22.8057861328125,
      "KL/rejected_KL_mean": -34.586517333984375,
      "KL/std": 29.75993537902832,
      "epoch": 0.23664921465968586,
      "fcm_dpo/beta": 0.019119691103696823,
      "fcm_dpo/delta": -0.07287082821130753,
      "fcm_dpo/margin": 23.56146240234375,
      "fcm_dpo/q_t": 0.40193551778793335,
      "grad_norm": 67.12195587158203,
      "learning_rate": 4.7304180152725024e-07,
      "logits/chosen": -0.8043359518051147,
      "logits/rejected": -0.8086544275283813,
      "logps/chosen": -297.0667419433594,
      "logps/ref_chosen": -286.0416564941406,
      "logps/ref_rejected": -270.374267578125,
      "logps/rejected": -304.96075439453125,
      "loss": 4.4367,
      "margin_dpo/margin_mean": 23.561458587646484,
      "margin_dpo/margin_std": 37.81447982788086,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": -13.95980453491211,
      "KL/mean": -20.528303146362305,
      "KL/rejected_KL_mean": -27.096799850463867,
      "KL/std": 26.299388885498047,
      "epoch": 0.2387434554973822,
      "fcm_dpo/beta": 0.019839925691485405,
      "fcm_dpo/delta": 0.07308865338563919,
      "fcm_dpo/margin": 13.136993408203125,
      "fcm_dpo/q_t": 0.44073614478111267,
      "grad_norm": 73.29736328125,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": -0.8429809212684631,
      "logits/rejected": -0.8446506857872009,
      "logps/chosen": -273.96826171875,
      "logps/ref_chosen": -260.0084533691406,
      "logps/ref_rejected": -246.67190551757812,
      "logps/rejected": -273.7687072753906,
      "loss": 5.0185,
      "margin_dpo/margin_mean": 13.136993408203125,
      "margin_dpo/margin_std": 33.331825256347656,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": -14.82553482055664,
      "KL/mean": -21.3352108001709,
      "KL/rejected_KL_mean": -27.844898223876953,
      "KL/std": 28.884841918945312,
      "epoch": 0.24083769633507854,
      "fcm_dpo/beta": 0.019825151190161705,
      "fcm_dpo/delta": 0.020870685577392578,
      "fcm_dpo/margin": 13.01936149597168,
      "fcm_dpo/q_t": 0.4461846351623535,
      "grad_norm": 84.87544250488281,
      "learning_rate": 4.7136400641330245e-07,
      "logits/chosen": -0.8594487905502319,
      "logits/rejected": -0.8216477036476135,
      "logps/chosen": -314.2485046386719,
      "logps/ref_chosen": -299.4229736328125,
      "logps/ref_rejected": -272.1186828613281,
      "logps/rejected": -299.9635925292969,
      "loss": 5.0856,
      "margin_dpo/margin_mean": 13.019363403320312,
      "margin_dpo/margin_std": 36.30274963378906,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": -9.854787826538086,
      "KL/mean": -18.613672256469727,
      "KL/rejected_KL_mean": -27.372556686401367,
      "KL/std": 26.864543914794922,
      "epoch": 0.24293193717277486,
      "fcm_dpo/beta": 0.021149354055523872,
      "fcm_dpo/delta": 0.08208739012479782,
      "fcm_dpo/margin": 17.517778396606445,
      "fcm_dpo/q_t": 0.41770169138908386,
      "grad_norm": 73.28572082519531,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -0.8507480025291443,
      "logits/rejected": -0.8137615919113159,
      "logps/chosen": -289.11871337890625,
      "logps/ref_chosen": -279.263916015625,
      "logps/ref_rejected": -253.6192169189453,
      "logps/rejected": -280.99176025390625,
      "loss": 4.6579,
      "margin_dpo/margin_mean": 17.517778396606445,
      "margin_dpo/margin_std": 32.44772720336914,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": -11.684080123901367,
      "KL/mean": -21.36503028869629,
      "KL/rejected_KL_mean": -31.045974731445312,
      "KL/std": 25.47930908203125,
      "epoch": 0.2450261780104712,
      "fcm_dpo/beta": 0.02145807445049286,
      "fcm_dpo/delta": -0.07507812231779099,
      "fcm_dpo/margin": 19.361888885498047,
      "fcm_dpo/q_t": 0.40848401188850403,
      "grad_norm": 83.57816314697266,
      "learning_rate": 4.6963872761652834e-07,
      "logits/chosen": -0.8087530732154846,
      "logits/rejected": -0.812309205532074,
      "logps/chosen": -270.9089660644531,
      "logps/ref_chosen": -259.2248840332031,
      "logps/ref_rejected": -229.3042755126953,
      "logps/rejected": -260.3502502441406,
      "loss": 4.544,
      "margin_dpo/margin_mean": 19.361888885498047,
      "margin_dpo/margin_std": 30.126319885253906,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": -15.969053268432617,
      "KL/mean": -26.981752395629883,
      "KL/rejected_KL_mean": -37.99445343017578,
      "KL/std": 31.549070358276367,
      "epoch": 0.24712041884816754,
      "fcm_dpo/beta": 0.02033599652349949,
      "fcm_dpo/delta": -0.03844477981328964,
      "fcm_dpo/margin": 22.025394439697266,
      "fcm_dpo/q_t": 0.4039861857891083,
      "grad_norm": 77.6352767944336,
      "learning_rate": 4.687583970916486e-07,
      "logits/chosen": -0.7924171090126038,
      "logits/rejected": -0.7818830013275146,
      "logps/chosen": -283.0397644042969,
      "logps/ref_chosen": -267.0707092285156,
      "logps/ref_rejected": -272.7322082519531,
      "logps/rejected": -310.7266540527344,
      "loss": 4.531,
      "margin_dpo/margin_mean": 22.025394439697266,
      "margin_dpo/margin_std": 38.41499710083008,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": -19.581802368164062,
      "KL/mean": -27.771333694458008,
      "KL/rejected_KL_mean": -35.96086120605469,
      "KL/std": 31.19072914123535,
      "epoch": 0.24921465968586387,
      "fcm_dpo/beta": 0.020136775448918343,
      "fcm_dpo/delta": 0.009949930012226105,
      "fcm_dpo/margin": 16.37906265258789,
      "fcm_dpo/q_t": 0.4295521378517151,
      "grad_norm": 86.62454223632812,
      "learning_rate": 4.6786633521783005e-07,
      "logits/chosen": -0.869214653968811,
      "logits/rejected": -0.8705704212188721,
      "logps/chosen": -344.25848388671875,
      "logps/ref_chosen": -324.6766357421875,
      "logps/ref_rejected": -306.0322265625,
      "logps/rejected": -341.99310302734375,
      "loss": 4.9316,
      "margin_dpo/margin_mean": 16.379058837890625,
      "margin_dpo/margin_std": 37.8284797668457,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": -16.1070556640625,
      "KL/mean": -24.741243362426758,
      "KL/rejected_KL_mean": -33.37543487548828,
      "KL/std": 29.768617630004883,
      "epoch": 0.2513089005235602,
      "fcm_dpo/beta": 0.0198788121342659,
      "fcm_dpo/delta": -0.03065360151231289,
      "fcm_dpo/margin": 17.268369674682617,
      "fcm_dpo/q_t": 0.4261540174484253,
      "grad_norm": 73.87181854248047,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": -0.8205348253250122,
      "logits/rejected": -0.8364426493644714,
      "logps/chosen": -331.3687744140625,
      "logps/ref_chosen": -315.2617492675781,
      "logps/ref_rejected": -265.32501220703125,
      "logps/rejected": -298.700439453125,
      "loss": 4.8544,
      "margin_dpo/margin_mean": 17.26837158203125,
      "margin_dpo/margin_std": 36.85932922363281,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": -20.559627532958984,
      "KL/mean": -27.440988540649414,
      "KL/rejected_KL_mean": -34.32234573364258,
      "KL/std": 28.46826934814453,
      "epoch": 0.2534031413612565,
      "fcm_dpo/beta": 0.02005799487233162,
      "fcm_dpo/delta": 0.07063166797161102,
      "fcm_dpo/margin": 13.762717247009277,
      "fcm_dpo/q_t": 0.4420020878314972,
      "grad_norm": 81.6031723022461,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -0.8442025780677795,
      "logits/rejected": -0.8601630926132202,
      "logps/chosen": -243.5557403564453,
      "logps/ref_chosen": -222.99609375,
      "logps/ref_rejected": -226.92860412597656,
      "logps/rejected": -261.2509460449219,
      "loss": 5.0214,
      "margin_dpo/margin_mean": 13.762717247009277,
      "margin_dpo/margin_std": 36.12423324584961,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": -20.970739364624023,
      "KL/mean": -29.586122512817383,
      "KL/rejected_KL_mean": -38.20151138305664,
      "KL/std": 32.805198669433594,
      "epoch": 0.2554973821989529,
      "fcm_dpo/beta": 0.020505763590335846,
      "fcm_dpo/delta": 0.006074222736060619,
      "fcm_dpo/margin": 17.230777740478516,
      "fcm_dpo/q_t": 0.42443346977233887,
      "grad_norm": 76.76606750488281,
      "learning_rate": 4.651202430186092e-07,
      "logits/chosen": -0.8923279643058777,
      "logits/rejected": -0.8551546335220337,
      "logps/chosen": -296.9970703125,
      "logps/ref_chosen": -276.02630615234375,
      "logps/ref_rejected": -277.97418212890625,
      "logps/rejected": -316.1756896972656,
      "loss": 4.8902,
      "margin_dpo/margin_mean": 17.230777740478516,
      "margin_dpo/margin_std": 39.82228469848633,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": -15.346237182617188,
      "KL/mean": -26.684646606445312,
      "KL/rejected_KL_mean": -38.02306365966797,
      "KL/std": 31.95981788635254,
      "epoch": 0.25759162303664923,
      "fcm_dpo/beta": 0.02076392062008381,
      "fcm_dpo/delta": -0.022594626992940903,
      "fcm_dpo/margin": 22.676815032958984,
      "fcm_dpo/q_t": 0.39692890644073486,
      "grad_norm": 84.68063354492188,
      "learning_rate": 4.6418174038722924e-07,
      "logits/chosen": -0.824637234210968,
      "logits/rejected": -0.8221345543861389,
      "logps/chosen": -343.5008239746094,
      "logps/ref_chosen": -328.1546325683594,
      "logps/ref_rejected": -280.6911315917969,
      "logps/rejected": -318.7142028808594,
      "loss": 4.4356,
      "margin_dpo/margin_mean": 22.67681312561035,
      "margin_dpo/margin_std": 36.58369445800781,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": -17.993309020996094,
      "KL/mean": -27.482837677001953,
      "KL/rejected_KL_mean": -36.972373962402344,
      "KL/std": 30.592172622680664,
      "epoch": 0.25968586387434556,
      "fcm_dpo/beta": 0.020410845056176186,
      "fcm_dpo/delta": 0.026647605001926422,
      "fcm_dpo/margin": 18.979074478149414,
      "fcm_dpo/q_t": 0.4138387143611908,
      "grad_norm": 82.6035385131836,
      "learning_rate": 4.6323175183912023e-07,
      "logits/chosen": -0.8424913883209229,
      "logits/rejected": -0.8110065460205078,
      "logps/chosen": -293.6894836425781,
      "logps/ref_chosen": -275.6961975097656,
      "logps/ref_rejected": -225.361572265625,
      "logps/rejected": -262.33392333984375,
      "loss": 4.6519,
      "margin_dpo/margin_mean": 18.979074478149414,
      "margin_dpo/margin_std": 33.819339752197266,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": -19.577743530273438,
      "KL/mean": -28.70401382446289,
      "KL/rejected_KL_mean": -37.83027648925781,
      "KL/std": 31.274581909179688,
      "epoch": 0.2617801047120419,
      "fcm_dpo/beta": 0.020590508356690407,
      "fcm_dpo/delta": -0.04334668815135956,
      "fcm_dpo/margin": 18.252531051635742,
      "fcm_dpo/q_t": 0.4232672154903412,
      "grad_norm": 88.59578704833984,
      "learning_rate": 4.6227032831928483e-07,
      "logits/chosen": -0.8063909411430359,
      "logits/rejected": -0.7640178799629211,
      "logps/chosen": -297.6474914550781,
      "logps/ref_chosen": -278.06976318359375,
      "logps/ref_rejected": -265.63873291015625,
      "logps/rejected": -303.468994140625,
      "loss": 4.9155,
      "margin_dpo/margin_mean": 18.25252914428711,
      "margin_dpo/margin_std": 42.61320495605469,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": -16.76774024963379,
      "KL/mean": -26.777427673339844,
      "KL/rejected_KL_mean": -36.78712463378906,
      "KL/std": 32.2203369140625,
      "epoch": 0.2638743455497382,
      "fcm_dpo/beta": 0.019941801205277443,
      "fcm_dpo/delta": 0.0006201639771461487,
      "fcm_dpo/margin": 20.019378662109375,
      "fcm_dpo/q_t": 0.4121100902557373,
      "grad_norm": 87.61568450927734,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -0.8104768991470337,
      "logits/rejected": -0.8245810270309448,
      "logps/chosen": -338.163818359375,
      "logps/ref_chosen": -321.3960876464844,
      "logps/ref_rejected": -285.37664794921875,
      "logps/rejected": -322.16375732421875,
      "loss": 4.6351,
      "margin_dpo/margin_mean": 20.01937484741211,
      "margin_dpo/margin_std": 37.10312271118164,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": -15.001045227050781,
      "KL/mean": -26.565738677978516,
      "KL/rejected_KL_mean": -38.13043212890625,
      "KL/std": 33.49531555175781,
      "epoch": 0.26596858638743454,
      "fcm_dpo/beta": 0.01989825814962387,
      "fcm_dpo/delta": -0.08690318465232849,
      "fcm_dpo/margin": 23.129390716552734,
      "fcm_dpo/q_t": 0.4016273617744446,
      "grad_norm": 86.43777465820312,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": -0.8895217180252075,
      "logits/rejected": -0.8410606980323792,
      "logps/chosen": -321.559814453125,
      "logps/ref_chosen": -306.55877685546875,
      "logps/ref_rejected": -274.8651428222656,
      "logps/rejected": -312.9955749511719,
      "loss": 4.5097,
      "margin_dpo/margin_mean": 23.129390716552734,
      "margin_dpo/margin_std": 37.702880859375,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": -10.226837158203125,
      "KL/mean": -23.31568717956543,
      "KL/rejected_KL_mean": -36.40452575683594,
      "KL/std": 34.26681900024414,
      "epoch": 0.2680628272251309,
      "fcm_dpo/beta": 0.018154213204979897,
      "fcm_dpo/delta": -0.06348306685686111,
      "fcm_dpo/margin": 26.177688598632812,
      "fcm_dpo/q_t": 0.39639878273010254,
      "grad_norm": 67.177001953125,
      "learning_rate": 4.5931796656116837e-07,
      "logits/chosen": -0.7768441438674927,
      "logits/rejected": -0.7786611914634705,
      "logps/chosen": -275.62420654296875,
      "logps/ref_chosen": -265.3973693847656,
      "logps/ref_rejected": -250.9737548828125,
      "logps/rejected": -287.3782653808594,
      "loss": 4.3915,
      "margin_dpo/margin_mean": 26.177684783935547,
      "margin_dpo/margin_std": 40.99726867675781,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": -11.085042953491211,
      "KL/mean": -22.47225570678711,
      "KL/rejected_KL_mean": -33.85946273803711,
      "KL/std": 32.29938888549805,
      "epoch": 0.27015706806282724,
      "fcm_dpo/beta": 0.017715973779559135,
      "fcm_dpo/delta": -0.021403057500720024,
      "fcm_dpo/margin": 22.774415969848633,
      "fcm_dpo/q_t": 0.41196408867836,
      "grad_norm": 68.8297348022461,
      "learning_rate": 4.5831132482724193e-07,
      "logits/chosen": -0.8033672571182251,
      "logits/rejected": -0.8051390647888184,
      "logps/chosen": -314.24346923828125,
      "logps/ref_chosen": -303.158447265625,
      "logps/ref_rejected": -275.9891052246094,
      "logps/rejected": -309.84857177734375,
      "loss": 4.5798,
      "margin_dpo/margin_mean": 22.774417877197266,
      "margin_dpo/margin_std": 39.89845275878906,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": -12.726642608642578,
      "KL/mean": -23.57893943786621,
      "KL/rejected_KL_mean": -34.431236267089844,
      "KL/std": 30.693164825439453,
      "epoch": 0.27225130890052357,
      "fcm_dpo/beta": 0.017723431810736656,
      "fcm_dpo/delta": 0.03383774310350418,
      "fcm_dpo/margin": 21.7045841217041,
      "fcm_dpo/q_t": 0.41520535945892334,
      "grad_norm": 79.62198638916016,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": -0.7834247350692749,
      "logits/rejected": -0.8247291445732117,
      "logps/chosen": -299.1340026855469,
      "logps/ref_chosen": -286.4073486328125,
      "logps/ref_rejected": -294.38665771484375,
      "logps/rejected": -328.8179016113281,
      "loss": 4.6135,
      "margin_dpo/margin_mean": 21.704587936401367,
      "margin_dpo/margin_std": 39.322349548339844,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": -13.694978713989258,
      "KL/mean": -23.010234832763672,
      "KL/rejected_KL_mean": -32.32548522949219,
      "KL/std": 32.626258850097656,
      "epoch": 0.2743455497382199,
      "fcm_dpo/beta": 0.01847611553966999,
      "fcm_dpo/delta": 0.03835005685687065,
      "fcm_dpo/margin": 18.63050651550293,
      "fcm_dpo/q_t": 0.42452722787857056,
      "grad_norm": 91.76261901855469,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -0.8536990284919739,
      "logits/rejected": -0.8031719923019409,
      "logps/chosen": -325.2600402832031,
      "logps/ref_chosen": -311.5650634765625,
      "logps/ref_rejected": -291.62432861328125,
      "logps/rejected": -323.94989013671875,
      "loss": 4.8167,
      "margin_dpo/margin_mean": 18.63050651550293,
      "margin_dpo/margin_std": 39.410621643066406,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": -9.488594055175781,
      "KL/mean": -22.28922462463379,
      "KL/rejected_KL_mean": -35.089866638183594,
      "KL/std": 33.668479919433594,
      "epoch": 0.2764397905759162,
      "fcm_dpo/beta": 0.0188762079924345,
      "fcm_dpo/delta": -0.03586321324110031,
      "fcm_dpo/margin": 25.601276397705078,
      "fcm_dpo/q_t": 0.39772170782089233,
      "grad_norm": 103.5088119506836,
      "learning_rate": 4.5522459192551166e-07,
      "logits/chosen": -0.80729079246521,
      "logits/rejected": -0.7911918759346008,
      "logps/chosen": -279.5704040527344,
      "logps/ref_chosen": -270.0818176269531,
      "logps/ref_rejected": -284.3084411621094,
      "logps/rejected": -319.3983154296875,
      "loss": 4.4698,
      "margin_dpo/margin_mean": 25.601276397705078,
      "margin_dpo/margin_std": 42.3785400390625,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": -10.182044982910156,
      "KL/mean": -21.770816802978516,
      "KL/rejected_KL_mean": -33.359588623046875,
      "KL/std": 31.6209716796875,
      "epoch": 0.27853403141361255,
      "fcm_dpo/beta": 0.018013302236795425,
      "fcm_dpo/delta": -0.008887548930943012,
      "fcm_dpo/margin": 23.177547454833984,
      "fcm_dpo/q_t": 0.4075259268283844,
      "grad_norm": 66.81080627441406,
      "learning_rate": 4.541735956498554e-07,
      "logits/chosen": -0.8384092450141907,
      "logits/rejected": -0.8414251804351807,
      "logps/chosen": -295.80340576171875,
      "logps/ref_chosen": -285.6213684082031,
      "logps/ref_rejected": -251.19386291503906,
      "logps/rejected": -284.5534362792969,
      "loss": 4.5392,
      "margin_dpo/margin_mean": 23.17754364013672,
      "margin_dpo/margin_std": 39.17509078979492,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": -19.709896087646484,
      "KL/mean": -28.813058853149414,
      "KL/rejected_KL_mean": -37.91621398925781,
      "KL/std": 32.60590744018555,
      "epoch": 0.2806282722513089,
      "fcm_dpo/beta": 0.01814555935561657,
      "fcm_dpo/delta": 0.0036682691425085068,
      "fcm_dpo/margin": 18.20633316040039,
      "fcm_dpo/q_t": 0.4251437783241272,
      "grad_norm": 70.55476379394531,
      "learning_rate": 4.5311165016389914e-07,
      "logits/chosen": -0.8507990837097168,
      "logits/rejected": -0.8520804047584534,
      "logps/chosen": -338.6307067871094,
      "logps/ref_chosen": -318.92083740234375,
      "logps/ref_rejected": -293.1894836425781,
      "logps/rejected": -331.1056823730469,
      "loss": 4.7793,
      "margin_dpo/margin_mean": 18.20633316040039,
      "margin_dpo/margin_std": 35.95256805419922,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": -15.35184097290039,
      "KL/mean": -26.096458435058594,
      "KL/rejected_KL_mean": -36.84107208251953,
      "KL/std": 33.33781433105469,
      "epoch": 0.28272251308900526,
      "fcm_dpo/beta": 0.017903445288538933,
      "fcm_dpo/delta": -0.01947990246117115,
      "fcm_dpo/margin": 21.48922348022461,
      "fcm_dpo/q_t": 0.41302791237831116,
      "grad_norm": 93.3125228881836,
      "learning_rate": 4.520388124165564e-07,
      "logits/chosen": -0.7447824478149414,
      "logits/rejected": -0.7873469591140747,
      "logps/chosen": -308.173583984375,
      "logps/ref_chosen": -292.8217468261719,
      "logps/ref_rejected": -269.2896728515625,
      "logps/rejected": -306.1307678222656,
      "loss": 4.6282,
      "margin_dpo/margin_mean": 21.48922348022461,
      "margin_dpo/margin_std": 37.27495574951172,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": -16.38927459716797,
      "KL/mean": -28.0660343170166,
      "KL/rejected_KL_mean": -39.74279022216797,
      "KL/std": 33.52198028564453,
      "epoch": 0.2848167539267016,
      "fcm_dpo/beta": 0.017689906060695648,
      "fcm_dpo/delta": 0.0002649594098329544,
      "fcm_dpo/margin": 23.35352325439453,
      "fcm_dpo/q_t": 0.410957008600235,
      "grad_norm": 86.35053253173828,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -0.8071991801261902,
      "logits/rejected": -0.8032166361808777,
      "logps/chosen": -289.2417907714844,
      "logps/ref_chosen": -272.8525390625,
      "logps/ref_rejected": -252.68202209472656,
      "logps/rejected": -292.4248352050781,
      "loss": 4.608,
      "margin_dpo/margin_mean": 23.35352325439453,
      "margin_dpo/margin_std": 41.85772705078125,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": -18.957660675048828,
      "KL/mean": -29.26618766784668,
      "KL/rejected_KL_mean": -39.57472610473633,
      "KL/std": 35.587188720703125,
      "epoch": 0.2869109947643979,
      "fcm_dpo/beta": 0.017967944964766502,
      "fcm_dpo/delta": 0.00379769504070282,
      "fcm_dpo/margin": 20.617074966430664,
      "fcm_dpo/q_t": 0.4201981723308563,
      "grad_norm": 75.80419921875,
      "learning_rate": 4.498606908508753e-07,
      "logits/chosen": -0.848484992980957,
      "logits/rejected": -0.8344926238059998,
      "logps/chosen": -319.70989990234375,
      "logps/ref_chosen": -300.7522277832031,
      "logps/ref_rejected": -286.1935119628906,
      "logps/rejected": -325.7682189941406,
      "loss": 4.7075,
      "margin_dpo/margin_mean": 20.617076873779297,
      "margin_dpo/margin_std": 40.14009094238281,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": -17.301189422607422,
      "KL/mean": -28.845211029052734,
      "KL/rejected_KL_mean": -40.38924026489258,
      "KL/std": 35.3648796081543,
      "epoch": 0.28900523560209423,
      "fcm_dpo/beta": 0.017695462331175804,
      "fcm_dpo/delta": -0.0005754795856773853,
      "fcm_dpo/margin": 23.088045120239258,
      "fcm_dpo/q_t": 0.41459810733795166,
      "grad_norm": 70.88033294677734,
      "learning_rate": 4.487555238385862e-07,
      "logits/chosen": -0.7724663615226746,
      "logits/rejected": -0.7530030012130737,
      "logps/chosen": -306.2381591796875,
      "logps/ref_chosen": -288.9369812011719,
      "logps/ref_rejected": -263.7076416015625,
      "logps/rejected": -304.0968933105469,
      "loss": 4.6684,
      "margin_dpo/margin_mean": 23.088045120239258,
      "margin_dpo/margin_std": 45.1626091003418,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": -16.706771850585938,
      "KL/mean": -24.65461540222168,
      "KL/rejected_KL_mean": -32.60246276855469,
      "KL/std": 32.76335906982422,
      "epoch": 0.29109947643979056,
      "fcm_dpo/beta": 0.017869288101792336,
      "fcm_dpo/delta": 0.03685586899518967,
      "fcm_dpo/margin": 15.89570426940918,
      "fcm_dpo/q_t": 0.43886858224868774,
      "grad_norm": 73.8459243774414,
      "learning_rate": 4.476396981707453e-07,
      "logits/chosen": -0.794594943523407,
      "logits/rejected": -0.8251434564590454,
      "logps/chosen": -286.7510681152344,
      "logps/ref_chosen": -270.0443115234375,
      "logps/ref_rejected": -267.3226013183594,
      "logps/rejected": -299.925048828125,
      "loss": 5.0029,
      "margin_dpo/margin_mean": 15.895706176757812,
      "margin_dpo/margin_std": 39.51491165161133,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": -16.193626403808594,
      "KL/mean": -28.337804794311523,
      "KL/rejected_KL_mean": -40.48198318481445,
      "KL/std": 34.695430755615234,
      "epoch": 0.2931937172774869,
      "fcm_dpo/beta": 0.01853698305785656,
      "fcm_dpo/delta": -0.001993805170059204,
      "fcm_dpo/margin": 24.28836441040039,
      "fcm_dpo/q_t": 0.401250422000885,
      "grad_norm": 87.66632080078125,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": -0.8747819662094116,
      "logits/rejected": -0.8443241119384766,
      "logps/chosen": -299.1492004394531,
      "logps/ref_chosen": -282.9555969238281,
      "logps/ref_rejected": -251.17181396484375,
      "logps/rejected": -291.6537780761719,
      "loss": 4.4093,
      "margin_dpo/margin_mean": 24.28836441040039,
      "margin_dpo/margin_std": 37.079402923583984,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": -14.100862503051758,
      "KL/mean": -27.639755249023438,
      "KL/rejected_KL_mean": -41.17864227294922,
      "KL/std": 34.405853271484375,
      "epoch": 0.29528795811518327,
      "fcm_dpo/beta": 0.01770884171128273,
      "fcm_dpo/delta": -0.03292696550488472,
      "fcm_dpo/margin": 27.077795028686523,
      "fcm_dpo/q_t": 0.39341747760772705,
      "grad_norm": 66.9410629272461,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -0.7984528541564941,
      "logits/rejected": -0.7945541143417358,
      "logps/chosen": -310.4010009765625,
      "logps/ref_chosen": -296.3001708984375,
      "logps/ref_rejected": -279.8486633300781,
      "logps/rejected": -321.02728271484375,
      "loss": 4.3845,
      "margin_dpo/margin_mean": 27.07779312133789,
      "margin_dpo/margin_std": 42.000736236572266,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": -16.328510284423828,
      "KL/mean": -27.186431884765625,
      "KL/rejected_KL_mean": -38.04435729980469,
      "KL/std": 35.612178802490234,
      "epoch": 0.2973821989528796,
      "fcm_dpo/beta": 0.01755218766629696,
      "fcm_dpo/delta": -0.01755281537771225,
      "fcm_dpo/margin": 21.715831756591797,
      "fcm_dpo/q_t": 0.41808396577835083,
      "grad_norm": 71.61498260498047,
      "learning_rate": 4.4422887045602674e-07,
      "logits/chosen": -0.8274088501930237,
      "logits/rejected": -0.8313606381416321,
      "logps/chosen": -316.89434814453125,
      "logps/ref_chosen": -300.56585693359375,
      "logps/ref_rejected": -231.43316650390625,
      "logps/rejected": -269.4775085449219,
      "loss": 4.7633,
      "margin_dpo/margin_mean": 21.715831756591797,
      "margin_dpo/margin_std": 44.84808349609375,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": -16.43931007385254,
      "KL/mean": -29.863502502441406,
      "KL/rejected_KL_mean": -43.28770065307617,
      "KL/std": 34.502262115478516,
      "epoch": 0.2994764397905759,
      "fcm_dpo/beta": 0.01699257083237171,
      "fcm_dpo/delta": -0.04298718273639679,
      "fcm_dpo/margin": 26.848386764526367,
      "fcm_dpo/q_t": 0.39975064992904663,
      "grad_norm": 66.07804107666016,
      "learning_rate": 4.4307101421701755e-07,
      "logits/chosen": -0.8160648345947266,
      "logits/rejected": -0.7986904382705688,
      "logps/chosen": -313.1716613769531,
      "logps/ref_chosen": -296.73236083984375,
      "logps/ref_rejected": -266.45257568359375,
      "logps/rejected": -309.74029541015625,
      "loss": 4.3731,
      "margin_dpo/margin_mean": 26.848386764526367,
      "margin_dpo/margin_std": 39.39161682128906,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": -18.505847930908203,
      "KL/mean": -29.042064666748047,
      "KL/rejected_KL_mean": -39.57829284667969,
      "KL/std": 37.26365661621094,
      "epoch": 0.30157068062827225,
      "fcm_dpo/beta": 0.01667700707912445,
      "fcm_dpo/delta": 0.016044626012444496,
      "fcm_dpo/margin": 21.07245635986328,
      "fcm_dpo/q_t": 0.42446640133857727,
      "grad_norm": 67.42301940917969,
      "learning_rate": 4.419028041654559e-07,
      "logits/chosen": -0.877031683921814,
      "logits/rejected": -0.864963948726654,
      "logps/chosen": -317.3498229980469,
      "logps/ref_chosen": -298.843994140625,
      "logps/ref_rejected": -266.120849609375,
      "logps/rejected": -305.69915771484375,
      "loss": 4.7903,
      "margin_dpo/margin_mean": 21.07245635986328,
      "margin_dpo/margin_std": 44.20958709716797,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": -15.375284194946289,
      "KL/mean": -29.93511199951172,
      "KL/rejected_KL_mean": -44.49494171142578,
      "KL/std": 34.767147064208984,
      "epoch": 0.3036649214659686,
      "fcm_dpo/beta": 0.016189519315958023,
      "fcm_dpo/delta": -0.10411692410707474,
      "fcm_dpo/margin": 29.119653701782227,
      "fcm_dpo/q_t": 0.3948172926902771,
      "grad_norm": 67.205322265625,
      "learning_rate": 4.4072430294890166e-07,
      "logits/chosen": -0.8594289422035217,
      "logits/rejected": -0.8636488914489746,
      "logps/chosen": -291.128173828125,
      "logps/ref_chosen": -275.7528381347656,
      "logps/ref_rejected": -214.74807739257812,
      "logps/rejected": -259.2430114746094,
      "loss": 4.3357,
      "margin_dpo/margin_mean": 29.119651794433594,
      "margin_dpo/margin_std": 41.30836486816406,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": -24.110055923461914,
      "KL/mean": -37.73773193359375,
      "KL/rejected_KL_mean": -51.365413665771484,
      "KL/std": 37.291194915771484,
      "epoch": 0.3057591623036649,
      "fcm_dpo/beta": 0.01583903655409813,
      "fcm_dpo/delta": 0.01731536164879799,
      "fcm_dpo/margin": 27.2553653717041,
      "fcm_dpo/q_t": 0.4049854874610901,
      "grad_norm": 89.8564453125,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -0.8339921236038208,
      "logits/rejected": -0.8299652934074402,
      "logps/chosen": -301.2082824707031,
      "logps/ref_chosen": -277.09820556640625,
      "logps/ref_rejected": -265.41046142578125,
      "logps/rejected": -316.7758483886719,
      "loss": 4.4476,
      "margin_dpo/margin_mean": 27.2553653717041,
      "margin_dpo/margin_std": 41.389373779296875,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": -25.689922332763672,
      "KL/mean": -36.09031295776367,
      "KL/rejected_KL_mean": -46.49070358276367,
      "KL/std": 37.88399124145508,
      "epoch": 0.3078534031413613,
      "fcm_dpo/beta": 0.01601884514093399,
      "fcm_dpo/delta": -0.0459786057472229,
      "fcm_dpo/margin": 20.80077362060547,
      "fcm_dpo/q_t": 0.4257439374923706,
      "grad_norm": 75.98126983642578,
      "learning_rate": 4.3833668036708483e-07,
      "logits/chosen": -0.8321329355239868,
      "logits/rejected": -0.8304070830345154,
      "logps/chosen": -317.1085205078125,
      "logps/ref_chosen": -291.4185791015625,
      "logps/ref_rejected": -253.43051147460938,
      "logps/rejected": -299.92120361328125,
      "loss": 4.8453,
      "margin_dpo/margin_mean": 20.80077362060547,
      "margin_dpo/margin_std": 43.209800720214844,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": -28.248117446899414,
      "KL/mean": -39.14514923095703,
      "KL/rejected_KL_mean": -50.04218292236328,
      "KL/std": 37.5369873046875,
      "epoch": 0.3099476439790576,
      "fcm_dpo/beta": 0.016223322600126266,
      "fcm_dpo/delta": 0.09934838861227036,
      "fcm_dpo/margin": 21.794063568115234,
      "fcm_dpo/q_t": 0.42599159479141235,
      "grad_norm": 72.12030792236328,
      "learning_rate": 4.3712768704277524e-07,
      "logits/chosen": -0.9002828598022461,
      "logits/rejected": -0.9044474959373474,
      "logps/chosen": -264.9966125488281,
      "logps/ref_chosen": -236.74850463867188,
      "logps/ref_rejected": -231.4674072265625,
      "logps/rejected": -281.5095520019531,
      "loss": 4.8145,
      "margin_dpo/margin_mean": 21.794065475463867,
      "margin_dpo/margin_std": 47.81521224975586,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": -30.28792953491211,
      "KL/mean": -45.19022750854492,
      "KL/rejected_KL_mean": -60.09253692626953,
      "KL/std": 35.71250915527344,
      "epoch": 0.31204188481675393,
      "fcm_dpo/beta": 0.016364028677344322,
      "fcm_dpo/delta": -0.04023426026105881,
      "fcm_dpo/margin": 29.804597854614258,
      "fcm_dpo/q_t": 0.3911668658256531,
      "grad_norm": 76.862548828125,
      "learning_rate": 4.3590865862851263e-07,
      "logits/chosen": -0.8551154732704163,
      "logits/rejected": -0.8408536911010742,
      "logps/chosen": -350.2164306640625,
      "logps/ref_chosen": -319.9284973144531,
      "logps/ref_rejected": -308.20233154296875,
      "logps/rejected": -368.29486083984375,
      "loss": 4.2113,
      "margin_dpo/margin_mean": 29.80459976196289,
      "margin_dpo/margin_std": 38.87329864501953,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": -38.25591278076172,
      "KL/mean": -49.75065994262695,
      "KL/rejected_KL_mean": -61.245399475097656,
      "KL/std": 37.375423431396484,
      "epoch": 0.31413612565445026,
      "fcm_dpo/beta": 0.016272133216261864,
      "fcm_dpo/delta": 0.012661180458962917,
      "fcm_dpo/margin": 22.989498138427734,
      "fcm_dpo/q_t": 0.4161220192909241,
      "grad_norm": 77.92076873779297,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": -0.8266889452934265,
      "logits/rejected": -0.8134939074516296,
      "logps/chosen": -314.57415771484375,
      "logps/ref_chosen": -276.3182373046875,
      "logps/ref_rejected": -273.02215576171875,
      "logps/rejected": -334.2675476074219,
      "loss": 4.6428,
      "margin_dpo/margin_mean": 22.989498138427734,
      "margin_dpo/margin_std": 41.48157501220703,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": -37.31572341918945,
      "KL/mean": -53.76444625854492,
      "KL/rejected_KL_mean": -70.2131576538086,
      "KL/std": 37.55862045288086,
      "epoch": 0.3162303664921466,
      "fcm_dpo/beta": 0.015896625816822052,
      "fcm_dpo/delta": -0.07649703323841095,
      "fcm_dpo/margin": 32.897438049316406,
      "fcm_dpo/q_t": 0.38501039147377014,
      "grad_norm": 92.1038589477539,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -0.8431829214096069,
      "logits/rejected": -0.8407465815544128,
      "logps/chosen": -334.6285705566406,
      "logps/ref_chosen": -297.31280517578125,
      "logps/ref_rejected": -266.1003723144531,
      "logps/rejected": -336.3135681152344,
      "loss": 4.2038,
      "margin_dpo/margin_mean": 32.89743423461914,
      "margin_dpo/margin_std": 44.485939025878906,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": -32.810096740722656,
      "KL/mean": -46.666194915771484,
      "KL/rejected_KL_mean": -60.52229690551758,
      "KL/std": 37.67792510986328,
      "epoch": 0.3183246073298429,
      "fcm_dpo/beta": 0.014960749074816704,
      "fcm_dpo/delta": -0.03140158951282501,
      "fcm_dpo/margin": 27.71218490600586,
      "fcm_dpo/q_t": 0.4084896147251129,
      "grad_norm": 89.98988342285156,
      "learning_rate": 4.3219201924364323e-07,
      "logits/chosen": -0.8643499612808228,
      "logits/rejected": -0.8640848398208618,
      "logps/chosen": -303.05712890625,
      "logps/ref_chosen": -270.2470397949219,
      "logps/ref_rejected": -269.7749328613281,
      "logps/rejected": -330.2972412109375,
      "loss": 4.4942,
      "margin_dpo/margin_mean": 27.71218490600586,
      "margin_dpo/margin_std": 43.75712203979492,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": -36.5927848815918,
      "KL/mean": -56.02995681762695,
      "KL/rejected_KL_mean": -75.4671401977539,
      "KL/std": 40.93180465698242,
      "epoch": 0.3204188481675393,
      "fcm_dpo/beta": 0.014236779883503914,
      "fcm_dpo/delta": -0.10994573682546616,
      "fcm_dpo/margin": 38.874351501464844,
      "fcm_dpo/q_t": 0.37528523802757263,
      "grad_norm": 77.16065216064453,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": -0.8517962694168091,
      "logits/rejected": -0.8410588502883911,
      "logps/chosen": -310.371826171875,
      "logps/ref_chosen": -273.779052734375,
      "logps/ref_rejected": -280.9530944824219,
      "logps/rejected": -356.4202575683594,
      "loss": 3.9851,
      "margin_dpo/margin_mean": 38.874351501464844,
      "margin_dpo/margin_std": 42.69178009033203,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -35.76517868041992,
      "KL/mean": -51.064208984375,
      "KL/rejected_KL_mean": -66.36325073242188,
      "KL/std": 43.183956146240234,
      "epoch": 0.3225130890052356,
      "fcm_dpo/beta": 0.013467169366776943,
      "fcm_dpo/delta": -0.04377404600381851,
      "fcm_dpo/margin": 30.598079681396484,
      "fcm_dpo/q_t": 0.4081461429595947,
      "grad_norm": 72.88811492919922,
      "learning_rate": 4.2966529689388064e-07,
      "logits/chosen": -0.8891314268112183,
      "logits/rejected": -0.8717354536056519,
      "logps/chosen": -325.66839599609375,
      "logps/ref_chosen": -289.9031982421875,
      "logps/ref_rejected": -261.5166320800781,
      "logps/rejected": -327.8798828125,
      "loss": 4.5527,
      "margin_dpo/margin_mean": 30.598079681396484,
      "margin_dpo/margin_std": 51.222938537597656,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -36.35716247558594,
      "KL/mean": -50.091156005859375,
      "KL/rejected_KL_mean": -63.82514953613281,
      "KL/std": 42.52281951904297,
      "epoch": 0.32460732984293195,
      "fcm_dpo/beta": 0.013422807678580284,
      "fcm_dpo/delta": 0.03404915705323219,
      "fcm_dpo/margin": 27.467987060546875,
      "fcm_dpo/q_t": 0.4197961986064911,
      "grad_norm": 86.76924133300781,
      "learning_rate": 4.2838744935687716e-07,
      "logits/chosen": -0.8197731971740723,
      "logits/rejected": -0.8191108703613281,
      "logps/chosen": -322.2183532714844,
      "logps/ref_chosen": -285.8612060546875,
      "logps/ref_rejected": -300.1272888183594,
      "logps/rejected": -363.95245361328125,
      "loss": 4.6113,
      "margin_dpo/margin_mean": 27.467988967895508,
      "margin_dpo/margin_std": 48.1390380859375,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -29.28577423095703,
      "KL/mean": -49.058311462402344,
      "KL/rejected_KL_mean": -68.83085632324219,
      "KL/std": 44.749488830566406,
      "epoch": 0.3267015706806283,
      "fcm_dpo/beta": 0.013466731645166874,
      "fcm_dpo/delta": -0.08618871867656708,
      "fcm_dpo/margin": 39.54508972167969,
      "fcm_dpo/q_t": 0.38364699482917786,
      "grad_norm": 57.406463623046875,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -0.8465090990066528,
      "logits/rejected": -0.8451894521713257,
      "logps/chosen": -308.32122802734375,
      "logps/ref_chosen": -279.0354919433594,
      "logps/ref_rejected": -244.2198486328125,
      "logps/rejected": -313.05072021484375,
      "loss": 4.2433,
      "margin_dpo/margin_mean": 39.54508590698242,
      "margin_dpo/margin_std": 55.98197555541992,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -30.07285499572754,
      "KL/mean": -45.59950637817383,
      "KL/rejected_KL_mean": -61.12614440917969,
      "KL/std": 44.99113845825195,
      "epoch": 0.3287958115183246,
      "fcm_dpo/beta": 0.012722576968371868,
      "fcm_dpo/delta": 0.021720336750149727,
      "fcm_dpo/margin": 31.05329132080078,
      "fcm_dpo/q_t": 0.4115702211856842,
      "grad_norm": 70.75900268554688,
      "learning_rate": 4.258031241903777e-07,
      "logits/chosen": -0.9161993265151978,
      "logits/rejected": -0.9153949618339539,
      "logps/chosen": -300.9031677246094,
      "logps/ref_chosen": -270.830322265625,
      "logps/ref_rejected": -259.08319091796875,
      "logps/rejected": -320.2093811035156,
      "loss": 4.5153,
      "margin_dpo/margin_mean": 31.05329132080078,
      "margin_dpo/margin_std": 49.046295166015625,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -28.351633071899414,
      "KL/mean": -45.13512420654297,
      "KL/rejected_KL_mean": -61.91861343383789,
      "KL/std": 44.75823974609375,
      "epoch": 0.3308900523560209,
      "fcm_dpo/beta": 0.012885518372058868,
      "fcm_dpo/delta": -0.04067499563097954,
      "fcm_dpo/margin": 33.56697082519531,
      "fcm_dpo/q_t": 0.40187203884124756,
      "grad_norm": 70.9114761352539,
      "learning_rate": 4.2449678515039743e-07,
      "logits/chosen": -0.8407530784606934,
      "logits/rejected": -0.8268633484840393,
      "logps/chosen": -318.3179626464844,
      "logps/ref_chosen": -289.9663391113281,
      "logps/ref_rejected": -271.335693359375,
      "logps/rejected": -333.2543029785156,
      "loss": 4.4345,
      "margin_dpo/margin_mean": 33.56697082519531,
      "margin_dpo/margin_std": 50.192176818847656,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -30.78000259399414,
      "KL/mean": -42.23385238647461,
      "KL/rejected_KL_mean": -53.68769836425781,
      "KL/std": 43.12301254272461,
      "epoch": 0.33298429319371725,
      "fcm_dpo/beta": 0.012710182927548885,
      "fcm_dpo/delta": 0.03412369266152382,
      "fcm_dpo/margin": 22.907712936401367,
      "fcm_dpo/q_t": 0.4374562203884125,
      "grad_norm": 81.2497787475586,
      "learning_rate": 4.2318108837739986e-07,
      "logits/chosen": -0.938960075378418,
      "logits/rejected": -0.8962432742118835,
      "logps/chosen": -352.15838623046875,
      "logps/ref_chosen": -321.37835693359375,
      "logps/ref_rejected": -250.45652770996094,
      "logps/rejected": -304.14422607421875,
      "loss": 4.9487,
      "margin_dpo/margin_mean": 22.907711029052734,
      "margin_dpo/margin_std": 53.699588775634766,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": -24.17879867553711,
      "KL/mean": -43.73112487792969,
      "KL/rejected_KL_mean": -63.283443450927734,
      "KL/std": 44.90157699584961,
      "epoch": 0.33507853403141363,
      "fcm_dpo/beta": 0.012634380720555782,
      "fcm_dpo/delta": -0.04687969386577606,
      "fcm_dpo/margin": 39.104652404785156,
      "fcm_dpo/q_t": 0.38788533210754395,
      "grad_norm": 70.6510009765625,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": -0.864124596118927,
      "logits/rejected": -0.8694305419921875,
      "logps/chosen": -300.4623107910156,
      "logps/ref_chosen": -276.28350830078125,
      "logps/ref_rejected": -262.7477722167969,
      "logps/rejected": -326.03118896484375,
      "loss": 4.1333,
      "margin_dpo/margin_mean": 39.104652404785156,
      "margin_dpo/margin_std": 45.4080810546875,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": -29.19247817993164,
      "KL/mean": -46.5194206237793,
      "KL/rejected_KL_mean": -63.84636688232422,
      "KL/std": 45.96017074584961,
      "epoch": 0.33717277486910996,
      "fcm_dpo/beta": 0.012704461812973022,
      "fcm_dpo/delta": 0.00892152264714241,
      "fcm_dpo/margin": 34.65388488769531,
      "fcm_dpo/q_t": 0.4035447835922241,
      "grad_norm": 67.25022888183594,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -0.8771299719810486,
      "logits/rejected": -0.8650283813476562,
      "logps/chosen": -339.6852722167969,
      "logps/ref_chosen": -310.4927978515625,
      "logps/ref_rejected": -250.25347900390625,
      "logps/rejected": -314.099853515625,
      "loss": 4.4398,
      "margin_dpo/margin_mean": 34.65388488769531,
      "margin_dpo/margin_std": 54.128562927246094,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": -26.494173049926758,
      "KL/mean": -42.02763748168945,
      "KL/rejected_KL_mean": -57.56110763549805,
      "KL/std": 43.25554275512695,
      "epoch": 0.3392670157068063,
      "fcm_dpo/beta": 0.012288411147892475,
      "fcm_dpo/delta": -0.004753082990646362,
      "fcm_dpo/margin": 31.066944122314453,
      "fcm_dpo/q_t": 0.4138047993183136,
      "grad_norm": 68.60234069824219,
      "learning_rate": 4.1917855971495763e-07,
      "logits/chosen": -0.8540545701980591,
      "logits/rejected": -0.8474206924438477,
      "logps/chosen": -322.60467529296875,
      "logps/ref_chosen": -296.1105041503906,
      "logps/ref_rejected": -253.4247589111328,
      "logps/rejected": -310.9858703613281,
      "loss": 4.5424,
      "margin_dpo/margin_mean": 31.066944122314453,
      "margin_dpo/margin_std": 49.06950759887695,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": -28.311044692993164,
      "KL/mean": -44.8758659362793,
      "KL/rejected_KL_mean": -61.440673828125,
      "KL/std": 42.441444396972656,
      "epoch": 0.3413612565445026,
      "fcm_dpo/beta": 0.012807728722691536,
      "fcm_dpo/delta": 0.02517436258494854,
      "fcm_dpo/margin": 33.129638671875,
      "fcm_dpo/q_t": 0.40372800827026367,
      "grad_norm": 84.29925537109375,
      "learning_rate": 4.1782614253949255e-07,
      "logits/chosen": -0.8831664323806763,
      "logits/rejected": -0.883603572845459,
      "logps/chosen": -321.8109130859375,
      "logps/ref_chosen": -293.4999084472656,
      "logps/ref_rejected": -266.7116394042969,
      "logps/rejected": -328.1523132324219,
      "loss": 4.4049,
      "margin_dpo/margin_mean": 33.129638671875,
      "margin_dpo/margin_std": 47.85710906982422,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -34.13151550292969,
      "KL/mean": -50.54680252075195,
      "KL/rejected_KL_mean": -66.96208953857422,
      "KL/std": 47.98040771484375,
      "epoch": 0.34345549738219894,
      "fcm_dpo/beta": 0.012849204242229462,
      "fcm_dpo/delta": -0.0024485718458890915,
      "fcm_dpo/margin": 32.83057403564453,
      "fcm_dpo/q_t": 0.40761861205101013,
      "grad_norm": 85.03250122070312,
      "learning_rate": 4.164647253573289e-07,
      "logits/chosen": -0.8475548028945923,
      "logits/rejected": -0.8656323552131653,
      "logps/chosen": -301.1810302734375,
      "logps/ref_chosen": -267.04949951171875,
      "logps/ref_rejected": -215.9768829345703,
      "logps/rejected": -282.93896484375,
      "loss": 4.4942,
      "margin_dpo/margin_mean": 32.83057403564453,
      "margin_dpo/margin_std": 53.879093170166016,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -24.416061401367188,
      "KL/mean": -36.756309509277344,
      "KL/rejected_KL_mean": -49.096561431884766,
      "KL/std": 39.948421478271484,
      "epoch": 0.34554973821989526,
      "fcm_dpo/beta": 0.01267604622989893,
      "fcm_dpo/delta": 0.021274425089359283,
      "fcm_dpo/margin": 24.680496215820312,
      "fcm_dpo/q_t": 0.4301545023918152,
      "grad_norm": 83.57882690429688,
      "learning_rate": 4.1509438117713863e-07,
      "logits/chosen": -0.8797891736030579,
      "logits/rejected": -0.8548811674118042,
      "logps/chosen": -302.4775695800781,
      "logps/ref_chosen": -278.06146240234375,
      "logps/ref_rejected": -260.4288635253906,
      "logps/rejected": -309.5254211425781,
      "loss": 4.7389,
      "margin_dpo/margin_mean": 24.68049430847168,
      "margin_dpo/margin_std": 45.41236114501953,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -22.908170700073242,
      "KL/mean": -37.26128005981445,
      "KL/rejected_KL_mean": -51.614383697509766,
      "KL/std": 41.975738525390625,
      "epoch": 0.34764397905759165,
      "fcm_dpo/beta": 0.01322500966489315,
      "fcm_dpo/delta": 0.07342680543661118,
      "fcm_dpo/margin": 28.70620346069336,
      "fcm_dpo/q_t": 0.418113112449646,
      "grad_norm": 69.62107849121094,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -0.8337739706039429,
      "logits/rejected": -0.8009495139122009,
      "logps/chosen": -298.8572082519531,
      "logps/ref_chosen": -275.9490661621094,
      "logps/ref_rejected": -232.13473510742188,
      "logps/rejected": -283.7491149902344,
      "loss": 4.6112,
      "margin_dpo/margin_mean": 28.706205368041992,
      "margin_dpo/margin_std": 51.448551177978516,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -26.39204978942871,
      "KL/mean": -41.472129821777344,
      "KL/rejected_KL_mean": -56.55219650268555,
      "KL/std": 44.509002685546875,
      "epoch": 0.34973821989528797,
      "fcm_dpo/beta": 0.013839630410075188,
      "fcm_dpo/delta": -0.0003445260226726532,
      "fcm_dpo/margin": 30.1601505279541,
      "fcm_dpo/q_t": 0.40879589319229126,
      "grad_norm": 71.26337432861328,
      "learning_rate": 4.123272062470633e-07,
      "logits/chosen": -0.8282614350318909,
      "logits/rejected": -0.8159167766571045,
      "logps/chosen": -306.9434814453125,
      "logps/ref_chosen": -280.5514221191406,
      "logps/ref_rejected": -255.2896728515625,
      "logps/rejected": -311.84185791015625,
      "loss": 4.5625,
      "margin_dpo/margin_mean": 30.1601505279541,
      "margin_dpo/margin_std": 52.742855072021484,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": -27.492712020874023,
      "KL/mean": -44.34742736816406,
      "KL/rejected_KL_mean": -61.202144622802734,
      "KL/std": 47.94118881225586,
      "epoch": 0.3518324607329843,
      "fcm_dpo/beta": 0.01329370029270649,
      "fcm_dpo/delta": -0.044873230159282684,
      "fcm_dpo/margin": 33.709434509277344,
      "fcm_dpo/q_t": 0.3987385332584381,
      "grad_norm": 179.1513214111328,
      "learning_rate": 4.1093052389237174e-07,
      "logits/chosen": -0.8270904421806335,
      "logits/rejected": -0.8042003512382507,
      "logps/chosen": -343.2909240722656,
      "logps/ref_chosen": -315.7982177734375,
      "logps/ref_rejected": -291.48406982421875,
      "logps/rejected": -352.68621826171875,
      "loss": 4.4376,
      "margin_dpo/margin_mean": 33.709434509277344,
      "margin_dpo/margin_std": 52.37152099609375,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -22.084400177001953,
      "KL/mean": -45.16008758544922,
      "KL/rejected_KL_mean": -68.23577880859375,
      "KL/std": 46.33564758300781,
      "epoch": 0.3539267015706806,
      "fcm_dpo/beta": 0.01245945319533348,
      "fcm_dpo/delta": -0.13363215327262878,
      "fcm_dpo/margin": 46.15138626098633,
      "fcm_dpo/q_t": 0.3718532919883728,
      "grad_norm": 80.84683990478516,
      "learning_rate": 4.0952521132208267e-07,
      "logits/chosen": -0.800991952419281,
      "logits/rejected": -0.813478946685791,
      "logps/chosen": -283.148681640625,
      "logps/ref_chosen": -261.06427001953125,
      "logps/ref_rejected": -235.40663146972656,
      "logps/rejected": -303.6424255371094,
      "loss": 3.9411,
      "margin_dpo/margin_mean": 46.15138626098633,
      "margin_dpo/margin_std": 49.535850524902344,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": -25.441959381103516,
      "KL/mean": -43.37663269042969,
      "KL/rejected_KL_mean": -61.311302185058594,
      "KL/std": 49.5772590637207,
      "epoch": 0.35602094240837695,
      "fcm_dpo/beta": 0.011865227483212948,
      "fcm_dpo/delta": 0.02527080476284027,
      "fcm_dpo/margin": 35.86933898925781,
      "fcm_dpo/q_t": 0.4078722298145294,
      "grad_norm": 72.24082946777344,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": -0.7917501330375671,
      "logits/rejected": -0.7938278913497925,
      "logps/chosen": -334.4092102050781,
      "logps/ref_chosen": -308.96722412109375,
      "logps/ref_rejected": -263.8466796875,
      "logps/rejected": -325.1579895019531,
      "loss": 4.5026,
      "margin_dpo/margin_mean": 35.86933898925781,
      "margin_dpo/margin_std": 61.43064880371094,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -22.703048706054688,
      "KL/mean": -44.2454833984375,
      "KL/rejected_KL_mean": -65.78791809082031,
      "KL/std": 45.76507568359375,
      "epoch": 0.3581151832460733,
      "fcm_dpo/beta": 0.011545907706022263,
      "fcm_dpo/delta": -0.09887011349201202,
      "fcm_dpo/margin": 43.084861755371094,
      "fcm_dpo/q_t": 0.3863128423690796,
      "grad_norm": 99.41549682617188,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -0.8209888339042664,
      "logits/rejected": -0.834601104259491,
      "logps/chosen": -281.59210205078125,
      "logps/ref_chosen": -258.8890380859375,
      "logps/ref_rejected": -262.19140625,
      "logps/rejected": -327.97930908203125,
      "loss": 4.1007,
      "margin_dpo/margin_mean": 43.084861755371094,
      "margin_dpo/margin_std": 46.76802062988281,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": -32.60451126098633,
      "KL/mean": -43.717533111572266,
      "KL/rejected_KL_mean": -54.83055877685547,
      "KL/std": 43.6480827331543,
      "epoch": 0.36020942408376966,
      "fcm_dpo/beta": 0.011506731621921062,
      "fcm_dpo/delta": 0.05297376587986946,
      "fcm_dpo/margin": 22.226043701171875,
      "fcm_dpo/q_t": 0.4411145746707916,
      "grad_norm": 74.37825775146484,
      "learning_rate": 4.0525824823390043e-07,
      "logits/chosen": -0.8419041037559509,
      "logits/rejected": -0.8574936389923096,
      "logps/chosen": -371.6268615722656,
      "logps/ref_chosen": -339.0223388671875,
      "logps/ref_rejected": -295.78759765625,
      "logps/rejected": -350.6181335449219,
      "loss": 4.9473,
      "margin_dpo/margin_mean": 22.226043701171875,
      "margin_dpo/margin_std": 51.92020034790039,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -35.38630676269531,
      "KL/mean": -51.06950759887695,
      "KL/rejected_KL_mean": -66.75271606445312,
      "KL/std": 46.81611633300781,
      "epoch": 0.362303664921466,
      "fcm_dpo/beta": 0.011485878378152847,
      "fcm_dpo/delta": -0.018621545284986496,
      "fcm_dpo/margin": 31.36641502380371,
      "fcm_dpo/q_t": 0.42096981406211853,
      "grad_norm": 86.6626205444336,
      "learning_rate": 4.0381917299505686e-07,
      "logits/chosen": -0.8707091212272644,
      "logits/rejected": -0.867925763130188,
      "logps/chosen": -335.49774169921875,
      "logps/ref_chosen": -300.1114501953125,
      "logps/ref_rejected": -273.78460693359375,
      "logps/rejected": -340.537353515625,
      "loss": 4.6455,
      "margin_dpo/margin_mean": 31.36641502380371,
      "margin_dpo/margin_std": 54.940711975097656,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -40.22490692138672,
      "KL/mean": -59.69932556152344,
      "KL/rejected_KL_mean": -79.17373657226562,
      "KL/std": 49.10664367675781,
      "epoch": 0.3643979057591623,
      "fcm_dpo/beta": 0.011537794955074787,
      "fcm_dpo/delta": 0.00011738482862710953,
      "fcm_dpo/margin": 38.94883728027344,
      "fcm_dpo/q_t": 0.3976377844810486,
      "grad_norm": 84.65153503417969,
      "learning_rate": 4.0237184890078243e-07,
      "logits/chosen": -0.8469346165657043,
      "logits/rejected": -0.8300192356109619,
      "logps/chosen": -375.27874755859375,
      "logps/ref_chosen": -335.0538635253906,
      "logps/ref_rejected": -257.4646911621094,
      "logps/rejected": -336.638427734375,
      "loss": 4.3152,
      "margin_dpo/margin_mean": 38.94883728027344,
      "margin_dpo/margin_std": 53.26359939575195,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -53.49272155761719,
      "KL/mean": -70.73078918457031,
      "KL/rejected_KL_mean": -87.96885681152344,
      "KL/std": 53.55436325073242,
      "epoch": 0.36649214659685864,
      "fcm_dpo/beta": 0.011354382149875164,
      "fcm_dpo/delta": -0.05507180839776993,
      "fcm_dpo/margin": 34.47612762451172,
      "fcm_dpo/q_t": 0.4168418347835541,
      "grad_norm": 104.92942810058594,
      "learning_rate": 4.00916353566676e-07,
      "logits/chosen": -0.8538421988487244,
      "logits/rejected": -0.8542951941490173,
      "logps/chosen": -337.8883361816406,
      "logps/ref_chosen": -284.39556884765625,
      "logps/ref_rejected": -283.3876647949219,
      "logps/rejected": -371.35650634765625,
      "loss": 4.6327,
      "margin_dpo/margin_mean": 34.47612762451172,
      "margin_dpo/margin_std": 61.267967224121094,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -55.18890380859375,
      "KL/mean": -69.37939453125,
      "KL/rejected_KL_mean": -83.56986999511719,
      "KL/std": 50.11482238769531,
      "epoch": 0.36858638743455496,
      "fcm_dpo/beta": 0.011205538176000118,
      "fcm_dpo/delta": 0.01238213200122118,
      "fcm_dpo/margin": 28.380977630615234,
      "fcm_dpo/q_t": 0.4288972318172455,
      "grad_norm": 77.06037139892578,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -0.8212220668792725,
      "logits/rejected": -0.8277608752250671,
      "logps/chosen": -307.001708984375,
      "logps/ref_chosen": -251.81280517578125,
      "logps/ref_rejected": -242.05328369140625,
      "logps/rejected": -325.6231689453125,
      "loss": 4.8665,
      "margin_dpo/margin_mean": 28.380977630615234,
      "margin_dpo/margin_std": 62.54559326171875,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -60.746646881103516,
      "KL/mean": -75.5807113647461,
      "KL/rejected_KL_mean": -90.41476440429688,
      "KL/std": 50.17369842529297,
      "epoch": 0.3706806282722513,
      "fcm_dpo/beta": 0.010977281257510185,
      "fcm_dpo/delta": -0.011295255273580551,
      "fcm_dpo/margin": 29.668132781982422,
      "fcm_dpo/q_t": 0.4275417625904083,
      "grad_norm": 76.1621322631836,
      "learning_rate": 3.979811618281705e-07,
      "logits/chosen": -0.9123234152793884,
      "logits/rejected": -0.8834874629974365,
      "logps/chosen": -359.39306640625,
      "logps/ref_chosen": -298.6463928222656,
      "logps/ref_rejected": -295.66534423828125,
      "logps/rejected": -386.08013916015625,
      "loss": 4.8843,
      "margin_dpo/margin_mean": 29.668132781982422,
      "margin_dpo/margin_std": 64.1531982421875,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -53.90563201904297,
      "KL/mean": -73.8095703125,
      "KL/rejected_KL_mean": -93.71351623535156,
      "KL/std": 53.449344635009766,
      "epoch": 0.37277486910994767,
      "fcm_dpo/beta": 0.011017758399248123,
      "fcm_dpo/delta": 0.011491580866277218,
      "fcm_dpo/margin": 39.807884216308594,
      "fcm_dpo/q_t": 0.40419143438339233,
      "grad_norm": 107.19924926757812,
      "learning_rate": 3.9650162282919654e-07,
      "logits/chosen": -0.8316702842712402,
      "logits/rejected": -0.8250093460083008,
      "logps/chosen": -340.163330078125,
      "logps/ref_chosen": -286.2576599121094,
      "logps/ref_rejected": -243.97491455078125,
      "logps/rejected": -337.6883850097656,
      "loss": 4.442,
      "margin_dpo/margin_mean": 39.807884216308594,
      "margin_dpo/margin_std": 62.75676727294922,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -56.6027717590332,
      "KL/mean": -75.01507568359375,
      "KL/rejected_KL_mean": -93.42736053466797,
      "KL/std": 53.532535552978516,
      "epoch": 0.374869109947644,
      "fcm_dpo/beta": 0.010806177742779255,
      "fcm_dpo/delta": -0.040453460067510605,
      "fcm_dpo/margin": 36.82459259033203,
      "fcm_dpo/q_t": 0.4110087752342224,
      "grad_norm": 88.93858337402344,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": -0.8121470808982849,
      "logits/rejected": -0.8135557174682617,
      "logps/chosen": -316.33984375,
      "logps/ref_chosen": -259.737060546875,
      "logps/ref_rejected": -277.8813171386719,
      "logps/rejected": -371.3086853027344,
      "loss": 4.5545,
      "margin_dpo/margin_mean": 36.82459259033203,
      "margin_dpo/margin_std": 60.663116455078125,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -59.500789642333984,
      "KL/mean": -78.8482437133789,
      "KL/rejected_KL_mean": -98.1956787109375,
      "KL/std": 51.27219009399414,
      "epoch": 0.3769633507853403,
      "fcm_dpo/beta": 0.01081792451441288,
      "fcm_dpo/delta": 0.0020559439435601234,
      "fcm_dpo/margin": 38.69487762451172,
      "fcm_dpo/q_t": 0.40881386399269104,
      "grad_norm": 74.5360336303711,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": -0.8298919796943665,
      "logits/rejected": -0.8545669913291931,
      "logps/chosen": -326.8096923828125,
      "logps/ref_chosen": -267.30889892578125,
      "logps/ref_rejected": -230.4376983642578,
      "logps/rejected": -328.63336181640625,
      "loss": 4.4548,
      "margin_dpo/margin_mean": 38.69487762451172,
      "margin_dpo/margin_std": 60.54326629638672,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -65.6080322265625,
      "KL/mean": -83.7745361328125,
      "KL/rejected_KL_mean": -101.9410400390625,
      "KL/std": 53.567073822021484,
      "epoch": 0.37905759162303665,
      "fcm_dpo/beta": 0.010863708332180977,
      "fcm_dpo/delta": 0.05775655806064606,
      "fcm_dpo/margin": 36.33304214477539,
      "fcm_dpo/q_t": 0.4135940670967102,
      "grad_norm": 84.6544189453125,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -0.8361238241195679,
      "logits/rejected": -0.8390278816223145,
      "logps/chosen": -366.0993957519531,
      "logps/ref_chosen": -300.49139404296875,
      "logps/ref_rejected": -278.98284912109375,
      "logps/rejected": -380.92388916015625,
      "loss": 4.598,
      "margin_dpo/margin_mean": 36.333045959472656,
      "margin_dpo/margin_std": 64.61507415771484,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -67.09426879882812,
      "KL/mean": -90.45288848876953,
      "KL/rejected_KL_mean": -113.8115234375,
      "KL/std": 52.626976013183594,
      "epoch": 0.381151832460733,
      "fcm_dpo/beta": 0.010970138013362885,
      "fcm_dpo/delta": -0.09437461197376251,
      "fcm_dpo/margin": 46.71723556518555,
      "fcm_dpo/q_t": 0.3862791955471039,
      "grad_norm": 75.15876770019531,
      "learning_rate": 3.90505702185e-07,
      "logits/chosen": -0.826999843120575,
      "logits/rejected": -0.8535282611846924,
      "logps/chosen": -346.5923767089844,
      "logps/ref_chosen": -279.4981689453125,
      "logps/ref_rejected": -263.6926574707031,
      "logps/rejected": -377.504150390625,
      "loss": 4.1942,
      "margin_dpo/margin_mean": 46.71723937988281,
      "margin_dpo/margin_std": 60.688663482666016,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -67.47649383544922,
      "KL/mean": -88.29644012451172,
      "KL/rejected_KL_mean": -109.11637115478516,
      "KL/std": 55.01540756225586,
      "epoch": 0.3832460732984293,
      "fcm_dpo/beta": 0.010420994833111763,
      "fcm_dpo/delta": 0.016744107007980347,
      "fcm_dpo/margin": 41.63987731933594,
      "fcm_dpo/q_t": 0.4054372012615204,
      "grad_norm": 78.68669891357422,
      "learning_rate": 3.889876827928156e-07,
      "logits/chosen": -0.864861249923706,
      "logits/rejected": -0.8679836392402649,
      "logps/chosen": -338.68218994140625,
      "logps/ref_chosen": -271.2057189941406,
      "logps/ref_rejected": -243.91549682617188,
      "logps/rejected": -353.0318298339844,
      "loss": 4.4626,
      "margin_dpo/margin_mean": 41.63987731933594,
      "margin_dpo/margin_std": 67.57550811767578,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -61.86859893798828,
      "KL/mean": -90.66870880126953,
      "KL/rejected_KL_mean": -119.46882629394531,
      "KL/std": 61.09760284423828,
      "epoch": 0.38534031413612563,
      "fcm_dpo/beta": 0.009740042500197887,
      "fcm_dpo/delta": -0.12229367345571518,
      "fcm_dpo/margin": 57.60023498535156,
      "fcm_dpo/q_t": 0.37497925758361816,
      "grad_norm": 64.23841094970703,
      "learning_rate": 3.874622099130087e-07,
      "logits/chosen": -0.9034750461578369,
      "logits/rejected": -0.8821806311607361,
      "logps/chosen": -380.3143615722656,
      "logps/ref_chosen": -318.4457702636719,
      "logps/ref_rejected": -266.640869140625,
      "logps/rejected": -386.1097106933594,
      "loss": 4.0518,
      "margin_dpo/margin_mean": 57.60023498535156,
      "margin_dpo/margin_std": 69.85670471191406,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -71.44171905517578,
      "KL/mean": -92.58551025390625,
      "KL/rejected_KL_mean": -113.72931671142578,
      "KL/std": 62.41941833496094,
      "epoch": 0.387434554973822,
      "fcm_dpo/beta": 0.009617242030799389,
      "fcm_dpo/delta": 0.015148327685892582,
      "fcm_dpo/margin": 42.28759765625,
      "fcm_dpo/q_t": 0.4099074900150299,
      "grad_norm": 71.3729248046875,
      "learning_rate": 3.859293653520604e-07,
      "logits/chosen": -0.8799754977226257,
      "logits/rejected": -0.874716579914093,
      "logps/chosen": -345.7505187988281,
      "logps/ref_chosen": -274.308837890625,
      "logps/ref_rejected": -260.7274169921875,
      "logps/rejected": -374.4566955566406,
      "loss": 4.484,
      "margin_dpo/margin_mean": 42.28759765625,
      "margin_dpo/margin_std": 66.53794860839844,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -68.40057373046875,
      "KL/mean": -89.52610778808594,
      "KL/rejected_KL_mean": -110.65161895751953,
      "KL/std": 57.584171295166016,
      "epoch": 0.38952879581151834,
      "fcm_dpo/beta": 0.009686894714832306,
      "fcm_dpo/delta": -0.005168679635971785,
      "fcm_dpo/margin": 42.25104522705078,
      "fcm_dpo/q_t": 0.40718692541122437,
      "grad_norm": 64.16702270507812,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -0.8893187046051025,
      "logits/rejected": -0.8921197056770325,
      "logps/chosen": -367.4059753417969,
      "logps/ref_chosen": -299.00537109375,
      "logps/ref_rejected": -274.4014587402344,
      "logps/rejected": -385.05303955078125,
      "loss": 4.4825,
      "margin_dpo/margin_mean": 42.25104904174805,
      "margin_dpo/margin_std": 64.57786560058594,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -73.64212036132812,
      "KL/mean": -91.94225311279297,
      "KL/rejected_KL_mean": -110.2424087524414,
      "KL/std": 59.46043395996094,
      "epoch": 0.39162303664921466,
      "fcm_dpo/beta": 0.010095559991896152,
      "fcm_dpo/delta": 0.08246435970067978,
      "fcm_dpo/margin": 36.600303649902344,
      "fcm_dpo/q_t": 0.41564512252807617,
      "grad_norm": 93.7572021484375,
      "learning_rate": 3.828418903848593e-07,
      "logits/chosen": -0.8366921544075012,
      "logits/rejected": -0.8246840834617615,
      "logps/chosen": -403.4673767089844,
      "logps/ref_chosen": -329.8253173828125,
      "logps/ref_rejected": -263.73175048828125,
      "logps/rejected": -373.9741516113281,
      "loss": 4.7465,
      "margin_dpo/margin_mean": 36.600303649902344,
      "margin_dpo/margin_std": 72.65327453613281,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -65.24836730957031,
      "KL/mean": -86.75069427490234,
      "KL/rejected_KL_mean": -108.25300598144531,
      "KL/std": 61.90050506591797,
      "epoch": 0.393717277486911,
      "fcm_dpo/beta": 0.010198064148426056,
      "fcm_dpo/delta": -0.015000926330685616,
      "fcm_dpo/margin": 43.004638671875,
      "fcm_dpo/q_t": 0.4053938090801239,
      "grad_norm": 64.57600402832031,
      "learning_rate": 3.812874255505191e-07,
      "logits/chosen": -0.8592841029167175,
      "logits/rejected": -0.8527074456214905,
      "logps/chosen": -328.2539978027344,
      "logps/ref_chosen": -263.005615234375,
      "logps/ref_rejected": -247.08668518066406,
      "logps/rejected": -355.33966064453125,
      "loss": 4.5873,
      "margin_dpo/margin_mean": 43.004634857177734,
      "margin_dpo/margin_std": 76.90157318115234,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -54.43677520751953,
      "KL/mean": -80.12665557861328,
      "KL/rejected_KL_mean": -105.81653594970703,
      "KL/std": 60.71977233886719,
      "epoch": 0.3958115183246073,
      "fcm_dpo/beta": 0.009668517857789993,
      "fcm_dpo/delta": -0.08323223888874054,
      "fcm_dpo/margin": 51.3797607421875,
      "fcm_dpo/q_t": 0.3885657787322998,
      "grad_norm": 73.50676727294922,
      "learning_rate": 3.797259201699833e-07,
      "logits/chosen": -0.8765964508056641,
      "logits/rejected": -0.8847308158874512,
      "logps/chosen": -327.39715576171875,
      "logps/ref_chosen": -272.96038818359375,
      "logps/ref_rejected": -275.13238525390625,
      "logps/rejected": -380.9489440917969,
      "loss": 4.1631,
      "margin_dpo/margin_mean": 51.3797607421875,
      "margin_dpo/margin_std": 62.679908752441406,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -54.90644073486328,
      "KL/mean": -76.46266174316406,
      "KL/rejected_KL_mean": -98.01889038085938,
      "KL/std": 57.6021614074707,
      "epoch": 0.39790575916230364,
      "fcm_dpo/beta": 0.009625312872231007,
      "fcm_dpo/delta": 0.0361747071146965,
      "fcm_dpo/margin": 43.112449645996094,
      "fcm_dpo/q_t": 0.4072265923023224,
      "grad_norm": 71.8258285522461,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": -0.8628578186035156,
      "logits/rejected": -0.8243663311004639,
      "logps/chosen": -312.7040100097656,
      "logps/ref_chosen": -257.79754638671875,
      "logps/ref_rejected": -225.2164306640625,
      "logps/rejected": -323.23529052734375,
      "loss": 4.4521,
      "margin_dpo/margin_mean": 43.112449645996094,
      "margin_dpo/margin_std": 65.7745590209961,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -49.148277282714844,
      "KL/mean": -71.07865905761719,
      "KL/rejected_KL_mean": -93.00904083251953,
      "KL/std": 58.52466583251953,
      "epoch": 0.4,
      "fcm_dpo/beta": 0.009782027453184128,
      "fcm_dpo/delta": -0.002552715130150318,
      "fcm_dpo/margin": 43.860774993896484,
      "fcm_dpo/q_t": 0.4050540328025818,
      "grad_norm": 67.49217987060547,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -0.8841782808303833,
      "logits/rejected": -0.8864907026290894,
      "logps/chosen": -293.00677490234375,
      "logps/ref_chosen": -243.8585205078125,
      "logps/ref_rejected": -245.12136840820312,
      "logps/rejected": -338.13043212890625,
      "loss": 4.4564,
      "margin_dpo/margin_mean": 43.860774993896484,
      "margin_dpo/margin_std": 68.90791320800781,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -57.2428092956543,
      "KL/mean": -75.18179321289062,
      "KL/rejected_KL_mean": -93.12078857421875,
      "KL/std": 55.425437927246094,
      "epoch": 0.40209424083769635,
      "fcm_dpo/beta": 0.009850271046161652,
      "fcm_dpo/delta": 0.011706957593560219,
      "fcm_dpo/margin": 35.87798309326172,
      "fcm_dpo/q_t": 0.4206148386001587,
      "grad_norm": 68.08995819091797,
      "learning_rate": 3.75e-07,
      "logits/chosen": -0.8322792053222656,
      "logits/rejected": -0.8185586333274841,
      "logps/chosen": -324.2227478027344,
      "logps/ref_chosen": -266.9799499511719,
      "logps/ref_rejected": -260.1697082519531,
      "logps/rejected": -353.2904968261719,
      "loss": 4.6712,
      "margin_dpo/margin_mean": 35.87798309326172,
      "margin_dpo/margin_std": 63.77705383300781,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -66.50089263916016,
      "KL/mean": -87.72234344482422,
      "KL/rejected_KL_mean": -108.94380187988281,
      "KL/std": 63.03485870361328,
      "epoch": 0.4041884816753927,
      "fcm_dpo/beta": 0.00970284454524517,
      "fcm_dpo/delta": -0.018693745136260986,
      "fcm_dpo/margin": 42.44292068481445,
      "fcm_dpo/q_t": 0.40921568870544434,
      "grad_norm": 73.63004302978516,
      "learning_rate": 3.734111735307796e-07,
      "logits/chosen": -0.8704419136047363,
      "logits/rejected": -0.8443101644515991,
      "logps/chosen": -346.7541198730469,
      "logps/ref_chosen": -280.25323486328125,
      "logps/ref_rejected": -291.0348815917969,
      "logps/rejected": -399.97869873046875,
      "loss": 4.5312,
      "margin_dpo/margin_mean": 42.44292068481445,
      "margin_dpo/margin_std": 69.8365707397461,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -67.01454162597656,
      "KL/mean": -84.28245544433594,
      "KL/rejected_KL_mean": -101.55038452148438,
      "KL/std": 58.918663024902344,
      "epoch": 0.406282722513089,
      "fcm_dpo/beta": 0.009924488142132759,
      "fcm_dpo/delta": 0.03909885138273239,
      "fcm_dpo/margin": 34.53584289550781,
      "fcm_dpo/q_t": 0.4241364598274231,
      "grad_norm": 76.46985626220703,
      "learning_rate": 3.7181572889485623e-07,
      "logits/chosen": -0.8477017283439636,
      "logits/rejected": -0.8394756317138672,
      "logps/chosen": -355.1540222167969,
      "logps/ref_chosen": -288.13946533203125,
      "logps/ref_rejected": -251.31529235839844,
      "logps/rejected": -352.8656921386719,
      "loss": 4.6924,
      "margin_dpo/margin_mean": 34.53583908081055,
      "margin_dpo/margin_std": 63.45747375488281,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -68.40491485595703,
      "KL/mean": -84.58940124511719,
      "KL/rejected_KL_mean": -100.77388000488281,
      "KL/std": 58.506221771240234,
      "epoch": 0.4083769633507853,
      "fcm_dpo/beta": 0.010217108763754368,
      "fcm_dpo/delta": 0.03366906940937042,
      "fcm_dpo/margin": 32.36897659301758,
      "fcm_dpo/q_t": 0.42875081300735474,
      "grad_norm": 74.3450927734375,
      "learning_rate": 3.7021375165108377e-07,
      "logits/chosen": -0.8514112830162048,
      "logits/rejected": -0.8563896417617798,
      "logps/chosen": -342.40557861328125,
      "logps/ref_chosen": -274.0006408691406,
      "logps/ref_rejected": -280.22723388671875,
      "logps/rejected": -381.0011291503906,
      "loss": 4.764,
      "margin_dpo/margin_mean": 32.36897659301758,
      "margin_dpo/margin_std": 64.095703125,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -59.28919982910156,
      "KL/mean": -80.04025268554688,
      "KL/rejected_KL_mean": -100.79130554199219,
      "KL/std": 59.27418518066406,
      "epoch": 0.41047120418848165,
      "fcm_dpo/beta": 0.010211086831986904,
      "fcm_dpo/delta": -0.04277971759438515,
      "fcm_dpo/margin": 41.502105712890625,
      "fcm_dpo/q_t": 0.4065576195716858,
      "grad_norm": 89.31893157958984,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -0.8397841453552246,
      "logits/rejected": -0.8489161729812622,
      "logps/chosen": -334.18988037109375,
      "logps/ref_chosen": -274.90069580078125,
      "logps/ref_rejected": -248.7281951904297,
      "logps/rejected": -349.5195007324219,
      "loss": 4.4928,
      "margin_dpo/margin_mean": 41.50210189819336,
      "margin_dpo/margin_std": 68.57027435302734,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -60.77907943725586,
      "KL/mean": -86.62120056152344,
      "KL/rejected_KL_mean": -112.46331024169922,
      "KL/std": 62.46781921386719,
      "epoch": 0.41256544502617803,
      "fcm_dpo/beta": 0.009665084071457386,
      "fcm_dpo/delta": -0.09250638633966446,
      "fcm_dpo/margin": 51.68422317504883,
      "fcm_dpo/q_t": 0.3887063264846802,
      "grad_norm": 76.1441879272461,
      "learning_rate": 3.6699054332241985e-07,
      "logits/chosen": -0.873895525932312,
      "logits/rejected": -0.8611553907394409,
      "logps/chosen": -370.31390380859375,
      "logps/ref_chosen": -309.5348205566406,
      "logps/ref_rejected": -264.3179931640625,
      "logps/rejected": -376.7812805175781,
      "loss": 4.2031,
      "margin_dpo/margin_mean": 51.68421936035156,
      "margin_dpo/margin_std": 64.98551940917969,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -60.24945831298828,
      "KL/mean": -85.34254455566406,
      "KL/rejected_KL_mean": -110.43563842773438,
      "KL/std": 64.69019317626953,
      "epoch": 0.41465968586387436,
      "fcm_dpo/beta": 0.009018555283546448,
      "fcm_dpo/delta": -0.0340389683842659,
      "fcm_dpo/margin": 50.1861686706543,
      "fcm_dpo/q_t": 0.3996533751487732,
      "grad_norm": 81.87539672851562,
      "learning_rate": 3.653694850884091e-07,
      "logits/chosen": -0.8749064803123474,
      "logits/rejected": -0.8507733345031738,
      "logps/chosen": -361.2629089355469,
      "logps/ref_chosen": -301.0134582519531,
      "logps/ref_rejected": -292.84185791015625,
      "logps/rejected": -403.2774658203125,
      "loss": 4.3526,
      "margin_dpo/margin_mean": 50.1861686706543,
      "margin_dpo/margin_std": 72.89933776855469,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -65.08032989501953,
      "KL/mean": -87.90782928466797,
      "KL/rejected_KL_mean": -110.73532104492188,
      "KL/std": 60.86561584472656,
      "epoch": 0.4167539267015707,
      "fcm_dpo/beta": 0.009127413854002953,
      "fcm_dpo/delta": -0.0032155942171812057,
      "fcm_dpo/margin": 45.65498352050781,
      "fcm_dpo/q_t": 0.40774354338645935,
      "grad_norm": 69.87248992919922,
      "learning_rate": 3.6374223993904124e-07,
      "logits/chosen": -0.8552732467651367,
      "logits/rejected": -0.8153829574584961,
      "logps/chosen": -329.6861877441406,
      "logps/ref_chosen": -264.6058654785156,
      "logps/ref_rejected": -214.9014892578125,
      "logps/rejected": -325.63677978515625,
      "loss": 4.4771,
      "margin_dpo/margin_mean": 45.65498352050781,
      "margin_dpo/margin_std": 72.94841003417969,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -72.34476470947266,
      "KL/mean": -92.62354278564453,
      "KL/rejected_KL_mean": -112.90231323242188,
      "KL/std": 65.0808334350586,
      "epoch": 0.418848167539267,
      "fcm_dpo/beta": 0.009073810651898384,
      "fcm_dpo/delta": 0.03890574350953102,
      "fcm_dpo/margin": 40.557559967041016,
      "fcm_dpo/q_t": 0.4206618368625641,
      "grad_norm": 84.76322174072266,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": -0.9047858715057373,
      "logits/rejected": -0.882804811000824,
      "logps/chosen": -396.5035400390625,
      "logps/ref_chosen": -324.1588134765625,
      "logps/ref_rejected": -277.80218505859375,
      "logps/rejected": -390.7044982910156,
      "loss": 4.7729,
      "margin_dpo/margin_mean": 40.557559967041016,
      "margin_dpo/margin_std": 82.80643463134766,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_KL/chosen_KL_mean": -78.68721008300781,
      "eval_KL/mean": -101.52086639404297,
      "eval_KL/rejected_KL_mean": -124.35453033447266,
      "eval_KL/std": 64.70138549804688,
      "eval_fcm_dpo/beta": 0.009299466386437416,
      "eval_logits/chosen": -0.8782966136932373,
      "eval_logits/rejected": -0.8629190921783447,
      "eval_logps/chosen": -366.51397705078125,
      "eval_logps/ref_chosen": -287.8268127441406,
      "eval_logps/ref_rejected": -266.9300231933594,
      "eval_logps/rejected": -391.2845764160156,
      "eval_loss": 0.564861536026001,
      "eval_margin_dpo/margin_mean": 45.667320251464844,
      "eval_margin_dpo/margin_std": 71.59063720703125,
      "eval_runtime": 78.7771,
      "eval_samples_per_second": 25.388,
      "eval_steps_per_second": 3.174,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -79.14714050292969,
      "KL/mean": -103.06765747070312,
      "KL/rejected_KL_mean": -126.98815155029297,
      "KL/std": 66.84412384033203,
      "epoch": 0.42094240837696334,
      "fcm_dpo/beta": 0.009224808774888515,
      "fcm_dpo/delta": -0.04461335390806198,
      "fcm_dpo/margin": 47.841033935546875,
      "fcm_dpo/q_t": 0.4025250971317291,
      "grad_norm": 93.6387939453125,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -0.8715479373931885,
      "logits/rejected": -0.85959392786026,
      "logps/chosen": -350.642822265625,
      "logps/ref_chosen": -271.49566650390625,
      "logps/ref_rejected": -245.71414184570312,
      "logps/rejected": -372.7023010253906,
      "loss": 4.4768,
      "margin_dpo/margin_mean": 47.841033935546875,
      "margin_dpo/margin_std": 76.881103515625,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -79.991455078125,
      "KL/mean": -103.26058197021484,
      "KL/rejected_KL_mean": -126.52972412109375,
      "KL/std": 64.5134506225586,
      "epoch": 0.42303664921465967,
      "fcm_dpo/beta": 0.008786546997725964,
      "fcm_dpo/delta": -0.03564701974391937,
      "fcm_dpo/margin": 46.53825378417969,
      "fcm_dpo/q_t": 0.40920382738113403,
      "grad_norm": 75.6667251586914,
      "learning_rate": 3.588242572718162e-07,
      "logits/chosen": -0.8820033073425293,
      "logits/rejected": -0.8704773783683777,
      "logps/chosen": -352.08941650390625,
      "logps/ref_chosen": -272.0979309082031,
      "logps/ref_rejected": -235.94805908203125,
      "logps/rejected": -362.4778137207031,
      "loss": 4.5454,
      "margin_dpo/margin_mean": 46.53825759887695,
      "margin_dpo/margin_std": 78.24699401855469,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -90.44757843017578,
      "KL/mean": -107.79891204833984,
      "KL/rejected_KL_mean": -125.15023040771484,
      "KL/std": 63.41851043701172,
      "epoch": 0.42513089005235605,
      "fcm_dpo/beta": 0.008778889663517475,
      "fcm_dpo/delta": 0.024683255702257156,
      "fcm_dpo/margin": 34.70266342163086,
      "fcm_dpo/q_t": 0.4303639233112335,
      "grad_norm": 97.3357162475586,
      "learning_rate": 3.571731403507635e-07,
      "logits/chosen": -0.8611288070678711,
      "logits/rejected": -0.8692326545715332,
      "logps/chosen": -370.669677734375,
      "logps/ref_chosen": -280.2221374511719,
      "logps/ref_rejected": -251.79798889160156,
      "logps/rejected": -376.9482116699219,
      "loss": 4.7743,
      "margin_dpo/margin_mean": 34.70266342163086,
      "margin_dpo/margin_std": 67.28954315185547,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -82.49101257324219,
      "KL/mean": -108.38375091552734,
      "KL/rejected_KL_mean": -134.27651977539062,
      "KL/std": 65.71077728271484,
      "epoch": 0.4272251308900524,
      "fcm_dpo/beta": 0.008794787339866161,
      "fcm_dpo/delta": -0.006122041493654251,
      "fcm_dpo/margin": 51.78548812866211,
      "fcm_dpo/q_t": 0.39820095896720886,
      "grad_norm": 86.19092559814453,
      "learning_rate": 3.5551627605944746e-07,
      "logits/chosen": -0.9122729897499084,
      "logits/rejected": -0.8871258497238159,
      "logps/chosen": -401.28704833984375,
      "logps/ref_chosen": -318.7960510253906,
      "logps/ref_rejected": -269.69921875,
      "logps/rejected": -403.97576904296875,
      "loss": 4.3224,
      "margin_dpo/margin_mean": 51.78548812866211,
      "margin_dpo/margin_std": 73.36933135986328,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -85.63226318359375,
      "KL/mean": -113.24519348144531,
      "KL/rejected_KL_mean": -140.85812377929688,
      "KL/std": 69.36121368408203,
      "epoch": 0.4293193717277487,
      "fcm_dpo/beta": 0.008790346793830395,
      "fcm_dpo/delta": -0.03806339204311371,
      "fcm_dpo/margin": 55.225860595703125,
      "fcm_dpo/q_t": 0.3941951394081116,
      "grad_norm": 85.52545166015625,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": -0.8519253730773926,
      "logits/rejected": -0.8198663592338562,
      "logps/chosen": -369.39434814453125,
      "logps/ref_chosen": -283.7620544433594,
      "logps/ref_rejected": -297.69439697265625,
      "logps/rejected": -438.55255126953125,
      "loss": 4.3018,
      "margin_dpo/margin_mean": 55.225860595703125,
      "margin_dpo/margin_std": 79.28327941894531,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -92.6011962890625,
      "KL/mean": -114.87051391601562,
      "KL/rejected_KL_mean": -137.1398162841797,
      "KL/std": 69.04804992675781,
      "epoch": 0.431413612565445,
      "fcm_dpo/beta": 0.008483370766043663,
      "fcm_dpo/delta": -0.005092136561870575,
      "fcm_dpo/margin": 44.53862380981445,
      "fcm_dpo/q_t": 0.4163409173488617,
      "grad_norm": 79.53068542480469,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -0.8874562978744507,
      "logits/rejected": -0.9088807106018066,
      "logps/chosen": -386.2650451660156,
      "logps/ref_chosen": -293.66387939453125,
      "logps/ref_rejected": -291.3056640625,
      "logps/rejected": -428.44549560546875,
      "loss": 4.6649,
      "margin_dpo/margin_mean": 44.53862380981445,
      "margin_dpo/margin_std": 81.93328094482422,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -85.2120590209961,
      "KL/mean": -107.74081420898438,
      "KL/rejected_KL_mean": -130.26959228515625,
      "KL/std": 75.20499420166016,
      "epoch": 0.43350785340314135,
      "fcm_dpo/beta": 0.008478044532239437,
      "fcm_dpo/delta": -0.0054905530996620655,
      "fcm_dpo/margin": 45.057533264160156,
      "fcm_dpo/q_t": 0.4155900478363037,
      "grad_norm": 88.462158203125,
      "learning_rate": 3.505120890024195e-07,
      "logits/chosen": -0.8253107070922852,
      "logits/rejected": -0.8333117365837097,
      "logps/chosen": -355.7471618652344,
      "logps/ref_chosen": -270.5350646972656,
      "logps/ref_rejected": -278.7747497558594,
      "logps/rejected": -409.0443420410156,
      "loss": 4.6867,
      "margin_dpo/margin_mean": 45.057533264160156,
      "margin_dpo/margin_std": 87.40779113769531,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -91.10249328613281,
      "KL/mean": -119.2254409790039,
      "KL/rejected_KL_mean": -147.348388671875,
      "KL/std": 74.38504791259766,
      "epoch": 0.4356020942408377,
      "fcm_dpo/beta": 0.008469990454614162,
      "fcm_dpo/delta": -0.02837543934583664,
      "fcm_dpo/margin": 56.245914459228516,
      "fcm_dpo/q_t": 0.39643940329551697,
      "grad_norm": 67.61432647705078,
      "learning_rate": 3.4883312676665534e-07,
      "logits/chosen": -0.8974396586418152,
      "logits/rejected": -0.849094569683075,
      "logps/chosen": -370.6852722167969,
      "logps/ref_chosen": -279.582763671875,
      "logps/ref_rejected": -290.041015625,
      "logps/rejected": -437.389404296875,
      "loss": 4.3014,
      "margin_dpo/margin_mean": 56.245914459228516,
      "margin_dpo/margin_std": 81.291748046875,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -95.63153076171875,
      "KL/mean": -115.15742492675781,
      "KL/rejected_KL_mean": -134.68331909179688,
      "KL/std": 72.36209869384766,
      "epoch": 0.437696335078534,
      "fcm_dpo/beta": 0.008484529331326485,
      "fcm_dpo/delta": 0.048819221556186676,
      "fcm_dpo/margin": 39.05179214477539,
      "fcm_dpo/q_t": 0.4259859621524811,
      "grad_norm": 87.5654067993164,
      "learning_rate": 3.4714886441024573e-07,
      "logits/chosen": -0.801007091999054,
      "logits/rejected": -0.7979186773300171,
      "logps/chosen": -414.5040283203125,
      "logps/ref_chosen": -318.8725280761719,
      "logps/ref_rejected": -270.64324951171875,
      "logps/rejected": -405.3265686035156,
      "loss": 4.818,
      "margin_dpo/margin_mean": 39.051788330078125,
      "margin_dpo/margin_std": 82.47584533691406,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -86.31488037109375,
      "KL/mean": -111.37737274169922,
      "KL/rejected_KL_mean": -136.43988037109375,
      "KL/std": 67.89924621582031,
      "epoch": 0.4397905759162304,
      "fcm_dpo/beta": 0.008412555791437626,
      "fcm_dpo/delta": -0.03865387663245201,
      "fcm_dpo/margin": 50.12498092651367,
      "fcm_dpo/q_t": 0.40592333674430847,
      "grad_norm": 89.16227722167969,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": -0.8425436615943909,
      "logits/rejected": -0.8308712244033813,
      "logps/chosen": -369.4552001953125,
      "logps/ref_chosen": -283.14031982421875,
      "logps/ref_rejected": -287.2986755371094,
      "logps/rejected": -423.7385559082031,
      "loss": 4.4544,
      "margin_dpo/margin_mean": 50.124977111816406,
      "margin_dpo/margin_std": 77.4962158203125,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -77.87045288085938,
      "KL/mean": -108.02945709228516,
      "KL/rejected_KL_mean": -138.1884765625,
      "KL/std": 70.27757263183594,
      "epoch": 0.4418848167539267,
      "fcm_dpo/beta": 0.008026684634387493,
      "fcm_dpo/delta": -0.07496143877506256,
      "fcm_dpo/margin": 60.318023681640625,
      "fcm_dpo/q_t": 0.3887871503829956,
      "grad_norm": 70.60808563232422,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -0.8669252991676331,
      "logits/rejected": -0.8451690673828125,
      "logps/chosen": -354.2933654785156,
      "logps/ref_chosen": -276.4228515625,
      "logps/ref_rejected": -252.40603637695312,
      "logps/rejected": -390.59454345703125,
      "loss": 4.1549,
      "margin_dpo/margin_mean": 60.318023681640625,
      "margin_dpo/margin_std": 68.43733215332031,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -86.872802734375,
      "KL/mean": -109.01505279541016,
      "KL/rejected_KL_mean": -131.15731811523438,
      "KL/std": 68.31483459472656,
      "epoch": 0.44397905759162304,
      "fcm_dpo/beta": 0.0080754105001688,
      "fcm_dpo/delta": 0.03572115674614906,
      "fcm_dpo/margin": 44.28451919555664,
      "fcm_dpo/q_t": 0.417788565158844,
      "grad_norm": 97.84551239013672,
      "learning_rate": 3.4206518122800055e-07,
      "logits/chosen": -0.8510583639144897,
      "logits/rejected": -0.8505109548568726,
      "logps/chosen": -358.57830810546875,
      "logps/ref_chosen": -271.7055358886719,
      "logps/ref_rejected": -241.18511962890625,
      "logps/rejected": -372.3424987792969,
      "loss": 4.6176,
      "margin_dpo/margin_mean": 44.28451919555664,
      "margin_dpo/margin_std": 74.0612564086914,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -86.41748809814453,
      "KL/mean": -109.79473876953125,
      "KL/rejected_KL_mean": -133.17198181152344,
      "KL/std": 77.1285629272461,
      "epoch": 0.44607329842931936,
      "fcm_dpo/beta": 0.008197562769055367,
      "fcm_dpo/delta": 0.022071676328778267,
      "fcm_dpo/margin": 46.75450897216797,
      "fcm_dpo/q_t": 0.4164155125617981,
      "grad_norm": 114.4952163696289,
      "learning_rate": 3.403606243773448e-07,
      "logits/chosen": -0.8412494659423828,
      "logits/rejected": -0.8560968637466431,
      "logps/chosen": -388.71514892578125,
      "logps/ref_chosen": -302.2976379394531,
      "logps/ref_rejected": -303.6202087402344,
      "logps/rejected": -436.7921447753906,
      "loss": 4.6167,
      "margin_dpo/margin_mean": 46.7545051574707,
      "margin_dpo/margin_std": 83.95932006835938,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -101.88623046875,
      "KL/mean": -128.86212158203125,
      "KL/rejected_KL_mean": -155.8380126953125,
      "KL/std": 69.71839141845703,
      "epoch": 0.4481675392670157,
      "fcm_dpo/beta": 0.008262883871793747,
      "fcm_dpo/delta": 0.0022384002804756165,
      "fcm_dpo/margin": 53.95178985595703,
      "fcm_dpo/q_t": 0.39985811710357666,
      "grad_norm": 93.7610092163086,
      "learning_rate": 3.3865122176063385e-07,
      "logits/chosen": -0.8471960425376892,
      "logits/rejected": -0.8478716015815735,
      "logps/chosen": -374.01885986328125,
      "logps/ref_chosen": -272.13262939453125,
      "logps/ref_rejected": -294.82354736328125,
      "logps/rejected": -450.6615295410156,
      "loss": 4.3196,
      "margin_dpo/margin_mean": 53.95178985595703,
      "margin_dpo/margin_std": 70.07292175292969,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -97.45832824707031,
      "KL/mean": -118.72871398925781,
      "KL/rejected_KL_mean": -139.99908447265625,
      "KL/std": 78.48653411865234,
      "epoch": 0.450261780104712,
      "fcm_dpo/beta": 0.008306055329740047,
      "fcm_dpo/delta": 0.003920149058103561,
      "fcm_dpo/margin": 42.54074478149414,
      "fcm_dpo/q_t": 0.4239475727081299,
      "grad_norm": 75.21717834472656,
      "learning_rate": 3.3693706504794243e-07,
      "logits/chosen": -0.8880847692489624,
      "logits/rejected": -0.8689481616020203,
      "logps/chosen": -388.836669921875,
      "logps/ref_chosen": -291.3782958984375,
      "logps/ref_rejected": -261.05792236328125,
      "logps/rejected": -401.0570068359375,
      "loss": 4.7426,
      "margin_dpo/margin_mean": 42.540748596191406,
      "margin_dpo/margin_std": 82.39286041259766,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -92.51470947265625,
      "KL/mean": -117.88108825683594,
      "KL/rejected_KL_mean": -143.24746704101562,
      "KL/std": 74.65367889404297,
      "epoch": 0.4523560209424084,
      "fcm_dpo/beta": 0.008165874518454075,
      "fcm_dpo/delta": -0.00769144669175148,
      "fcm_dpo/margin": 50.73276901245117,
      "fcm_dpo/q_t": 0.40882280468940735,
      "grad_norm": 81.9669418334961,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -0.9206624031066895,
      "logits/rejected": -0.9146580696105957,
      "logps/chosen": -431.0201110839844,
      "logps/ref_chosen": -338.50543212890625,
      "logps/ref_rejected": -305.76104736328125,
      "logps/rejected": -449.00848388671875,
      "loss": 4.575,
      "margin_dpo/margin_mean": 50.73276901245117,
      "margin_dpo/margin_std": 88.36000061035156,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -93.37016296386719,
      "KL/mean": -125.15242004394531,
      "KL/rejected_KL_mean": -156.93467712402344,
      "KL/std": 75.62151336669922,
      "epoch": 0.4544502617801047,
      "fcm_dpo/beta": 0.00799505040049553,
      "fcm_dpo/delta": -0.06237509846687317,
      "fcm_dpo/margin": 63.564537048339844,
      "fcm_dpo/q_t": 0.3884640336036682,
      "grad_norm": 79.78565216064453,
      "learning_rate": 3.334948572847253e-07,
      "logits/chosen": -0.8152042031288147,
      "logits/rejected": -0.7791997790336609,
      "logps/chosen": -386.9199523925781,
      "logps/ref_chosen": -293.5498046875,
      "logps/ref_rejected": -256.7830810546875,
      "logps/rejected": -413.7177734375,
      "loss": 4.2191,
      "margin_dpo/margin_mean": 63.564537048339844,
      "margin_dpo/margin_std": 86.10081481933594,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -90.66104888916016,
      "KL/mean": -118.52891540527344,
      "KL/rejected_KL_mean": -146.39678955078125,
      "KL/std": 75.47489166259766,
      "epoch": 0.45654450261780105,
      "fcm_dpo/beta": 0.007838774472475052,
      "fcm_dpo/delta": 0.012707204557955265,
      "fcm_dpo/margin": 55.73572540283203,
      "fcm_dpo/q_t": 0.4026331305503845,
      "grad_norm": 73.84992218017578,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": -0.8508147597312927,
      "logits/rejected": -0.87086021900177,
      "logps/chosen": -411.2403869628906,
      "logps/ref_chosen": -320.579345703125,
      "logps/ref_rejected": -294.0381164550781,
      "logps/rejected": -440.4349060058594,
      "loss": 4.3899,
      "margin_dpo/margin_mean": 55.735721588134766,
      "margin_dpo/margin_std": 82.4550552368164,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -86.11782836914062,
      "KL/mean": -114.6595458984375,
      "KL/rejected_KL_mean": -143.20126342773438,
      "KL/std": 71.78998565673828,
      "epoch": 0.4586387434554974,
      "fcm_dpo/beta": 0.007814719341695309,
      "fcm_dpo/delta": -0.04120901972055435,
      "fcm_dpo/margin": 57.08345413208008,
      "fcm_dpo/q_t": 0.4004015326499939,
      "grad_norm": 87.77101135253906,
      "learning_rate": 3.300347394584172e-07,
      "logits/chosen": -0.8301923871040344,
      "logits/rejected": -0.8554092645645142,
      "logps/chosen": -354.5364685058594,
      "logps/ref_chosen": -268.4186096191406,
      "logps/ref_rejected": -265.7808837890625,
      "logps/rejected": -408.98211669921875,
      "loss": 4.3993,
      "margin_dpo/margin_mean": 57.08345413208008,
      "margin_dpo/margin_std": 84.8154525756836,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -86.95809936523438,
      "KL/mean": -117.26864624023438,
      "KL/rejected_KL_mean": -147.57919311523438,
      "KL/std": 71.51134490966797,
      "epoch": 0.4607329842931937,
      "fcm_dpo/beta": 0.00759973144158721,
      "fcm_dpo/delta": -0.011871591210365295,
      "fcm_dpo/margin": 60.62110900878906,
      "fcm_dpo/q_t": 0.39555221796035767,
      "grad_norm": 66.27194213867188,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": -0.8920990824699402,
      "logits/rejected": -0.8657739758491516,
      "logps/chosen": -399.84454345703125,
      "logps/ref_chosen": -312.8864440917969,
      "logps/ref_rejected": -259.5191955566406,
      "logps/rejected": -407.0984191894531,
      "loss": 4.2828,
      "margin_dpo/margin_mean": 60.62110900878906,
      "margin_dpo/margin_std": 80.68228912353516,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -93.1514663696289,
      "KL/mean": -117.5575942993164,
      "KL/rejected_KL_mean": -141.96372985839844,
      "KL/std": 74.3085708618164,
      "epoch": 0.46282722513089003,
      "fcm_dpo/beta": 0.007745365612208843,
      "fcm_dpo/delta": -0.005188461393117905,
      "fcm_dpo/margin": 48.81223678588867,
      "fcm_dpo/q_t": 0.4172177016735077,
      "grad_norm": 83.33233642578125,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -0.8098776936531067,
      "logits/rejected": -0.8209348320960999,
      "logps/chosen": -393.4773254394531,
      "logps/ref_chosen": -300.32586669921875,
      "logps/ref_rejected": -286.312255859375,
      "logps/rejected": -428.27593994140625,
      "loss": 4.631,
      "margin_dpo/margin_mean": 48.81223678588867,
      "margin_dpo/margin_std": 87.1366195678711,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -90.11768341064453,
      "KL/mean": -116.3790283203125,
      "KL/rejected_KL_mean": -142.640380859375,
      "KL/std": 67.91397857666016,
      "epoch": 0.4649214659685864,
      "fcm_dpo/beta": 0.0075710914097726345,
      "fcm_dpo/delta": 0.021815583109855652,
      "fcm_dpo/margin": 52.52268981933594,
      "fcm_dpo/q_t": 0.4095645546913147,
      "grad_norm": 78.75975799560547,
      "learning_rate": 3.248126059518784e-07,
      "logits/chosen": -0.9023758769035339,
      "logits/rejected": -0.8792685270309448,
      "logps/chosen": -387.2290344238281,
      "logps/ref_chosen": -297.1113586425781,
      "logps/ref_rejected": -235.53146362304688,
      "logps/rejected": -378.1718444824219,
      "loss": 4.4566,
      "margin_dpo/margin_mean": 52.52268981933594,
      "margin_dpo/margin_std": 76.39035034179688,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -95.87744140625,
      "KL/mean": -122.85653686523438,
      "KL/rejected_KL_mean": -149.83566284179688,
      "KL/std": 73.8615493774414,
      "epoch": 0.46701570680628274,
      "fcm_dpo/beta": 0.007735181599855423,
      "fcm_dpo/delta": 0.0016023339703679085,
      "fcm_dpo/margin": 53.95820999145508,
      "fcm_dpo/q_t": 0.4059659242630005,
      "grad_norm": 83.75157928466797,
      "learning_rate": 3.230637461492043e-07,
      "logits/chosen": -0.8332967758178711,
      "logits/rejected": -0.8010643124580383,
      "logps/chosen": -382.29254150390625,
      "logps/ref_chosen": -286.41510009765625,
      "logps/ref_rejected": -241.1181640625,
      "logps/rejected": -390.95379638671875,
      "loss": 4.4586,
      "margin_dpo/margin_mean": 53.95820999145508,
      "margin_dpo/margin_std": 82.53077697753906,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -87.61356353759766,
      "KL/mean": -118.28937530517578,
      "KL/rejected_KL_mean": -148.96517944335938,
      "KL/std": 72.82476043701172,
      "epoch": 0.46910994764397906,
      "fcm_dpo/beta": 0.007617848459631205,
      "fcm_dpo/delta": -0.059060439467430115,
      "fcm_dpo/margin": 61.35163879394531,
      "fcm_dpo/q_t": 0.3955758213996887,
      "grad_norm": 84.39550018310547,
      "learning_rate": 3.213109681595612e-07,
      "logits/chosen": -0.8018732070922852,
      "logits/rejected": -0.8176414966583252,
      "logps/chosen": -337.10589599609375,
      "logps/ref_chosen": -249.49234008789062,
      "logps/ref_rejected": -233.10752868652344,
      "logps/rejected": -382.07275390625,
      "loss": 4.2637,
      "margin_dpo/margin_mean": 61.35163116455078,
      "margin_dpo/margin_std": 80.01378631591797,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -104.31511688232422,
      "KL/mean": -130.7264862060547,
      "KL/rejected_KL_mean": -157.13787841796875,
      "KL/std": 80.04324340820312,
      "epoch": 0.4712041884816754,
      "fcm_dpo/beta": 0.007368174381554127,
      "fcm_dpo/delta": 0.02769225277006626,
      "fcm_dpo/margin": 52.822750091552734,
      "fcm_dpo/q_t": 0.4149695634841919,
      "grad_norm": 99.06048583984375,
      "learning_rate": 3.1955436597911315e-07,
      "logits/chosen": -0.8439582586288452,
      "logits/rejected": -0.8197477459907532,
      "logps/chosen": -416.1734924316406,
      "logps/ref_chosen": -311.8583679199219,
      "logps/ref_rejected": -336.8523864746094,
      "logps/rejected": -493.9902648925781,
      "loss": 4.5916,
      "margin_dpo/margin_mean": 52.82275390625,
      "margin_dpo/margin_std": 92.71817779541016,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -95.63789367675781,
      "KL/mean": -119.00685119628906,
      "KL/rejected_KL_mean": -142.37582397460938,
      "KL/std": 75.05003356933594,
      "epoch": 0.4732984293193717,
      "fcm_dpo/beta": 0.0077144792303442955,
      "fcm_dpo/delta": 0.04893975704908371,
      "fcm_dpo/margin": 46.7379264831543,
      "fcm_dpo/q_t": 0.42095378041267395,
      "grad_norm": 93.10845184326172,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -0.869759202003479,
      "logits/rejected": -0.8595449924468994,
      "logps/chosen": -347.83917236328125,
      "logps/ref_chosen": -252.20123291015625,
      "logps/ref_rejected": -254.41162109375,
      "logps/rejected": -396.78741455078125,
      "loss": 4.6597,
      "margin_dpo/margin_mean": 46.7379264831543,
      "margin_dpo/margin_std": 86.7938461303711,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -92.13594055175781,
      "KL/mean": -122.5858154296875,
      "KL/rejected_KL_mean": -153.03567504882812,
      "KL/std": 73.54113006591797,
      "epoch": 0.47539267015706804,
      "fcm_dpo/beta": 0.0077185663394629955,
      "fcm_dpo/delta": -0.02202037163078785,
      "fcm_dpo/margin": 60.89973831176758,
      "fcm_dpo/q_t": 0.39498773217201233,
      "grad_norm": 83.64966583251953,
      "learning_rate": 3.160300660508064e-07,
      "logits/chosen": -0.8317752480506897,
      "logits/rejected": -0.8278069496154785,
      "logps/chosen": -377.3953857421875,
      "logps/ref_chosen": -285.25946044921875,
      "logps/ref_rejected": -261.3220520019531,
      "logps/rejected": -414.3576965332031,
      "loss": 4.3443,
      "margin_dpo/margin_mean": 60.89973831176758,
      "margin_dpo/margin_std": 89.26444244384766,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -83.26864624023438,
      "KL/mean": -114.1445083618164,
      "KL/rejected_KL_mean": -145.02035522460938,
      "KL/std": 72.33168029785156,
      "epoch": 0.4774869109947644,
      "fcm_dpo/beta": 0.007702453061938286,
      "fcm_dpo/delta": -0.027863051742315292,
      "fcm_dpo/margin": 61.751712799072266,
      "fcm_dpo/q_t": 0.39441847801208496,
      "grad_norm": 83.65489959716797,
      "learning_rate": 3.1426255730045695e-07,
      "logits/chosen": -0.8527706861495972,
      "logits/rejected": -0.8191694617271423,
      "logps/chosen": -397.08746337890625,
      "logps/ref_chosen": -313.81878662109375,
      "logps/ref_rejected": -258.07061767578125,
      "logps/rejected": -403.09100341796875,
      "loss": 4.261,
      "margin_dpo/margin_mean": 61.75171661376953,
      "margin_dpo/margin_std": 81.5987777709961,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -94.83325958251953,
      "KL/mean": -129.89315795898438,
      "KL/rejected_KL_mean": -164.9530792236328,
      "KL/std": 77.78330993652344,
      "epoch": 0.47958115183246075,
      "fcm_dpo/beta": 0.007329674903303385,
      "fcm_dpo/delta": -0.06856437027454376,
      "fcm_dpo/margin": 70.11983489990234,
      "fcm_dpo/q_t": 0.38379472494125366,
      "grad_norm": 93.06826782226562,
      "learning_rate": 3.1249160234418644e-07,
      "logits/chosen": -0.8340184092521667,
      "logits/rejected": -0.8422746062278748,
      "logps/chosen": -386.80401611328125,
      "logps/ref_chosen": -291.9707946777344,
      "logps/ref_rejected": -263.42059326171875,
      "logps/rejected": -428.3736572265625,
      "loss": 4.1763,
      "margin_dpo/margin_mean": 70.11983489990234,
      "margin_dpo/margin_std": 88.9627685546875,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -91.6850357055664,
      "KL/mean": -123.18827819824219,
      "KL/rejected_KL_mean": -154.6915283203125,
      "KL/std": 80.88753509521484,
      "epoch": 0.4816753926701571,
      "fcm_dpo/beta": 0.006949055008590221,
      "fcm_dpo/delta": -0.025295164436101913,
      "fcm_dpo/margin": 63.006465911865234,
      "fcm_dpo/q_t": 0.4024781584739685,
      "grad_norm": 106.9558334350586,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": -0.8891492486000061,
      "logits/rejected": -0.8887955546379089,
      "logps/chosen": -324.9451599121094,
      "logps/ref_chosen": -233.2601318359375,
      "logps/ref_rejected": -238.922119140625,
      "logps/rejected": -393.6136169433594,
      "loss": 4.3941,
      "margin_dpo/margin_mean": 63.006465911865234,
      "margin_dpo/margin_std": 92.54320526123047,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -103.29402160644531,
      "KL/mean": -127.58589935302734,
      "KL/rejected_KL_mean": -151.87777709960938,
      "KL/std": 73.30841064453125,
      "epoch": 0.4837696335078534,
      "fcm_dpo/beta": 0.0071630934253335,
      "fcm_dpo/delta": 0.05592847988009453,
      "fcm_dpo/margin": 48.58376693725586,
      "fcm_dpo/q_t": 0.4209722876548767,
      "grad_norm": 79.0353012084961,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -0.855336606502533,
      "logits/rejected": -0.8427362442016602,
      "logps/chosen": -425.4491882324219,
      "logps/ref_chosen": -322.1551818847656,
      "logps/ref_rejected": -280.97613525390625,
      "logps/rejected": -432.85394287109375,
      "loss": 4.6309,
      "margin_dpo/margin_mean": 48.583763122558594,
      "margin_dpo/margin_std": 82.25651550292969,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -99.70679473876953,
      "KL/mean": -127.00975036621094,
      "KL/rejected_KL_mean": -154.3126983642578,
      "KL/std": 78.97307586669922,
      "epoch": 0.48586387434554973,
      "fcm_dpo/beta": 0.007290012203156948,
      "fcm_dpo/delta": -0.014765803702175617,
      "fcm_dpo/margin": 54.60588836669922,
      "fcm_dpo/q_t": 0.4110748767852783,
      "grad_norm": 104.36587524414062,
      "learning_rate": 3.071590108427243e-07,
      "logits/chosen": -0.8259695768356323,
      "logits/rejected": -0.8063985705375671,
      "logps/chosen": -371.4505615234375,
      "logps/ref_chosen": -271.7437744140625,
      "logps/ref_rejected": -249.94981384277344,
      "logps/rejected": -404.26251220703125,
      "loss": 4.5585,
      "margin_dpo/margin_mean": 54.60589599609375,
      "margin_dpo/margin_std": 89.1750259399414,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -98.04922485351562,
      "KL/mean": -128.1886444091797,
      "KL/rejected_KL_mean": -158.32806396484375,
      "KL/std": 76.76235961914062,
      "epoch": 0.48795811518324606,
      "fcm_dpo/beta": 0.007130426354706287,
      "fcm_dpo/delta": -0.04825960099697113,
      "fcm_dpo/margin": 60.278846740722656,
      "fcm_dpo/q_t": 0.40295225381851196,
      "grad_norm": 87.40482330322266,
      "learning_rate": 3.05375222543809e-07,
      "logits/chosen": -0.8777355551719666,
      "logits/rejected": -0.8683412671089172,
      "logps/chosen": -383.39154052734375,
      "logps/ref_chosen": -285.3423156738281,
      "logps/ref_rejected": -266.34320068359375,
      "logps/rejected": -424.67120361328125,
      "loss": 4.3604,
      "margin_dpo/margin_mean": 60.27885437011719,
      "margin_dpo/margin_std": 81.72352600097656,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -96.27633666992188,
      "KL/mean": -123.4105224609375,
      "KL/rejected_KL_mean": -150.54470825195312,
      "KL/std": 77.66207122802734,
      "epoch": 0.4900523560209424,
      "fcm_dpo/beta": 0.007043421734124422,
      "fcm_dpo/delta": 0.03778496012091637,
      "fcm_dpo/margin": 54.268367767333984,
      "fcm_dpo/q_t": 0.41539156436920166,
      "grad_norm": 95.67949676513672,
      "learning_rate": 3.035884646397637e-07,
      "logits/chosen": -0.8511905074119568,
      "logits/rejected": -0.835209846496582,
      "logps/chosen": -391.1820983886719,
      "logps/ref_chosen": -294.9057312011719,
      "logps/ref_rejected": -299.37054443359375,
      "logps/rejected": -449.9152526855469,
      "loss": 4.6191,
      "margin_dpo/margin_mean": 54.26837158203125,
      "margin_dpo/margin_std": 96.68840789794922,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -97.85145568847656,
      "KL/mean": -126.2460708618164,
      "KL/rejected_KL_mean": -154.6407012939453,
      "KL/std": 81.84039306640625,
      "epoch": 0.49214659685863876,
      "fcm_dpo/beta": 0.007273434661328793,
      "fcm_dpo/delta": -0.0011693723499774933,
      "fcm_dpo/margin": 56.78924560546875,
      "fcm_dpo/q_t": 0.4076460003852844,
      "grad_norm": 85.81295013427734,
      "learning_rate": 3.017988329489923e-07,
      "logits/chosen": -0.8631937503814697,
      "logits/rejected": -0.8571017384529114,
      "logps/chosen": -387.3490295410156,
      "logps/ref_chosen": -289.49755859375,
      "logps/ref_rejected": -247.55076599121094,
      "logps/rejected": -402.19146728515625,
      "loss": 4.4794,
      "margin_dpo/margin_mean": 56.78924560546875,
      "margin_dpo/margin_std": 89.91757202148438,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -94.6702880859375,
      "KL/mean": -124.10176086425781,
      "KL/rejected_KL_mean": -153.53323364257812,
      "KL/std": 81.59674072265625,
      "epoch": 0.4942408376963351,
      "fcm_dpo/beta": 0.007358678616583347,
      "fcm_dpo/delta": 0.016236484050750732,
      "fcm_dpo/margin": 58.862953186035156,
      "fcm_dpo/q_t": 0.40372276306152344,
      "grad_norm": 108.43024444580078,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -0.8880228996276855,
      "logits/rejected": -0.8885826468467712,
      "logps/chosen": -383.554931640625,
      "logps/ref_chosen": -288.8846435546875,
      "logps/ref_rejected": -242.0452880859375,
      "logps/rejected": -395.5784912109375,
      "loss": 4.4283,
      "margin_dpo/margin_mean": 58.86294937133789,
      "margin_dpo/margin_std": 89.9344482421875,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -102.37773132324219,
      "KL/mean": -130.71270751953125,
      "KL/rejected_KL_mean": -159.04766845703125,
      "KL/std": 74.09178161621094,
      "epoch": 0.4963350785340314,
      "fcm_dpo/beta": 0.007313611917197704,
      "fcm_dpo/delta": -0.006238970439881086,
      "fcm_dpo/margin": 56.669918060302734,
      "fcm_dpo/q_t": 0.4077945947647095,
      "grad_norm": 97.0417251586914,
      "learning_rate": 2.9821133224630223e-07,
      "logits/chosen": -0.8524031639099121,
      "logits/rejected": -0.8318221569061279,
      "logps/chosen": -367.8564453125,
      "logps/ref_chosen": -265.47869873046875,
      "logps/ref_rejected": -267.9891357421875,
      "logps/rejected": -427.03680419921875,
      "loss": 4.4652,
      "margin_dpo/margin_mean": 56.66991424560547,
      "margin_dpo/margin_std": 88.27312469482422,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -106.47254180908203,
      "KL/mean": -135.31869506835938,
      "KL/rejected_KL_mean": -164.16485595703125,
      "KL/std": 78.32185363769531,
      "epoch": 0.49842931937172774,
      "fcm_dpo/beta": 0.007186025846749544,
      "fcm_dpo/delta": -0.01761997863650322,
      "fcm_dpo/margin": 57.69231033325195,
      "fcm_dpo/q_t": 0.4093272089958191,
      "grad_norm": 104.14332580566406,
      "learning_rate": 2.964136556211588e-07,
      "logits/chosen": -0.8547470569610596,
      "logits/rejected": -0.8244792819023132,
      "logps/chosen": -418.4751892089844,
      "logps/ref_chosen": -312.0026550292969,
      "logps/ref_rejected": -270.0257263183594,
      "logps/rejected": -434.1905822753906,
      "loss": 4.4873,
      "margin_dpo/margin_mean": 57.69231033325195,
      "margin_dpo/margin_std": 92.6472396850586,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -100.793701171875,
      "KL/mean": -124.86351013183594,
      "KL/rejected_KL_mean": -148.93331909179688,
      "KL/std": 80.29716491699219,
      "epoch": 0.5005235602094241,
      "fcm_dpo/beta": 0.0072754379361867905,
      "fcm_dpo/delta": 0.06706035137176514,
      "fcm_dpo/margin": 48.139625549316406,
      "fcm_dpo/q_t": 0.42403724789619446,
      "grad_norm": 104.30741882324219,
      "learning_rate": 2.946134899725226e-07,
      "logits/chosen": -0.8372979164123535,
      "logits/rejected": -0.876313328742981,
      "logps/chosen": -367.9609375,
      "logps/ref_chosen": -267.167236328125,
      "logps/ref_rejected": -275.99468994140625,
      "logps/rejected": -424.927978515625,
      "loss": 4.8145,
      "margin_dpo/margin_mean": 48.13962173461914,
      "margin_dpo/margin_std": 104.11760711669922,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -98.02072143554688,
      "KL/mean": -129.68580627441406,
      "KL/rejected_KL_mean": -161.3508758544922,
      "KL/std": 80.33810424804688,
      "epoch": 0.5026178010471204,
      "fcm_dpo/beta": 0.007256215903908014,
      "fcm_dpo/delta": -0.05167040228843689,
      "fcm_dpo/margin": 63.3301887512207,
      "fcm_dpo/q_t": 0.39842382073402405,
      "grad_norm": 133.92942810058594,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": -0.9134539365768433,
      "logits/rejected": -0.9070130586624146,
      "logps/chosen": -384.0003356933594,
      "logps/ref_chosen": -285.9796142578125,
      "logps/ref_rejected": -256.8258056640625,
      "logps/rejected": -418.17669677734375,
      "loss": 4.356,
      "margin_dpo/margin_mean": 63.33018493652344,
      "margin_dpo/margin_std": 92.22478485107422,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -110.93141174316406,
      "KL/mean": -136.90054321289062,
      "KL/rejected_KL_mean": -162.8696746826172,
      "KL/std": 86.49472045898438,
      "epoch": 0.5047120418848168,
      "fcm_dpo/beta": 0.007087720092386007,
      "fcm_dpo/delta": 0.001744309440255165,
      "fcm_dpo/margin": 51.93826675415039,
      "fcm_dpo/q_t": 0.419202983379364,
      "grad_norm": 111.87834167480469,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -0.8242614269256592,
      "logits/rejected": -0.8014207482337952,
      "logps/chosen": -372.4482727050781,
      "logps/ref_chosen": -261.516845703125,
      "logps/ref_rejected": -250.2250518798828,
      "logps/rejected": -413.0947570800781,
      "loss": 4.686,
      "margin_dpo/margin_mean": 51.938262939453125,
      "margin_dpo/margin_std": 96.19532775878906,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -104.3585433959961,
      "KL/mean": -135.87460327148438,
      "KL/rejected_KL_mean": -167.3906707763672,
      "KL/std": 84.62409210205078,
      "epoch": 0.506806282722513,
      "fcm_dpo/beta": 0.0071941278874874115,
      "fcm_dpo/delta": -0.004318598657846451,
      "fcm_dpo/margin": 63.03213119506836,
      "fcm_dpo/q_t": 0.40028345584869385,
      "grad_norm": 101.0625228881836,
      "learning_rate": 2.891990248961871e-07,
      "logits/chosen": -0.889725923538208,
      "logits/rejected": -0.8675822019577026,
      "logps/chosen": -374.87255859375,
      "logps/ref_chosen": -270.51397705078125,
      "logps/ref_rejected": -244.8560791015625,
      "logps/rejected": -412.24676513671875,
      "loss": 4.3397,
      "margin_dpo/margin_mean": 63.03213119506836,
      "margin_dpo/margin_std": 91.31277465820312,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -96.94046020507812,
      "KL/mean": -131.6834716796875,
      "KL/rejected_KL_mean": -166.426513671875,
      "KL/std": 83.37822723388672,
      "epoch": 0.5089005235602094,
      "fcm_dpo/beta": 0.0071937511675059795,
      "fcm_dpo/delta": -0.0525360032916069,
      "fcm_dpo/margin": 69.48601531982422,
      "fcm_dpo/q_t": 0.3922196328639984,
      "grad_norm": 124.29874420166016,
      "learning_rate": 2.873898697848762e-07,
      "logits/chosen": -0.8835136294364929,
      "logits/rejected": -0.8683462142944336,
      "logps/chosen": -421.6225280761719,
      "logps/ref_chosen": -324.68206787109375,
      "logps/ref_rejected": -307.1111755371094,
      "logps/rejected": -473.53765869140625,
      "loss": 4.2819,
      "margin_dpo/margin_mean": 69.48601531982422,
      "margin_dpo/margin_std": 100.506591796875,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -98.72238159179688,
      "KL/mean": -132.43075561523438,
      "KL/rejected_KL_mean": -166.13912963867188,
      "KL/std": 78.39314270019531,
      "epoch": 0.5109947643979058,
      "fcm_dpo/beta": 0.00676451763138175,
      "fcm_dpo/delta": -0.00790400430560112,
      "fcm_dpo/margin": 67.41674041748047,
      "fcm_dpo/q_t": 0.39651355147361755,
      "grad_norm": 68.7562484741211,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": -0.8715980648994446,
      "logits/rejected": -0.8241588473320007,
      "logps/chosen": -417.70159912109375,
      "logps/ref_chosen": -318.979248046875,
      "logps/ref_rejected": -269.67572021484375,
      "logps/rejected": -435.8148498535156,
      "loss": 4.3089,
      "margin_dpo/margin_mean": 67.41674041748047,
      "margin_dpo/margin_std": 89.44083404541016,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -100.52628326416016,
      "KL/mean": -134.77716064453125,
      "KL/rejected_KL_mean": -169.02804565429688,
      "KL/std": 79.51602172851562,
      "epoch": 0.5130890052356021,
      "fcm_dpo/beta": 0.006687359418720007,
      "fcm_dpo/delta": -0.06442329287528992,
      "fcm_dpo/margin": 68.50174713134766,
      "fcm_dpo/q_t": 0.3954485058784485,
      "grad_norm": 87.22725677490234,
      "learning_rate": 2.837656413735479e-07,
      "logits/chosen": -0.8819712996482849,
      "logits/rejected": -0.8864008784294128,
      "logps/chosen": -395.42431640625,
      "logps/ref_chosen": -294.8980712890625,
      "logps/ref_rejected": -239.8111114501953,
      "logps/rejected": -408.839111328125,
      "loss": 4.3026,
      "margin_dpo/margin_mean": 68.50174713134766,
      "margin_dpo/margin_std": 89.4078140258789,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -116.78390502929688,
      "KL/mean": -140.38560485839844,
      "KL/rejected_KL_mean": -163.98727416992188,
      "KL/std": 78.14247131347656,
      "epoch": 0.5151832460732985,
      "fcm_dpo/beta": 0.006635972298681736,
      "fcm_dpo/delta": 0.04852314293384552,
      "fcm_dpo/margin": 47.20338439941406,
      "fcm_dpo/q_t": 0.4310261905193329,
      "grad_norm": 97.5146255493164,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -0.848738968372345,
      "logits/rejected": -0.8533201217651367,
      "logps/chosen": -397.4693298339844,
      "logps/ref_chosen": -280.6854248046875,
      "logps/ref_rejected": -253.65382385253906,
      "logps/rejected": -417.64111328125,
      "loss": 4.8281,
      "margin_dpo/margin_mean": 47.2033805847168,
      "margin_dpo/margin_std": 100.99250030517578,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -105.44461059570312,
      "KL/mean": -135.54441833496094,
      "KL/rejected_KL_mean": -165.6442108154297,
      "KL/std": 81.02322387695312,
      "epoch": 0.5172774869109947,
      "fcm_dpo/beta": 0.006691988091915846,
      "fcm_dpo/delta": -0.026615649461746216,
      "fcm_dpo/margin": 60.19961166381836,
      "fcm_dpo/q_t": 0.41041994094848633,
      "grad_norm": 85.68934631347656,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": -0.8649847507476807,
      "logits/rejected": -0.8633992075920105,
      "logps/chosen": -386.55364990234375,
      "logps/ref_chosen": -281.1091003417969,
      "logps/ref_rejected": -260.3700866699219,
      "logps/rejected": -426.0142822265625,
      "loss": 4.4867,
      "margin_dpo/margin_mean": 60.19961166381836,
      "margin_dpo/margin_std": 92.49925994873047,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -102.54319763183594,
      "KL/mean": -133.4750213623047,
      "KL/rejected_KL_mean": -164.40682983398438,
      "KL/std": 82.3779296875,
      "epoch": 0.5193717277486911,
      "fcm_dpo/beta": 0.006554853171110153,
      "fcm_dpo/delta": 0.0034740203991532326,
      "fcm_dpo/margin": 61.8636589050293,
      "fcm_dpo/q_t": 0.40719956159591675,
      "grad_norm": 79.59374237060547,
      "learning_rate": 2.7831596169367227e-07,
      "logits/chosen": -0.8320137858390808,
      "logits/rejected": -0.8406758308410645,
      "logps/chosen": -372.861572265625,
      "logps/ref_chosen": -270.318359375,
      "logps/ref_rejected": -233.46778869628906,
      "logps/rejected": -397.8746337890625,
      "loss": 4.4641,
      "margin_dpo/margin_mean": 61.86365509033203,
      "margin_dpo/margin_std": 91.04399871826172,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -118.85231018066406,
      "KL/mean": -145.34910583496094,
      "KL/rejected_KL_mean": -171.84588623046875,
      "KL/std": 85.65460205078125,
      "epoch": 0.5214659685863874,
      "fcm_dpo/beta": 0.0067500886507332325,
      "fcm_dpo/delta": 0.0437769889831543,
      "fcm_dpo/margin": 52.993560791015625,
      "fcm_dpo/q_t": 0.41927647590637207,
      "grad_norm": 81.46465301513672,
      "learning_rate": 2.7649623482442274e-07,
      "logits/chosen": -0.8606888651847839,
      "logits/rejected": -0.8353737592697144,
      "logps/chosen": -394.66119384765625,
      "logps/ref_chosen": -275.8088684082031,
      "logps/ref_rejected": -243.45138549804688,
      "logps/rejected": -415.2972717285156,
      "loss": 4.6874,
      "margin_dpo/margin_mean": 52.993560791015625,
      "margin_dpo/margin_std": 99.65210723876953,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -120.21992492675781,
      "KL/mean": -153.18345642089844,
      "KL/rejected_KL_mean": -186.14695739746094,
      "KL/std": 92.27700805664062,
      "epoch": 0.5235602094240838,
      "fcm_dpo/beta": 0.006649673916399479,
      "fcm_dpo/delta": -0.058766674250364304,
      "fcm_dpo/margin": 65.92704010009766,
      "fcm_dpo/q_t": 0.4020792245864868,
      "grad_norm": 99.60740661621094,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": -0.8576858043670654,
      "logits/rejected": -0.8540810346603394,
      "logps/chosen": -412.7144470214844,
      "logps/ref_chosen": -292.4945373535156,
      "logps/ref_rejected": -284.2869567871094,
      "logps/rejected": -470.4339294433594,
      "loss": 4.4061,
      "margin_dpo/margin_mean": 65.92704010009766,
      "margin_dpo/margin_std": 98.60626220703125,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -108.93125915527344,
      "KL/mean": -139.29469299316406,
      "KL/rejected_KL_mean": -169.6581573486328,
      "KL/std": 89.49659729003906,
      "epoch": 0.5256544502617801,
      "fcm_dpo/beta": 0.0064571150578558445,
      "fcm_dpo/delta": -0.06133609637618065,
      "fcm_dpo/margin": 60.72690200805664,
      "fcm_dpo/q_t": 0.4157108962535858,
      "grad_norm": 110.13551330566406,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -0.8620572686195374,
      "logits/rejected": -0.8455150127410889,
      "logps/chosen": -390.6678161621094,
      "logps/ref_chosen": -281.736572265625,
      "logps/ref_rejected": -255.9419708251953,
      "logps/rejected": -425.60015869140625,
      "loss": 4.5366,
      "margin_dpo/margin_mean": 60.72690200805664,
      "margin_dpo/margin_std": 96.52062225341797,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -118.67613983154297,
      "KL/mean": -148.42953491210938,
      "KL/rejected_KL_mean": -178.18292236328125,
      "KL/std": 86.1222152709961,
      "epoch": 0.5277486910994764,
      "fcm_dpo/beta": 0.006251195445656776,
      "fcm_dpo/delta": 0.005373414605855942,
      "fcm_dpo/margin": 59.50675964355469,
      "fcm_dpo/q_t": 0.41494932770729065,
      "grad_norm": 66.31450653076172,
      "learning_rate": 2.7102891946217994e-07,
      "logits/chosen": -0.9113876223564148,
      "logits/rejected": -0.8850067853927612,
      "logps/chosen": -414.6435852050781,
      "logps/ref_chosen": -295.9674072265625,
      "logps/ref_rejected": -280.111572265625,
      "logps/rejected": -458.29449462890625,
      "loss": 4.5616,
      "margin_dpo/margin_mean": 59.50675582885742,
      "margin_dpo/margin_std": 95.56446075439453,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -127.55453491210938,
      "KL/mean": -154.7516632080078,
      "KL/rejected_KL_mean": -181.94879150390625,
      "KL/std": 91.63578796386719,
      "epoch": 0.5298429319371728,
      "fcm_dpo/beta": 0.006313451565802097,
      "fcm_dpo/delta": 0.028458960354328156,
      "fcm_dpo/margin": 54.394264221191406,
      "fcm_dpo/q_t": 0.42388641834259033,
      "grad_norm": 102.37186431884766,
      "learning_rate": 2.692040951966617e-07,
      "logits/chosen": -0.8840073347091675,
      "logits/rejected": -0.8747701644897461,
      "logps/chosen": -404.6268310546875,
      "logps/ref_chosen": -277.072265625,
      "logps/ref_rejected": -247.31643676757812,
      "logps/rejected": -429.2652282714844,
      "loss": 4.7004,
      "margin_dpo/margin_mean": 54.394264221191406,
      "margin_dpo/margin_std": 100.25741577148438,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -115.7991714477539,
      "KL/mean": -146.30148315429688,
      "KL/rejected_KL_mean": -176.80380249023438,
      "KL/std": 87.80889129638672,
      "epoch": 0.5319371727748691,
      "fcm_dpo/beta": 0.006625975947827101,
      "fcm_dpo/delta": 0.01226937398314476,
      "fcm_dpo/margin": 61.004638671875,
      "fcm_dpo/q_t": 0.4097105860710144,
      "grad_norm": 72.49644470214844,
      "learning_rate": 2.6737824107379947e-07,
      "logits/chosen": -0.8181810975074768,
      "logits/rejected": -0.7995843291282654,
      "logps/chosen": -385.74700927734375,
      "logps/ref_chosen": -269.9478454589844,
      "logps/ref_rejected": -249.45005798339844,
      "logps/rejected": -426.25384521484375,
      "loss": 4.4656,
      "margin_dpo/margin_mean": 61.004634857177734,
      "margin_dpo/margin_std": 92.15809631347656,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -113.15776062011719,
      "KL/mean": -148.5821533203125,
      "KL/rejected_KL_mean": -184.00653076171875,
      "KL/std": 89.79924011230469,
      "epoch": 0.5340314136125655,
      "fcm_dpo/beta": 0.006382976658642292,
      "fcm_dpo/delta": -0.03686467930674553,
      "fcm_dpo/margin": 70.8487777709961,
      "fcm_dpo/q_t": 0.39925360679626465,
      "grad_norm": 79.42298126220703,
      "learning_rate": 2.655514550086086e-07,
      "logits/chosen": -0.8431286811828613,
      "logits/rejected": -0.8071163892745972,
      "logps/chosen": -419.81304931640625,
      "logps/ref_chosen": -306.6552734375,
      "logps/ref_rejected": -254.47528076171875,
      "logps/rejected": -438.4818115234375,
      "loss": 4.379,
      "margin_dpo/margin_mean": 70.8487777709961,
      "margin_dpo/margin_std": 106.17488861083984,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -116.35472869873047,
      "KL/mean": -147.4508056640625,
      "KL/rejected_KL_mean": -178.546875,
      "KL/std": 93.63533020019531,
      "epoch": 0.5361256544502618,
      "fcm_dpo/beta": 0.006252347491681576,
      "fcm_dpo/delta": -0.009718809276819229,
      "fcm_dpo/margin": 62.1921272277832,
      "fcm_dpo/q_t": 0.40854233503341675,
      "grad_norm": 130.63528442382812,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -0.8665859699249268,
      "logits/rejected": -0.8540273308753967,
      "logps/chosen": -440.07293701171875,
      "logps/ref_chosen": -323.7181701660156,
      "logps/ref_rejected": -254.1871337890625,
      "logps/rejected": -432.7339782714844,
      "loss": 4.6628,
      "margin_dpo/margin_mean": 62.19212341308594,
      "margin_dpo/margin_std": 109.32525634765625,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -114.06240844726562,
      "KL/mean": -148.30873107910156,
      "KL/rejected_KL_mean": -182.55506896972656,
      "KL/std": 88.5597915649414,
      "epoch": 0.5382198952879581,
      "fcm_dpo/beta": 0.006275220774114132,
      "fcm_dpo/delta": 0.020723635330796242,
      "fcm_dpo/margin": 68.49266815185547,
      "fcm_dpo/q_t": 0.4038299322128296,
      "grad_norm": 78.33170318603516,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": -0.8573966026306152,
      "logits/rejected": -0.8473801016807556,
      "logps/chosen": -381.27447509765625,
      "logps/ref_chosen": -267.21209716796875,
      "logps/ref_rejected": -249.12579345703125,
      "logps/rejected": -431.68084716796875,
      "loss": 4.3952,
      "margin_dpo/margin_mean": 68.49266815185547,
      "margin_dpo/margin_std": 101.45110321044922,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -121.85859680175781,
      "KL/mean": -157.8359832763672,
      "KL/rejected_KL_mean": -193.81341552734375,
      "KL/std": 86.7300796508789,
      "epoch": 0.5403141361256545,
      "fcm_dpo/beta": 0.006057636346668005,
      "fcm_dpo/delta": -0.05791028216481209,
      "fcm_dpo/margin": 71.9548110961914,
      "fcm_dpo/q_t": 0.3997272253036499,
      "grad_norm": 96.5600357055664,
      "learning_rate": 2.600664850273538e-07,
      "logits/chosen": -0.877286434173584,
      "logits/rejected": -0.8453212976455688,
      "logps/chosen": -399.54132080078125,
      "logps/ref_chosen": -277.6827392578125,
      "logps/ref_rejected": -250.73385620117188,
      "logps/rejected": -444.5472412109375,
      "loss": 4.3241,
      "margin_dpo/margin_mean": 71.9548110961914,
      "margin_dpo/margin_std": 92.13557434082031,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -108.7266845703125,
      "KL/mean": -141.59259033203125,
      "KL/rejected_KL_mean": -174.45851135253906,
      "KL/std": 88.0118408203125,
      "epoch": 0.5424083769633508,
      "fcm_dpo/beta": 0.006230651400983334,
      "fcm_dpo/delta": 0.04143606126308441,
      "fcm_dpo/margin": 65.73184967041016,
      "fcm_dpo/q_t": 0.4081200957298279,
      "grad_norm": 73.45479583740234,
      "learning_rate": 2.582369512637302e-07,
      "logits/chosen": -0.8940492272377014,
      "logits/rejected": -0.8895572423934937,
      "logps/chosen": -403.336669921875,
      "logps/ref_chosen": -294.6099853515625,
      "logps/ref_rejected": -272.2725830078125,
      "logps/rejected": -446.7311096191406,
      "loss": 4.4185,
      "margin_dpo/margin_mean": 65.73184204101562,
      "margin_dpo/margin_std": 96.06785583496094,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -118.75645446777344,
      "KL/mean": -133.64230346679688,
      "KL/rejected_KL_mean": -148.5281219482422,
      "KL/std": 79.65795135498047,
      "epoch": 0.5445026178010471,
      "fcm_dpo/beta": 0.006530113518238068,
      "fcm_dpo/delta": 0.09596279263496399,
      "fcm_dpo/margin": 29.771697998046875,
      "fcm_dpo/q_t": 0.45730045437812805,
      "grad_norm": 86.8549575805664,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": -0.8781209588050842,
      "logits/rejected": -0.8734769821166992,
      "logps/chosen": -409.6135559082031,
      "logps/ref_chosen": -290.85711669921875,
      "logps/ref_rejected": -277.5970153808594,
      "logps/rejected": -426.1252136230469,
      "loss": 5.1721,
      "margin_dpo/margin_mean": 29.771697998046875,
      "margin_dpo/margin_std": 92.85311126708984,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -123.58172607421875,
      "KL/mean": -150.2753143310547,
      "KL/rejected_KL_mean": -176.9689178466797,
      "KL/std": 85.03296661376953,
      "epoch": 0.5465968586387434,
      "fcm_dpo/beta": 0.0068458509631454945,
      "fcm_dpo/delta": 0.005006203893572092,
      "fcm_dpo/margin": 53.3871955871582,
      "fcm_dpo/q_t": 0.42041996121406555,
      "grad_norm": 94.22754669189453,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -0.7682502865791321,
      "logits/rejected": -0.7791002988815308,
      "logps/chosen": -374.7139587402344,
      "logps/ref_chosen": -251.13223266601562,
      "logps/ref_rejected": -244.76016235351562,
      "logps/rejected": -421.72906494140625,
      "loss": 4.7162,
      "margin_dpo/margin_mean": 53.38719177246094,
      "margin_dpo/margin_std": 103.89097595214844,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -114.67688751220703,
      "KL/mean": -148.95294189453125,
      "KL/rejected_KL_mean": -183.22901916503906,
      "KL/std": 85.02909851074219,
      "epoch": 0.5486910994764398,
      "fcm_dpo/beta": 0.006849959958344698,
      "fcm_dpo/delta": -0.021255794912576675,
      "fcm_dpo/margin": 68.55213165283203,
      "fcm_dpo/q_t": 0.3964102268218994,
      "grad_norm": 106.8567123413086,
      "learning_rate": 2.527460921992209e-07,
      "logits/chosen": -0.7973865866661072,
      "logits/rejected": -0.7870882749557495,
      "logps/chosen": -414.398681640625,
      "logps/ref_chosen": -299.7217712402344,
      "logps/ref_rejected": -277.0969543457031,
      "logps/rejected": -460.32598876953125,
      "loss": 4.3019,
      "margin_dpo/margin_mean": 68.5521240234375,
      "margin_dpo/margin_std": 95.2181167602539,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -120.68804931640625,
      "KL/mean": -152.3499298095703,
      "KL/rejected_KL_mean": -184.01182556152344,
      "KL/std": 88.39866638183594,
      "epoch": 0.5507853403141362,
      "fcm_dpo/beta": 0.006576926913112402,
      "fcm_dpo/delta": -0.034953195601701736,
      "fcm_dpo/margin": 63.32377243041992,
      "fcm_dpo/q_t": 0.40581172704696655,
      "grad_norm": 76.1161117553711,
      "learning_rate": 2.509153804294318e-07,
      "logits/chosen": -0.8096163272857666,
      "logits/rejected": -0.7914470434188843,
      "logps/chosen": -400.6406555175781,
      "logps/ref_chosen": -279.95257568359375,
      "logps/ref_rejected": -256.5327453613281,
      "logps/rejected": -440.5445861816406,
      "loss": 4.4729,
      "margin_dpo/margin_mean": 63.32377243041992,
      "margin_dpo/margin_std": 97.14537811279297,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -112.83721923828125,
      "KL/mean": -149.47616577148438,
      "KL/rejected_KL_mean": -186.11514282226562,
      "KL/std": 89.69928741455078,
      "epoch": 0.5528795811518324,
      "fcm_dpo/beta": 0.006283737253397703,
      "fcm_dpo/delta": -0.05668240040540695,
      "fcm_dpo/margin": 73.27791595458984,
      "fcm_dpo/q_t": 0.3964623808860779,
      "grad_norm": 76.45155334472656,
      "learning_rate": 2.4908461957056825e-07,
      "logits/chosen": -0.8057646751403809,
      "logits/rejected": -0.8009496331214905,
      "logps/chosen": -373.3723449707031,
      "logps/ref_chosen": -260.53509521484375,
      "logps/ref_rejected": -255.53799438476562,
      "logps/rejected": -441.65313720703125,
      "loss": 4.2737,
      "margin_dpo/margin_mean": 73.27791595458984,
      "margin_dpo/margin_std": 96.34459686279297,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -120.75982666015625,
      "KL/mean": -157.77056884765625,
      "KL/rejected_KL_mean": -194.7813262939453,
      "KL/std": 96.8575668334961,
      "epoch": 0.5549738219895288,
      "fcm_dpo/beta": 0.006147218868136406,
      "fcm_dpo/delta": -0.006353056989610195,
      "fcm_dpo/margin": 74.02152252197266,
      "fcm_dpo/q_t": 0.39908432960510254,
      "grad_norm": 65.59973907470703,
      "learning_rate": 2.4725390780077905e-07,
      "logits/chosen": -0.8776726722717285,
      "logits/rejected": -0.8822907209396362,
      "logps/chosen": -404.472900390625,
      "logps/ref_chosen": -283.7130432128906,
      "logps/ref_rejected": -270.3209533691406,
      "logps/rejected": -465.1022644042969,
      "loss": 4.3596,
      "margin_dpo/margin_mean": 74.02152252197266,
      "margin_dpo/margin_std": 107.7284164428711,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -107.3155517578125,
      "KL/mean": -144.4962158203125,
      "KL/rejected_KL_mean": -181.67691040039062,
      "KL/std": 88.30040740966797,
      "epoch": 0.5570680628272251,
      "fcm_dpo/beta": 0.0062091536819934845,
      "fcm_dpo/delta": -0.01308838278055191,
      "fcm_dpo/margin": 74.3613510131836,
      "fcm_dpo/q_t": 0.3958033323287964,
      "grad_norm": 63.141605377197266,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -0.897355318069458,
      "logits/rejected": -0.8671724796295166,
      "logps/chosen": -385.41485595703125,
      "logps/ref_chosen": -278.09930419921875,
      "logps/ref_rejected": -260.6734619140625,
      "logps/rejected": -442.35040283203125,
      "loss": 4.249,
      "margin_dpo/margin_mean": 74.3613510131836,
      "margin_dpo/margin_std": 94.18682861328125,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -125.74674987792969,
      "KL/mean": -155.48345947265625,
      "KL/rejected_KL_mean": -185.22019958496094,
      "KL/std": 90.16264343261719,
      "epoch": 0.5591623036649215,
      "fcm_dpo/beta": 0.006180617026984692,
      "fcm_dpo/delta": 0.032296277582645416,
      "fcm_dpo/margin": 59.47342300415039,
      "fcm_dpo/q_t": 0.4170716404914856,
      "grad_norm": 95.7731704711914,
      "learning_rate": 2.435930242225919e-07,
      "logits/chosen": -0.8393760919570923,
      "logits/rejected": -0.849579393863678,
      "logps/chosen": -406.0799255371094,
      "logps/ref_chosen": -280.33319091796875,
      "logps/ref_rejected": -247.78099060058594,
      "logps/rejected": -433.0011901855469,
      "loss": 4.5723,
      "margin_dpo/margin_mean": 59.47342300415039,
      "margin_dpo/margin_std": 95.5189437866211,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -116.31019592285156,
      "KL/mean": -151.79098510742188,
      "KL/rejected_KL_mean": -187.27175903320312,
      "KL/std": 91.29608154296875,
      "epoch": 0.5612565445026177,
      "fcm_dpo/beta": 0.006230502855032682,
      "fcm_dpo/delta": -0.021773632615804672,
      "fcm_dpo/margin": 70.96157836914062,
      "fcm_dpo/q_t": 0.40059587359428406,
      "grad_norm": 92.79893493652344,
      "learning_rate": 2.4176304873626984e-07,
      "logits/chosen": -0.7975625991821289,
      "logits/rejected": -0.7774516940116882,
      "logps/chosen": -420.4888916015625,
      "logps/ref_chosen": -304.1787109375,
      "logps/ref_rejected": -272.80316162109375,
      "logps/rejected": -460.0749206542969,
      "loss": 4.3401,
      "margin_dpo/margin_mean": 70.96157836914062,
      "margin_dpo/margin_std": 98.81417846679688,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -117.4832763671875,
      "KL/mean": -144.62725830078125,
      "KL/rejected_KL_mean": -171.77127075195312,
      "KL/std": 84.31640625,
      "epoch": 0.5633507853403141,
      "fcm_dpo/beta": 0.006322528701275587,
      "fcm_dpo/delta": 0.07198101282119751,
      "fcm_dpo/margin": 54.2879753112793,
      "fcm_dpo/q_t": 0.4221433103084564,
      "grad_norm": 69.35298156738281,
      "learning_rate": 2.399335149726463e-07,
      "logits/chosen": -0.8315380811691284,
      "logits/rejected": -0.8260846734046936,
      "logps/chosen": -367.328369140625,
      "logps/ref_chosen": -249.84512329101562,
      "logps/ref_rejected": -223.37356567382812,
      "logps/rejected": -395.1448669433594,
      "loss": 4.6862,
      "margin_dpo/margin_mean": 54.28797912597656,
      "margin_dpo/margin_std": 100.50502014160156,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -120.90419006347656,
      "KL/mean": -152.5150604248047,
      "KL/rejected_KL_mean": -184.1259307861328,
      "KL/std": 91.83196258544922,
      "epoch": 0.5654450261780105,
      "fcm_dpo/beta": 0.006595199462026358,
      "fcm_dpo/delta": 0.034104883670806885,
      "fcm_dpo/margin": 63.22172927856445,
      "fcm_dpo/q_t": 0.4080127775669098,
      "grad_norm": 83.05050659179688,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": -0.9114333391189575,
      "logits/rejected": -0.9235341548919678,
      "logps/chosen": -439.4665832519531,
      "logps/ref_chosen": -318.5623779296875,
      "logps/ref_rejected": -281.1880798339844,
      "logps/rejected": -465.31402587890625,
      "loss": 4.5335,
      "margin_dpo/margin_mean": 63.22172546386719,
      "margin_dpo/margin_std": 108.9056167602539,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -115.72140502929688,
      "KL/mean": -145.15658569335938,
      "KL/rejected_KL_mean": -174.59176635742188,
      "KL/std": 88.58271789550781,
      "epoch": 0.5675392670157068,
      "fcm_dpo/beta": 0.006940803490579128,
      "fcm_dpo/delta": 0.04145065322518349,
      "fcm_dpo/margin": 58.870361328125,
      "fcm_dpo/q_t": 0.41082993149757385,
      "grad_norm": 92.02179718017578,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -0.7666244506835938,
      "logits/rejected": -0.762701153755188,
      "logps/chosen": -399.8260803222656,
      "logps/ref_chosen": -284.104736328125,
      "logps/ref_rejected": -253.9580535888672,
      "logps/rejected": -428.5498352050781,
      "loss": 4.5258,
      "margin_dpo/margin_mean": 58.870365142822266,
      "margin_dpo/margin_std": 97.83793640136719,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -109.43763732910156,
      "KL/mean": -139.37509155273438,
      "KL/rejected_KL_mean": -169.31253051757812,
      "KL/std": 90.08634948730469,
      "epoch": 0.5696335078534032,
      "fcm_dpo/beta": 0.006881224922835827,
      "fcm_dpo/delta": -0.003738149767741561,
      "fcm_dpo/margin": 59.874900817871094,
      "fcm_dpo/q_t": 0.4103890657424927,
      "grad_norm": 78.98287200927734,
      "learning_rate": 2.344485449913914e-07,
      "logits/chosen": -0.8604626059532166,
      "logits/rejected": -0.8469264507293701,
      "logps/chosen": -406.796630859375,
      "logps/ref_chosen": -297.3590087890625,
      "logps/ref_rejected": -279.20196533203125,
      "logps/rejected": -448.5144958496094,
      "loss": 4.5829,
      "margin_dpo/margin_mean": 59.87489700317383,
      "margin_dpo/margin_std": 107.67626953125,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -107.7457275390625,
      "KL/mean": -140.65785217285156,
      "KL/rejected_KL_mean": -173.57000732421875,
      "KL/std": 83.90080261230469,
      "epoch": 0.5717277486910994,
      "fcm_dpo/beta": 0.006859644316136837,
      "fcm_dpo/delta": -0.002420984208583832,
      "fcm_dpo/margin": 65.82425689697266,
      "fcm_dpo/q_t": 0.40162932872772217,
      "grad_norm": 97.58268737792969,
      "learning_rate": 2.3262175892620062e-07,
      "logits/chosen": -0.8500223755836487,
      "logits/rejected": -0.8640397191047668,
      "logps/chosen": -400.95147705078125,
      "logps/ref_chosen": -293.20574951171875,
      "logps/ref_rejected": -274.7646789550781,
      "logps/rejected": -448.3346862792969,
      "loss": 4.4279,
      "margin_dpo/margin_mean": 65.82425689697266,
      "margin_dpo/margin_std": 104.84503173828125,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -107.45245361328125,
      "KL/mean": -146.3524169921875,
      "KL/rejected_KL_mean": -185.25241088867188,
      "KL/std": 90.25546264648438,
      "epoch": 0.5738219895287958,
      "fcm_dpo/beta": 0.006637836806476116,
      "fcm_dpo/delta": -0.07071861624717712,
      "fcm_dpo/margin": 77.79995727539062,
      "fcm_dpo/q_t": 0.3843676745891571,
      "grad_norm": 84.57036590576172,
      "learning_rate": 2.3079590480333827e-07,
      "logits/chosen": -0.8090867400169373,
      "logits/rejected": -0.7798628807067871,
      "logps/chosen": -378.0111083984375,
      "logps/ref_chosen": -270.55865478515625,
      "logps/ref_rejected": -239.47048950195312,
      "logps/rejected": -424.722900390625,
      "loss": 4.149,
      "margin_dpo/margin_mean": 77.7999496459961,
      "margin_dpo/margin_std": 99.16637420654297,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -98.23279571533203,
      "KL/mean": -136.64784240722656,
      "KL/rejected_KL_mean": -175.06288146972656,
      "KL/std": 86.72215270996094,
      "epoch": 0.5759162303664922,
      "fcm_dpo/beta": 0.0063896216452121735,
      "fcm_dpo/delta": -0.04358269274234772,
      "fcm_dpo/margin": 76.83008575439453,
      "fcm_dpo/q_t": 0.3909229338169098,
      "grad_norm": 80.09928131103516,
      "learning_rate": 2.2897108053782e-07,
      "logits/chosen": -0.8569150567054749,
      "logits/rejected": -0.839890718460083,
      "logps/chosen": -348.552001953125,
      "logps/ref_chosen": -250.31922912597656,
      "logps/ref_rejected": -249.3187255859375,
      "logps/rejected": -424.381591796875,
      "loss": 4.1733,
      "margin_dpo/margin_mean": 76.83008575439453,
      "margin_dpo/margin_std": 96.37769317626953,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -104.65113830566406,
      "KL/mean": -139.7989959716797,
      "KL/rejected_KL_mean": -174.9468536376953,
      "KL/std": 89.53086853027344,
      "epoch": 0.5780104712041885,
      "fcm_dpo/beta": 0.006188282743096352,
      "fcm_dpo/delta": 0.015074246563017368,
      "fcm_dpo/margin": 70.29569244384766,
      "fcm_dpo/q_t": 0.40223774313926697,
      "grad_norm": 62.7150993347168,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -0.9228585958480835,
      "logits/rejected": -0.899926483631134,
      "logps/chosen": -402.2822265625,
      "logps/ref_chosen": -297.6310729980469,
      "logps/ref_rejected": -295.225830078125,
      "logps/rejected": -470.17266845703125,
      "loss": 4.3956,
      "margin_dpo/margin_mean": 70.29569244384766,
      "margin_dpo/margin_std": 102.95513153076172,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -116.80690002441406,
      "KL/mean": -141.62240600585938,
      "KL/rejected_KL_mean": -166.43795776367188,
      "KL/std": 88.11683654785156,
      "epoch": 0.5801047120418849,
      "fcm_dpo/beta": 0.00642796466127038,
      "fcm_dpo/delta": 0.022014472633600235,
      "fcm_dpo/margin": 49.63104248046875,
      "fcm_dpo/q_t": 0.42970696091651917,
      "grad_norm": 105.10576629638672,
      "learning_rate": 2.2532491295748865e-07,
      "logits/chosen": -0.8672906756401062,
      "logits/rejected": -0.8692587614059448,
      "logps/chosen": -383.16741943359375,
      "logps/ref_chosen": -266.3604736328125,
      "logps/ref_rejected": -253.36767578125,
      "logps/rejected": -419.8056335449219,
      "loss": 4.7944,
      "margin_dpo/margin_mean": 49.631038665771484,
      "margin_dpo/margin_std": 101.31917572021484,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -122.17640686035156,
      "KL/mean": -146.43406677246094,
      "KL/rejected_KL_mean": -170.6917266845703,
      "KL/std": 93.96764373779297,
      "epoch": 0.5821989528795811,
      "fcm_dpo/beta": 0.006559680216014385,
      "fcm_dpo/delta": 0.013672933913767338,
      "fcm_dpo/margin": 48.51532745361328,
      "fcm_dpo/q_t": 0.43570613861083984,
      "grad_norm": 100.32164764404297,
      "learning_rate": 2.2350376517557726e-07,
      "logits/chosen": -0.8850666284561157,
      "logits/rejected": -0.8482396006584167,
      "logps/chosen": -389.583740234375,
      "logps/ref_chosen": -267.40728759765625,
      "logps/ref_rejected": -229.5758514404297,
      "logps/rejected": -400.267578125,
      "loss": 4.9377,
      "margin_dpo/margin_mean": 48.51532745361328,
      "margin_dpo/margin_std": 115.62205505371094,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -114.18596649169922,
      "KL/mean": -151.2555694580078,
      "KL/rejected_KL_mean": -188.32513427734375,
      "KL/std": 97.55309295654297,
      "epoch": 0.5842931937172775,
      "fcm_dpo/beta": 0.006348501890897751,
      "fcm_dpo/delta": -0.07341407984495163,
      "fcm_dpo/margin": 74.13917541503906,
      "fcm_dpo/q_t": 0.3965674042701721,
      "grad_norm": 107.36109924316406,
      "learning_rate": 2.2168403830632769e-07,
      "logits/chosen": -0.8053258657455444,
      "logits/rejected": -0.7910168170928955,
      "logps/chosen": -427.5538024902344,
      "logps/ref_chosen": -313.3677978515625,
      "logps/ref_rejected": -299.1744384765625,
      "logps/rejected": -487.4996032714844,
      "loss": 4.3387,
      "margin_dpo/margin_mean": 74.13917541503906,
      "margin_dpo/margin_std": 108.39376831054688,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -105.1937255859375,
      "KL/mean": -134.20884704589844,
      "KL/rejected_KL_mean": -163.2239990234375,
      "KL/std": 81.9890365600586,
      "epoch": 0.5863874345549738,
      "fcm_dpo/beta": 0.006443873979151249,
      "fcm_dpo/delta": 0.07814231514930725,
      "fcm_dpo/margin": 58.03026580810547,
      "fcm_dpo/q_t": 0.4158723056316376,
      "grad_norm": 75.37612915039062,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": -0.8752536773681641,
      "logits/rejected": -0.8873091340065002,
      "logps/chosen": -370.74951171875,
      "logps/ref_chosen": -265.5558166503906,
      "logps/ref_rejected": -247.1573944091797,
      "logps/rejected": -410.38140869140625,
      "loss": 4.5904,
      "margin_dpo/margin_mean": 58.03026580810547,
      "margin_dpo/margin_std": 100.64657592773438,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -121.31245422363281,
      "KL/mean": -156.13644409179688,
      "KL/rejected_KL_mean": -190.96043395996094,
      "KL/std": 93.35604095458984,
      "epoch": 0.5884816753926702,
      "fcm_dpo/beta": 0.006514973007142544,
      "fcm_dpo/delta": -0.005094392225146294,
      "fcm_dpo/margin": 69.6479721069336,
      "fcm_dpo/q_t": 0.40105491876602173,
      "grad_norm": 73.65348815917969,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -0.8500332832336426,
      "logits/rejected": -0.8594211339950562,
      "logps/chosen": -416.612060546875,
      "logps/ref_chosen": -295.2995910644531,
      "logps/ref_rejected": -293.80877685546875,
      "logps/rejected": -484.7691650390625,
      "loss": 4.3375,
      "margin_dpo/margin_mean": 69.6479721069336,
      "margin_dpo/margin_std": 99.48728942871094,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -119.2851791381836,
      "KL/mean": -150.42616271972656,
      "KL/rejected_KL_mean": -181.567138671875,
      "KL/std": 90.56817626953125,
      "epoch": 0.5905759162303665,
      "fcm_dpo/beta": 0.006628056056797504,
      "fcm_dpo/delta": 0.038609880954027176,
      "fcm_dpo/margin": 62.281951904296875,
      "fcm_dpo/q_t": 0.40877842903137207,
      "grad_norm": 94.15211486816406,
      "learning_rate": 2.1623435862645205e-07,
      "logits/chosen": -0.8507745265960693,
      "logits/rejected": -0.8515887260437012,
      "logps/chosen": -437.92236328125,
      "logps/ref_chosen": -318.63714599609375,
      "logps/ref_rejected": -273.5943603515625,
      "logps/rejected": -455.1614990234375,
      "loss": 4.5337,
      "margin_dpo/margin_mean": 62.28194808959961,
      "margin_dpo/margin_std": 104.3048324584961,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -117.24681091308594,
      "KL/mean": -147.70277404785156,
      "KL/rejected_KL_mean": -178.1587371826172,
      "KL/std": 86.45025634765625,
      "epoch": 0.5926701570680628,
      "fcm_dpo/beta": 0.006779513321816921,
      "fcm_dpo/delta": -0.022145334631204605,
      "fcm_dpo/margin": 60.911922454833984,
      "fcm_dpo/q_t": 0.411343514919281,
      "grad_norm": 105.9840316772461,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": -0.8697434067726135,
      "logits/rejected": -0.862332284450531,
      "logps/chosen": -371.9073486328125,
      "logps/ref_chosen": -254.66053771972656,
      "logps/ref_rejected": -236.8627166748047,
      "logps/rejected": -415.021484375,
      "loss": 4.5091,
      "margin_dpo/margin_mean": 60.91192626953125,
      "margin_dpo/margin_std": 98.8734359741211,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -121.23307037353516,
      "KL/mean": -153.86758422851562,
      "KL/rejected_KL_mean": -186.50205993652344,
      "KL/std": 94.1243896484375,
      "epoch": 0.5947643979057592,
      "fcm_dpo/beta": 0.006658874452114105,
      "fcm_dpo/delta": 0.014612892642617226,
      "fcm_dpo/margin": 65.26899719238281,
      "fcm_dpo/q_t": 0.405399352312088,
      "grad_norm": 105.75376892089844,
      "learning_rate": 2.1261013021512378e-07,
      "logits/chosen": -0.8383694291114807,
      "logits/rejected": -0.8124727606773376,
      "logps/chosen": -394.5882873535156,
      "logps/ref_chosen": -273.355224609375,
      "logps/ref_rejected": -259.84759521484375,
      "logps/rejected": -446.34967041015625,
      "loss": 4.5241,
      "margin_dpo/margin_mean": 65.26899719238281,
      "margin_dpo/margin_std": 110.904541015625,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -127.04911804199219,
      "KL/mean": -148.49026489257812,
      "KL/rejected_KL_mean": -169.93136596679688,
      "KL/std": 87.51554870605469,
      "epoch": 0.5968586387434555,
      "fcm_dpo/beta": 0.00694432994350791,
      "fcm_dpo/delta": 0.04382363334298134,
      "fcm_dpo/margin": 42.88224411010742,
      "fcm_dpo/q_t": 0.437032550573349,
      "grad_norm": 92.78446960449219,
      "learning_rate": 2.1080097510381294e-07,
      "logits/chosen": -0.8475319147109985,
      "logits/rejected": -0.8434734344482422,
      "logps/chosen": -436.8513488769531,
      "logps/ref_chosen": -309.8022155761719,
      "logps/ref_rejected": -279.11846923828125,
      "logps/rejected": -449.0498046875,
      "loss": 4.9684,
      "margin_dpo/margin_mean": 42.88224411010742,
      "margin_dpo/margin_std": 103.6301040649414,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -118.22410583496094,
      "KL/mean": -149.9862823486328,
      "KL/rejected_KL_mean": -181.74847412109375,
      "KL/std": 94.90242004394531,
      "epoch": 0.5989528795811518,
      "fcm_dpo/beta": 0.006961252074688673,
      "fcm_dpo/delta": 0.007788022514432669,
      "fcm_dpo/margin": 63.52438735961914,
      "fcm_dpo/q_t": 0.4074631333351135,
      "grad_norm": 113.81287384033203,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -0.8364225625991821,
      "logits/rejected": -0.8266147971153259,
      "logps/chosen": -389.68963623046875,
      "logps/ref_chosen": -271.4655456542969,
      "logps/ref_rejected": -279.531494140625,
      "logps/rejected": -461.2799987792969,
      "loss": 4.5142,
      "margin_dpo/margin_mean": 63.524391174316406,
      "margin_dpo/margin_std": 111.7081069946289,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -124.833740234375,
      "KL/mean": -155.52198791503906,
      "KL/rejected_KL_mean": -186.2102508544922,
      "KL/std": 93.08025360107422,
      "epoch": 0.6010471204188481,
      "fcm_dpo/beta": 0.006955859251320362,
      "fcm_dpo/delta": -0.006341751664876938,
      "fcm_dpo/margin": 61.37652587890625,
      "fcm_dpo/q_t": 0.40705883502960205,
      "grad_norm": 84.64332580566406,
      "learning_rate": 2.0718906816218595e-07,
      "logits/chosen": -0.8593530654907227,
      "logits/rejected": -0.8454005718231201,
      "logps/chosen": -401.9269714355469,
      "logps/ref_chosen": -277.0932312011719,
      "logps/ref_rejected": -233.55599975585938,
      "logps/rejected": -419.7662353515625,
      "loss": 4.5686,
      "margin_dpo/margin_mean": 61.376522064208984,
      "margin_dpo/margin_std": 109.669677734375,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -125.54894256591797,
      "KL/mean": -152.85414123535156,
      "KL/rejected_KL_mean": -180.1593475341797,
      "KL/std": 95.55374145507812,
      "epoch": 0.6031413612565445,
      "fcm_dpo/beta": 0.00713388342410326,
      "fcm_dpo/delta": 0.01798185333609581,
      "fcm_dpo/margin": 54.610416412353516,
      "fcm_dpo/q_t": 0.4157886207103729,
      "grad_norm": 94.07899475097656,
      "learning_rate": 2.053865100274774e-07,
      "logits/chosen": -0.8650684356689453,
      "logits/rejected": -0.8751712441444397,
      "logps/chosen": -418.71710205078125,
      "logps/ref_chosen": -293.1681823730469,
      "logps/ref_rejected": -263.4059143066406,
      "logps/rejected": -443.5653076171875,
      "loss": 4.6917,
      "margin_dpo/margin_mean": 54.610416412353516,
      "margin_dpo/margin_std": 107.31198120117188,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -144.72457885742188,
      "KL/mean": -164.7733154296875,
      "KL/rejected_KL_mean": -184.8220672607422,
      "KL/std": 90.48400115966797,
      "epoch": 0.6052356020942409,
      "fcm_dpo/beta": 0.007081322837620974,
      "fcm_dpo/delta": 0.024072987958788872,
      "fcm_dpo/margin": 40.097476959228516,
      "fcm_dpo/q_t": 0.4378480911254883,
      "grad_norm": 130.06109619140625,
      "learning_rate": 2.035863443788411e-07,
      "logits/chosen": -0.8468744158744812,
      "logits/rejected": -0.8306528329849243,
      "logps/chosen": -474.6820068359375,
      "logps/ref_chosen": -329.9574279785156,
      "logps/ref_rejected": -276.7565002441406,
      "logps/rejected": -461.5785217285156,
      "loss": 4.9801,
      "margin_dpo/margin_mean": 40.097476959228516,
      "margin_dpo/margin_std": 99.13330078125,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -131.35699462890625,
      "KL/mean": -161.4181671142578,
      "KL/rejected_KL_mean": -191.4793701171875,
      "KL/std": 95.55599212646484,
      "epoch": 0.6073298429319371,
      "fcm_dpo/beta": 0.00695389648899436,
      "fcm_dpo/delta": -0.023072054609656334,
      "fcm_dpo/margin": 60.12239074707031,
      "fcm_dpo/q_t": 0.4101371765136719,
      "grad_norm": 95.39649963378906,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": -0.8528121113777161,
      "logits/rejected": -0.7898657917976379,
      "logps/chosen": -456.0260314941406,
      "logps/ref_chosen": -324.6690673828125,
      "logps/ref_rejected": -311.8439636230469,
      "logps/rejected": -503.3233642578125,
      "loss": 4.6413,
      "margin_dpo/margin_mean": 60.12238311767578,
      "margin_dpo/margin_std": 111.91159057617188,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -125.91886901855469,
      "KL/mean": -165.8174591064453,
      "KL/rejected_KL_mean": -205.716064453125,
      "KL/std": 97.74503326416016,
      "epoch": 0.6094240837696335,
      "fcm_dpo/beta": 0.0066472506150603294,
      "fcm_dpo/delta": -0.12439057230949402,
      "fcm_dpo/margin": 79.79717254638672,
      "fcm_dpo/q_t": 0.3836360573768616,
      "grad_norm": 84.64372253417969,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -0.8293840289115906,
      "logits/rejected": -0.8205679655075073,
      "logps/chosen": -400.0629577636719,
      "logps/ref_chosen": -274.1440734863281,
      "logps/ref_rejected": -278.07208251953125,
      "logps/rejected": -483.78814697265625,
      "loss": 4.1822,
      "margin_dpo/margin_mean": 79.79717254638672,
      "margin_dpo/margin_std": 106.24543762207031,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -117.97250366210938,
      "KL/mean": -147.27548217773438,
      "KL/rejected_KL_mean": -176.5784912109375,
      "KL/std": 94.88096618652344,
      "epoch": 0.6115183246073298,
      "fcm_dpo/beta": 0.0065348464995622635,
      "fcm_dpo/delta": 0.0695800855755806,
      "fcm_dpo/margin": 58.6059684753418,
      "fcm_dpo/q_t": 0.4158637821674347,
      "grad_norm": 77.95936584472656,
      "learning_rate": 1.9820116705100775e-07,
      "logits/chosen": -0.8154031038284302,
      "logits/rejected": -0.8117830157279968,
      "logps/chosen": -377.336181640625,
      "logps/ref_chosen": -259.3636779785156,
      "logps/ref_rejected": -279.30218505859375,
      "logps/rejected": -455.88067626953125,
      "loss": 4.6349,
      "margin_dpo/margin_mean": 58.6059684753418,
      "margin_dpo/margin_std": 106.47883605957031,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -131.37161254882812,
      "KL/mean": -167.30047607421875,
      "KL/rejected_KL_mean": -203.22933959960938,
      "KL/std": 99.58857727050781,
      "epoch": 0.6136125654450262,
      "fcm_dpo/beta": 0.006449914537370205,
      "fcm_dpo/delta": -0.07908377796411514,
      "fcm_dpo/margin": 71.85774230957031,
      "fcm_dpo/q_t": 0.3974929749965668,
      "grad_norm": 81.66584014892578,
      "learning_rate": 1.9641153536023642e-07,
      "logits/chosen": -0.9161055088043213,
      "logits/rejected": -0.8763890266418457,
      "logps/chosen": -435.1424560546875,
      "logps/ref_chosen": -303.77081298828125,
      "logps/ref_rejected": -270.07513427734375,
      "logps/rejected": -473.304443359375,
      "loss": 4.3291,
      "margin_dpo/margin_mean": 71.85774230957031,
      "margin_dpo/margin_std": 99.857666015625,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -120.24039459228516,
      "KL/mean": -152.52105712890625,
      "KL/rejected_KL_mean": -184.8016815185547,
      "KL/std": 96.08721160888672,
      "epoch": 0.6157068062827226,
      "fcm_dpo/beta": 0.00640620244666934,
      "fcm_dpo/delta": -0.0015837140381336212,
      "fcm_dpo/margin": 64.5613021850586,
      "fcm_dpo/q_t": 0.41089755296707153,
      "grad_norm": 77.5408706665039,
      "learning_rate": 1.9462477745619106e-07,
      "logits/chosen": -0.8121069073677063,
      "logits/rejected": -0.8190856575965881,
      "logps/chosen": -360.47869873046875,
      "logps/ref_chosen": -240.23831176757812,
      "logps/ref_rejected": -229.187744140625,
      "logps/rejected": -413.98944091796875,
      "loss": 4.5525,
      "margin_dpo/margin_mean": 64.5613021850586,
      "margin_dpo/margin_std": 111.48363494873047,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -124.74127197265625,
      "KL/mean": -154.8818817138672,
      "KL/rejected_KL_mean": -185.0224609375,
      "KL/std": 86.85227966308594,
      "epoch": 0.6178010471204188,
      "fcm_dpo/beta": 0.006512588821351528,
      "fcm_dpo/delta": 0.05958293005824089,
      "fcm_dpo/margin": 60.281211853027344,
      "fcm_dpo/q_t": 0.4139400124549866,
      "grad_norm": 69.21749877929688,
      "learning_rate": 1.928409891572757e-07,
      "logits/chosen": -0.8129090666770935,
      "logits/rejected": -0.8243449926376343,
      "logps/chosen": -375.7509765625,
      "logps/ref_chosen": -251.00970458984375,
      "logps/ref_rejected": -244.15142822265625,
      "logps/rejected": -429.17388916015625,
      "loss": 4.5495,
      "margin_dpo/margin_mean": 60.281211853027344,
      "margin_dpo/margin_std": 101.69961547851562,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -126.69507598876953,
      "KL/mean": -167.6036376953125,
      "KL/rejected_KL_mean": -208.51220703125,
      "KL/std": 97.69671630859375,
      "epoch": 0.6198952879581152,
      "fcm_dpo/beta": 0.00637055141851306,
      "fcm_dpo/delta": -0.07700033485889435,
      "fcm_dpo/margin": 81.81713104248047,
      "fcm_dpo/q_t": 0.38465678691864014,
      "grad_norm": 70.81564331054688,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -0.7942947745323181,
      "logits/rejected": -0.7639515399932861,
      "logps/chosen": -420.57568359375,
      "logps/ref_chosen": -293.880615234375,
      "logps/ref_rejected": -283.4175720214844,
      "logps/rejected": -491.9298095703125,
      "loss": 4.2619,
      "margin_dpo/margin_mean": 81.81713104248047,
      "margin_dpo/margin_std": 116.40140533447266,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -126.47355651855469,
      "KL/mean": -157.11270141601562,
      "KL/rejected_KL_mean": -187.75184631347656,
      "KL/std": 91.83148956298828,
      "epoch": 0.6219895287958115,
      "fcm_dpo/beta": 0.00614601606503129,
      "fcm_dpo/delta": 0.0018921680748462677,
      "fcm_dpo/margin": 61.27829360961914,
      "fcm_dpo/q_t": 0.41599759459495544,
      "grad_norm": 82.61782836914062,
      "learning_rate": 1.8928270384706582e-07,
      "logits/chosen": -0.890099287033081,
      "logits/rejected": -0.884902834892273,
      "logps/chosen": -415.93365478515625,
      "logps/ref_chosen": -289.4600830078125,
      "logps/ref_rejected": -283.69110107421875,
      "logps/rejected": -471.44293212890625,
      "loss": 4.5687,
      "margin_dpo/margin_mean": 61.27829360961914,
      "margin_dpo/margin_std": 99.77861022949219,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -116.15878295898438,
      "KL/mean": -150.7476043701172,
      "KL/rejected_KL_mean": -185.33644104003906,
      "KL/std": 90.60035705566406,
      "epoch": 0.6240837696335079,
      "fcm_dpo/beta": 0.006125608924776316,
      "fcm_dpo/delta": -0.05966220796108246,
      "fcm_dpo/margin": 69.17765045166016,
      "fcm_dpo/q_t": 0.40633291006088257,
      "grad_norm": 91.83101654052734,
      "learning_rate": 1.875083976558136e-07,
      "logits/chosen": -0.8133167028427124,
      "logits/rejected": -0.802819013595581,
      "logps/chosen": -422.673828125,
      "logps/ref_chosen": -306.5150146484375,
      "logps/ref_rejected": -280.6969909667969,
      "logps/rejected": -466.033447265625,
      "loss": 4.5212,
      "margin_dpo/margin_mean": 69.17765045166016,
      "margin_dpo/margin_std": 113.73353576660156,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -137.9058380126953,
      "KL/mean": -166.79852294921875,
      "KL/rejected_KL_mean": -195.6912078857422,
      "KL/std": 94.7004623413086,
      "epoch": 0.6261780104712041,
      "fcm_dpo/beta": 0.005867868196219206,
      "fcm_dpo/delta": -0.016072731465101242,
      "fcm_dpo/margin": 57.78535842895508,
      "fcm_dpo/q_t": 0.4231151044368744,
      "grad_norm": 80.73125457763672,
      "learning_rate": 1.8573744269954297e-07,
      "logits/chosen": -0.7859805822372437,
      "logits/rejected": -0.7729779481887817,
      "logps/chosen": -419.2696228027344,
      "logps/ref_chosen": -281.36376953125,
      "logps/ref_rejected": -270.39508056640625,
      "logps/rejected": -466.0862731933594,
      "loss": 4.6596,
      "margin_dpo/margin_mean": 57.785362243652344,
      "margin_dpo/margin_std": 96.48539733886719,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -136.34219360351562,
      "KL/mean": -165.18936157226562,
      "KL/rejected_KL_mean": -194.03652954101562,
      "KL/std": 97.7120361328125,
      "epoch": 0.6282722513089005,
      "fcm_dpo/beta": 0.006217634305357933,
      "fcm_dpo/delta": 0.09342236071825027,
      "fcm_dpo/margin": 57.6943359375,
      "fcm_dpo/q_t": 0.4204936623573303,
      "grad_norm": 86.95182800292969,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": -0.8234930038452148,
      "logits/rejected": -0.8005295991897583,
      "logps/chosen": -451.17791748046875,
      "logps/ref_chosen": -314.83575439453125,
      "logps/ref_rejected": -269.1154479980469,
      "logps/rejected": -463.1519775390625,
      "loss": 4.6977,
      "margin_dpo/margin_mean": 57.694339752197266,
      "margin_dpo/margin_std": 111.31985473632812,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -128.43812561035156,
      "KL/mean": -159.7366943359375,
      "KL/rejected_KL_mean": -191.0352783203125,
      "KL/std": 95.1756362915039,
      "epoch": 0.6303664921465969,
      "fcm_dpo/beta": 0.006506850477308035,
      "fcm_dpo/delta": 0.04372243583202362,
      "fcm_dpo/margin": 62.597137451171875,
      "fcm_dpo/q_t": 0.4100876450538635,
      "grad_norm": 77.13524627685547,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -0.8473941087722778,
      "logits/rejected": -0.8528563380241394,
      "logps/chosen": -408.3326416015625,
      "logps/ref_chosen": -279.89453125,
      "logps/ref_rejected": -271.6694641113281,
      "logps/rejected": -462.7047424316406,
      "loss": 4.5302,
      "margin_dpo/margin_mean": 62.59714126586914,
      "margin_dpo/margin_std": 104.21112823486328,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -126.29755401611328,
      "KL/mean": -163.54808044433594,
      "KL/rejected_KL_mean": -200.79859924316406,
      "KL/std": 100.25669860839844,
      "epoch": 0.6324607329842932,
      "fcm_dpo/beta": 0.006507527083158493,
      "fcm_dpo/delta": -0.03690612316131592,
      "fcm_dpo/margin": 74.50103759765625,
      "fcm_dpo/q_t": 0.3939831852912903,
      "grad_norm": 132.27984619140625,
      "learning_rate": 1.8044563402088682e-07,
      "logits/chosen": -0.806928813457489,
      "logits/rejected": -0.792854368686676,
      "logps/chosen": -397.6293640136719,
      "logps/ref_chosen": -271.3318176269531,
      "logps/ref_rejected": -256.5587158203125,
      "logps/rejected": -457.3572998046875,
      "loss": 4.3124,
      "margin_dpo/margin_mean": 74.50103759765625,
      "margin_dpo/margin_std": 109.38188171386719,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -130.99850463867188,
      "KL/mean": -164.8486328125,
      "KL/rejected_KL_mean": -198.69876098632812,
      "KL/std": 95.4461898803711,
      "epoch": 0.6345549738219896,
      "fcm_dpo/beta": 0.006358322221785784,
      "fcm_dpo/delta": -0.03674698993563652,
      "fcm_dpo/margin": 67.70026397705078,
      "fcm_dpo/q_t": 0.40360143780708313,
      "grad_norm": 81.8840103149414,
      "learning_rate": 1.7868903184043885e-07,
      "logits/chosen": -0.7925262451171875,
      "logits/rejected": -0.7735260725021362,
      "logps/chosen": -435.8795166015625,
      "logps/ref_chosen": -304.88104248046875,
      "logps/ref_rejected": -269.063720703125,
      "logps/rejected": -467.76251220703125,
      "loss": 4.4801,
      "margin_dpo/margin_mean": 67.70026397705078,
      "margin_dpo/margin_std": 107.40185546875,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -140.90077209472656,
      "KL/mean": -177.681396484375,
      "KL/rejected_KL_mean": -214.4619903564453,
      "KL/std": 107.98794555664062,
      "epoch": 0.6366492146596858,
      "fcm_dpo/beta": 0.00616060383617878,
      "fcm_dpo/delta": -0.0037967022508382797,
      "fcm_dpo/margin": 73.56123352050781,
      "fcm_dpo/q_t": 0.4018302261829376,
      "grad_norm": 79.75228118896484,
      "learning_rate": 1.7693625385079574e-07,
      "logits/chosen": -0.8166980743408203,
      "logits/rejected": -0.8301171064376831,
      "logps/chosen": -431.61175537109375,
      "logps/ref_chosen": -290.7109680175781,
      "logps/ref_rejected": -237.6885986328125,
      "logps/rejected": -452.1506042480469,
      "loss": 4.3959,
      "margin_dpo/margin_mean": 73.56124114990234,
      "margin_dpo/margin_std": 115.10319519042969,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -118.37123107910156,
      "KL/mean": -171.87124633789062,
      "KL/rejected_KL_mean": -225.37130737304688,
      "KL/std": 106.05049896240234,
      "epoch": 0.6387434554973822,
      "fcm_dpo/beta": 0.005740244407206774,
      "fcm_dpo/delta": -0.17579856514930725,
      "fcm_dpo/margin": 107.00006866455078,
      "fcm_dpo/q_t": 0.3632907569408417,
      "grad_norm": 97.41038513183594,
      "learning_rate": 1.7518739404812155e-07,
      "logits/chosen": -0.865484356880188,
      "logits/rejected": -0.8350971341133118,
      "logps/chosen": -374.855224609375,
      "logps/ref_chosen": -256.4839782714844,
      "logps/ref_rejected": -266.4063415527344,
      "logps/rejected": -491.777587890625,
      "loss": 3.891,
      "margin_dpo/margin_mean": 107.00006866455078,
      "margin_dpo/margin_std": 115.42047882080078,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -133.56973266601562,
      "KL/mean": -166.0826873779297,
      "KL/rejected_KL_mean": -198.59564208984375,
      "KL/std": 96.57077026367188,
      "epoch": 0.6408376963350786,
      "fcm_dpo/beta": 0.005324847064912319,
      "fcm_dpo/delta": -0.018631238490343094,
      "fcm_dpo/margin": 65.02589416503906,
      "fcm_dpo/q_t": 0.42107054591178894,
      "grad_norm": 96.88826751708984,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -0.855994701385498,
      "logits/rejected": -0.8400151133537292,
      "logps/chosen": -454.218994140625,
      "logps/ref_chosen": -320.6492004394531,
      "logps/ref_rejected": -273.36773681640625,
      "logps/rejected": -471.96337890625,
      "loss": 4.6054,
      "margin_dpo/margin_mean": 65.02589416503906,
      "margin_dpo/margin_std": 103.44615173339844,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -155.01339721679688,
      "KL/mean": -189.87005615234375,
      "KL/rejected_KL_mean": -224.72669982910156,
      "KL/std": 104.81556701660156,
      "epoch": 0.6429319371727749,
      "fcm_dpo/beta": 0.005341042298823595,
      "fcm_dpo/delta": 0.018805336207151413,
      "fcm_dpo/margin": 69.71328735351562,
      "fcm_dpo/q_t": 0.4142070412635803,
      "grad_norm": 97.63721466064453,
      "learning_rate": 1.717018039327053e-07,
      "logits/chosen": -0.7859216928482056,
      "logits/rejected": -0.8188137412071228,
      "logps/chosen": -434.46759033203125,
      "logps/ref_chosen": -279.4541931152344,
      "logps/ref_rejected": -240.3796844482422,
      "logps/rejected": -465.10638427734375,
      "loss": 4.517,
      "margin_dpo/margin_mean": 69.71328735351562,
      "margin_dpo/margin_std": 103.92877197265625,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -162.23863220214844,
      "KL/mean": -187.41622924804688,
      "KL/rejected_KL_mean": -212.59381103515625,
      "KL/std": 103.67611694335938,
      "epoch": 0.6450261780104712,
      "fcm_dpo/beta": 0.005620987620204687,
      "fcm_dpo/delta": 0.11048682034015656,
      "fcm_dpo/margin": 50.355186462402344,
      "fcm_dpo/q_t": 0.43715929985046387,
      "grad_norm": 84.04242706298828,
      "learning_rate": 1.699652605415828e-07,
      "logits/chosen": -0.8382729887962341,
      "logits/rejected": -0.8537087440490723,
      "logps/chosen": -458.8370056152344,
      "logps/ref_chosen": -296.598388671875,
      "logps/ref_rejected": -258.6953430175781,
      "logps/rejected": -471.28912353515625,
      "loss": 4.8974,
      "margin_dpo/margin_mean": 50.35519027709961,
      "margin_dpo/margin_std": 113.58354187011719,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -149.81771850585938,
      "KL/mean": -191.1851348876953,
      "KL/rejected_KL_mean": -232.55258178710938,
      "KL/std": 101.70248413085938,
      "epoch": 0.6471204188481675,
      "fcm_dpo/beta": 0.005824322812259197,
      "fcm_dpo/delta": -0.03359182924032211,
      "fcm_dpo/margin": 82.73484802246094,
      "fcm_dpo/q_t": 0.3939167261123657,
      "grad_norm": 89.10334014892578,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": -0.8472768664360046,
      "logits/rejected": -0.8594540357589722,
      "logps/chosen": -431.20587158203125,
      "logps/ref_chosen": -281.3881530761719,
      "logps/ref_rejected": -262.458740234375,
      "logps/rejected": -495.0112609863281,
      "loss": 4.2927,
      "margin_dpo/margin_mean": 82.73484802246094,
      "margin_dpo/margin_std": 119.54065704345703,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -145.44532775878906,
      "KL/mean": -184.14138793945312,
      "KL/rejected_KL_mean": -222.8374481201172,
      "KL/std": 108.22349548339844,
      "epoch": 0.6492146596858639,
      "fcm_dpo/beta": 0.005782703403383493,
      "fcm_dpo/delta": 0.0017282813787460327,
      "fcm_dpo/margin": 77.39212799072266,
      "fcm_dpo/q_t": 0.40068820118904114,
      "grad_norm": 131.081787109375,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": -0.8364096879959106,
      "logits/rejected": -0.812505304813385,
      "logps/chosen": -424.632568359375,
      "logps/ref_chosen": -279.1872863769531,
      "logps/ref_rejected": -261.8279724121094,
      "logps/rejected": -484.6654052734375,
      "loss": 4.3455,
      "margin_dpo/margin_mean": 77.39212799072266,
      "margin_dpo/margin_std": 109.58356475830078,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -148.7241973876953,
      "KL/mean": -187.99012756347656,
      "KL/rejected_KL_mean": -227.25604248046875,
      "KL/std": 106.33953094482422,
      "epoch": 0.6513089005235602,
      "fcm_dpo/beta": 0.005726814270019531,
      "fcm_dpo/delta": -0.0013048239052295685,
      "fcm_dpo/margin": 78.53182220458984,
      "fcm_dpo/q_t": 0.3983256220817566,
      "grad_norm": 87.45763397216797,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -0.8350776433944702,
      "logits/rejected": -0.817095160484314,
      "logps/chosen": -420.122314453125,
      "logps/ref_chosen": -271.39813232421875,
      "logps/ref_rejected": -266.12701416015625,
      "logps/rejected": -493.383056640625,
      "loss": 4.3831,
      "margin_dpo/margin_mean": 78.53181457519531,
      "margin_dpo/margin_std": 114.00733947753906,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -147.848876953125,
      "KL/mean": -182.12692260742188,
      "KL/rejected_KL_mean": -216.40496826171875,
      "KL/std": 103.54866027832031,
      "epoch": 0.6534031413612565,
      "fcm_dpo/beta": 0.005715105216950178,
      "fcm_dpo/delta": -0.0106766102835536,
      "fcm_dpo/margin": 68.55606079101562,
      "fcm_dpo/q_t": 0.4151096045970917,
      "grad_norm": 100.2896499633789,
      "learning_rate": 1.6306293495205755e-07,
      "logits/chosen": -0.8407636880874634,
      "logits/rejected": -0.8186388611793518,
      "logps/chosen": -430.23388671875,
      "logps/ref_chosen": -282.3850402832031,
      "logps/ref_rejected": -246.35389709472656,
      "logps/rejected": -462.75885009765625,
      "loss": 4.6939,
      "margin_dpo/margin_mean": 68.55606079101562,
      "margin_dpo/margin_std": 131.81167602539062,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -150.44068908691406,
      "KL/mean": -185.8656463623047,
      "KL/rejected_KL_mean": -221.29052734375,
      "KL/std": 109.65206909179688,
      "epoch": 0.6554973821989529,
      "fcm_dpo/beta": 0.005781089887022972,
      "fcm_dpo/delta": -0.00206906720995903,
      "fcm_dpo/margin": 70.84982299804688,
      "fcm_dpo/q_t": 0.41273972392082214,
      "grad_norm": 72.69104766845703,
      "learning_rate": 1.6134877823936607e-07,
      "logits/chosen": -0.8899011611938477,
      "logits/rejected": -0.8796027898788452,
      "logps/chosen": -454.0715637207031,
      "logps/ref_chosen": -303.630859375,
      "logps/ref_rejected": -273.1156921386719,
      "logps/rejected": -494.4062194824219,
      "loss": 4.5959,
      "margin_dpo/margin_mean": 70.84982299804688,
      "margin_dpo/margin_std": 123.04818725585938,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -138.6251983642578,
      "KL/mean": -175.03363037109375,
      "KL/rejected_KL_mean": -211.44204711914062,
      "KL/std": 99.44528198242188,
      "epoch": 0.6575916230366492,
      "fcm_dpo/beta": 0.005836997181177139,
      "fcm_dpo/delta": 0.024851497262716293,
      "fcm_dpo/margin": 72.81683349609375,
      "fcm_dpo/q_t": 0.4041312038898468,
      "grad_norm": 72.47408294677734,
      "learning_rate": 1.5963937562265522e-07,
      "logits/chosen": -0.8888888359069824,
      "logits/rejected": -0.875575065612793,
      "logps/chosen": -440.92950439453125,
      "logps/ref_chosen": -302.3042907714844,
      "logps/ref_rejected": -273.6416015625,
      "logps/rejected": -485.0836486816406,
      "loss": 4.4183,
      "margin_dpo/margin_mean": 72.81684112548828,
      "margin_dpo/margin_std": 108.23794555664062,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -138.11972045898438,
      "KL/mean": -178.6306915283203,
      "KL/rejected_KL_mean": -219.1416473388672,
      "KL/std": 100.35026550292969,
      "epoch": 0.6596858638743456,
      "fcm_dpo/beta": 0.005759485997259617,
      "fcm_dpo/delta": -0.018444210290908813,
      "fcm_dpo/margin": 81.02194213867188,
      "fcm_dpo/q_t": 0.395830899477005,
      "grad_norm": 94.68378448486328,
      "learning_rate": 1.5793481877199943e-07,
      "logits/chosen": -0.8739151954650879,
      "logits/rejected": -0.8597660660743713,
      "logps/chosen": -440.8489685058594,
      "logps/ref_chosen": -302.729248046875,
      "logps/ref_rejected": -270.26910400390625,
      "logps/rejected": -489.4107360839844,
      "loss": 4.3108,
      "margin_dpo/margin_mean": 81.02194213867188,
      "margin_dpo/margin_std": 112.66746520996094,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -140.5756378173828,
      "KL/mean": -178.59934997558594,
      "KL/rejected_KL_mean": -216.62306213378906,
      "KL/std": 100.80996704101562,
      "epoch": 0.6617801047120419,
      "fcm_dpo/beta": 0.005732722580432892,
      "fcm_dpo/delta": 0.013753227889537811,
      "fcm_dpo/margin": 76.04742431640625,
      "fcm_dpo/q_t": 0.4038986563682556,
      "grad_norm": 79.04898834228516,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -0.8868338465690613,
      "logits/rejected": -0.8746849894523621,
      "logps/chosen": -451.1462707519531,
      "logps/ref_chosen": -310.5706481933594,
      "logps/ref_rejected": -272.9354553222656,
      "logps/rejected": -489.5585021972656,
      "loss": 4.401,
      "margin_dpo/margin_mean": 76.04741668701172,
      "margin_dpo/margin_std": 113.82899475097656,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -119.98965454101562,
      "KL/mean": -157.03939819335938,
      "KL/rejected_KL_mean": -194.08912658691406,
      "KL/std": 92.58094024658203,
      "epoch": 0.6638743455497382,
      "fcm_dpo/beta": 0.005786577705293894,
      "fcm_dpo/delta": 0.02174123376607895,
      "fcm_dpo/margin": 74.0994644165039,
      "fcm_dpo/q_t": 0.40227121114730835,
      "grad_norm": 107.0450210571289,
      "learning_rate": 1.5454060774493065e-07,
      "logits/chosen": -0.8730836510658264,
      "logits/rejected": -0.842927098274231,
      "logps/chosen": -373.8900146484375,
      "logps/ref_chosen": -253.90036010742188,
      "logps/ref_rejected": -218.74078369140625,
      "logps/rejected": -412.8299255371094,
      "loss": 4.316,
      "margin_dpo/margin_mean": 74.09947204589844,
      "margin_dpo/margin_std": 94.06819915771484,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -126.95587158203125,
      "KL/mean": -171.29330444335938,
      "KL/rejected_KL_mean": -215.6306915283203,
      "KL/std": 98.33518981933594,
      "epoch": 0.6659685863874345,
      "fcm_dpo/beta": 0.005634589586406946,
      "fcm_dpo/delta": -0.05430690571665764,
      "fcm_dpo/margin": 88.67483520507812,
      "fcm_dpo/q_t": 0.38681909441947937,
      "grad_norm": 72.1392593383789,
      "learning_rate": 1.5285113558975427e-07,
      "logits/chosen": -0.8906688690185547,
      "logits/rejected": -0.8583005666732788,
      "logps/chosen": -397.7787780761719,
      "logps/ref_chosen": -270.8228759765625,
      "logps/ref_rejected": -255.30972290039062,
      "logps/rejected": -470.9404296875,
      "loss": 4.1316,
      "margin_dpo/margin_mean": 88.67483520507812,
      "margin_dpo/margin_std": 103.5236587524414,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -138.37754821777344,
      "KL/mean": -177.607421875,
      "KL/rejected_KL_mean": -216.8372802734375,
      "KL/std": 97.69280242919922,
      "epoch": 0.6680628272251309,
      "fcm_dpo/beta": 0.005532895214855671,
      "fcm_dpo/delta": 0.016384322196245193,
      "fcm_dpo/margin": 78.4597396850586,
      "fcm_dpo/q_t": 0.40047621726989746,
      "grad_norm": 117.88058471679688,
      "learning_rate": 1.5116687323334464e-07,
      "logits/chosen": -0.8735268712043762,
      "logits/rejected": -0.8470640182495117,
      "logps/chosen": -439.38037109375,
      "logps/ref_chosen": -301.0028076171875,
      "logps/ref_rejected": -242.39002990722656,
      "logps/rejected": -459.227294921875,
      "loss": 4.2872,
      "margin_dpo/margin_mean": 78.4597396850586,
      "margin_dpo/margin_std": 98.05628967285156,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -134.79701232910156,
      "KL/mean": -171.5314483642578,
      "KL/rejected_KL_mean": -208.26588439941406,
      "KL/std": 102.06668853759766,
      "epoch": 0.6701570680628273,
      "fcm_dpo/beta": 0.005700442939996719,
      "fcm_dpo/delta": 0.03195666894316673,
      "fcm_dpo/margin": 73.46886444091797,
      "fcm_dpo/q_t": 0.4078993797302246,
      "grad_norm": 105.50513458251953,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": -0.8399797677993774,
      "logits/rejected": -0.8374543786048889,
      "logps/chosen": -438.4195861816406,
      "logps/ref_chosen": -303.6225891113281,
      "logps/ref_rejected": -280.85174560546875,
      "logps/rejected": -489.11761474609375,
      "loss": 4.5165,
      "margin_dpo/margin_mean": 73.46886444091797,
      "margin_dpo/margin_std": 123.44187927246094,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -148.86854553222656,
      "KL/mean": -177.0805206298828,
      "KL/rejected_KL_mean": -205.29244995117188,
      "KL/std": 101.67632293701172,
      "epoch": 0.6722513089005235,
      "fcm_dpo/beta": 0.00595706794410944,
      "fcm_dpo/delta": 0.03193975239992142,
      "fcm_dpo/margin": 56.423927307128906,
      "fcm_dpo/q_t": 0.42655009031295776,
      "grad_norm": 90.00570678710938,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -0.8735283613204956,
      "logits/rejected": -0.8401560187339783,
      "logps/chosen": -437.8543701171875,
      "logps/ref_chosen": -288.98583984375,
      "logps/ref_rejected": -241.1822052001953,
      "logps/rejected": -446.4747009277344,
      "loss": 4.7412,
      "margin_dpo/margin_mean": 56.423927307128906,
      "margin_dpo/margin_std": 110.67036437988281,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -149.05581665039062,
      "KL/mean": -184.38429260253906,
      "KL/rejected_KL_mean": -219.71275329589844,
      "KL/std": 103.64341735839844,
      "epoch": 0.6743455497382199,
      "fcm_dpo/beta": 0.005965797230601311,
      "fcm_dpo/delta": -0.024056755006313324,
      "fcm_dpo/margin": 70.656982421875,
      "fcm_dpo/q_t": 0.4074922800064087,
      "grad_norm": 72.17630767822266,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": -0.9292232394218445,
      "logits/rejected": -0.889703631401062,
      "logps/chosen": -457.5992736816406,
      "logps/ref_chosen": -308.54345703125,
      "logps/ref_rejected": -269.7995910644531,
      "logps/rejected": -489.5123596191406,
      "loss": 4.4806,
      "margin_dpo/margin_mean": 70.656982421875,
      "margin_dpo/margin_std": 114.64525604248047,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -145.44345092773438,
      "KL/mean": -170.13607788085938,
      "KL/rejected_KL_mean": -194.82875061035156,
      "KL/std": 92.9245834350586,
      "epoch": 0.6764397905759162,
      "fcm_dpo/beta": 0.006021259352564812,
      "fcm_dpo/delta": 0.04875420406460762,
      "fcm_dpo/margin": 49.38530731201172,
      "fcm_dpo/q_t": 0.43559253215789795,
      "grad_norm": 105.05410766601562,
      "learning_rate": 1.4448372394055246e-07,
      "logits/chosen": -0.8854071497917175,
      "logits/rejected": -0.8787587881088257,
      "logps/chosen": -427.93707275390625,
      "logps/ref_chosen": -282.49365234375,
      "logps/ref_rejected": -227.7105255126953,
      "logps/rejected": -422.539306640625,
      "loss": 4.8916,
      "margin_dpo/margin_mean": 49.38531494140625,
      "margin_dpo/margin_std": 113.13801574707031,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -131.50189208984375,
      "KL/mean": -172.75851440429688,
      "KL/rejected_KL_mean": -214.01510620117188,
      "KL/std": 100.42484283447266,
      "epoch": 0.6785340314136126,
      "fcm_dpo/beta": 0.005899250041693449,
      "fcm_dpo/delta": -0.04072398692369461,
      "fcm_dpo/margin": 82.51322937011719,
      "fcm_dpo/q_t": 0.3896920084953308,
      "grad_norm": 80.49970245361328,
      "learning_rate": 1.428268596492364e-07,
      "logits/chosen": -0.8135544657707214,
      "logits/rejected": -0.8090764284133911,
      "logps/chosen": -370.84027099609375,
      "logps/ref_chosen": -239.33836364746094,
      "logps/ref_rejected": -230.53775024414062,
      "logps/rejected": -444.5528259277344,
      "loss": 4.1787,
      "margin_dpo/margin_mean": 82.51322174072266,
      "margin_dpo/margin_std": 99.66815185546875,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -145.798828125,
      "KL/mean": -181.84898376464844,
      "KL/rejected_KL_mean": -217.899169921875,
      "KL/std": 103.4723129272461,
      "epoch": 0.680628272251309,
      "fcm_dpo/beta": 0.005812216550111771,
      "fcm_dpo/delta": -0.04172111675143242,
      "fcm_dpo/margin": 72.1003189086914,
      "fcm_dpo/q_t": 0.40836799144744873,
      "grad_norm": 77.4819107055664,
      "learning_rate": 1.4117574272818386e-07,
      "logits/chosen": -0.8233944177627563,
      "logits/rejected": -0.807444155216217,
      "logps/chosen": -426.4278259277344,
      "logps/ref_chosen": -280.62896728515625,
      "logps/ref_rejected": -270.5085754394531,
      "logps/rejected": -488.40771484375,
      "loss": 4.5259,
      "margin_dpo/margin_mean": 72.1003189086914,
      "margin_dpo/margin_std": 120.4211654663086,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -134.10484313964844,
      "KL/mean": -170.2986602783203,
      "KL/rejected_KL_mean": -206.4924774169922,
      "KL/std": 96.89154052734375,
      "epoch": 0.6827225130890052,
      "fcm_dpo/beta": 0.005846397951245308,
      "fcm_dpo/delta": 0.026988931000232697,
      "fcm_dpo/margin": 72.38761901855469,
      "fcm_dpo/q_t": 0.4050830006599426,
      "grad_norm": 91.48040008544922,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -0.9331142902374268,
      "logits/rejected": -0.9238046407699585,
      "logps/chosen": -375.09197998046875,
      "logps/ref_chosen": -240.9871368408203,
      "logps/ref_rejected": -261.0238342285156,
      "logps/rejected": -467.51629638671875,
      "loss": 4.4286,
      "margin_dpo/margin_mean": 72.38761901855469,
      "margin_dpo/margin_std": 108.98831939697266,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -126.99838256835938,
      "KL/mean": -167.4870147705078,
      "KL/rejected_KL_mean": -207.97560119628906,
      "KL/std": 91.84928131103516,
      "epoch": 0.6848167539267016,
      "fcm_dpo/beta": 0.005785267800092697,
      "fcm_dpo/delta": -0.020716046914458275,
      "fcm_dpo/margin": 80.97723388671875,
      "fcm_dpo/q_t": 0.3955627977848053,
      "grad_norm": 76.74833679199219,
      "learning_rate": 1.3789110486146468e-07,
      "logits/chosen": -0.8904739618301392,
      "logits/rejected": -0.8675505518913269,
      "logps/chosen": -406.51837158203125,
      "logps/ref_chosen": -279.52001953125,
      "logps/ref_rejected": -269.51824951171875,
      "logps/rejected": -477.4938659667969,
      "loss": 4.2735,
      "margin_dpo/margin_mean": 80.97722625732422,
      "margin_dpo/margin_std": 108.63449096679688,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -143.81936645507812,
      "KL/mean": -180.50765991210938,
      "KL/rejected_KL_mean": -217.19595336914062,
      "KL/std": 95.98664093017578,
      "epoch": 0.6869109947643979,
      "fcm_dpo/beta": 0.005601266399025917,
      "fcm_dpo/delta": 0.003683714661747217,
      "fcm_dpo/margin": 73.3765869140625,
      "fcm_dpo/q_t": 0.40680986642837524,
      "grad_norm": 90.79134368896484,
      "learning_rate": 1.362577600609588e-07,
      "logits/chosen": -0.8505688905715942,
      "logits/rejected": -0.8505761623382568,
      "logps/chosen": -444.8528137207031,
      "logps/ref_chosen": -301.033447265625,
      "logps/ref_rejected": -284.2101135253906,
      "logps/rejected": -501.40606689453125,
      "loss": 4.395,
      "margin_dpo/margin_mean": 73.3765869140625,
      "margin_dpo/margin_std": 100.12046813964844,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -154.32469177246094,
      "KL/mean": -194.8263397216797,
      "KL/rejected_KL_mean": -235.32797241210938,
      "KL/std": 119.51981353759766,
      "epoch": 0.6890052356020943,
      "fcm_dpo/beta": 0.005723894573748112,
      "fcm_dpo/delta": -0.015375969931483269,
      "fcm_dpo/margin": 81.00324249267578,
      "fcm_dpo/q_t": 0.4020324647426605,
      "grad_norm": 113.09385681152344,
      "learning_rate": 1.3463051491159093e-07,
      "logits/chosen": -0.865266740322113,
      "logits/rejected": -0.8382850289344788,
      "logps/chosen": -474.31353759765625,
      "logps/ref_chosen": -319.9888610839844,
      "logps/ref_rejected": -307.5588684082031,
      "logps/rejected": -542.8868408203125,
      "loss": 4.4273,
      "margin_dpo/margin_mean": 81.00324249267578,
      "margin_dpo/margin_std": 129.8346710205078,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -154.2286376953125,
      "KL/mean": -190.36618041992188,
      "KL/rejected_KL_mean": -226.50372314453125,
      "KL/std": 106.14683532714844,
      "epoch": 0.6910994764397905,
      "fcm_dpo/beta": 0.005737834610044956,
      "fcm_dpo/delta": -0.0003470419906079769,
      "fcm_dpo/margin": 72.27506256103516,
      "fcm_dpo/q_t": 0.4071575701236725,
      "grad_norm": 86.67559051513672,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": -0.8535846471786499,
      "logits/rejected": -0.8648088574409485,
      "logps/chosen": -455.3433837890625,
      "logps/ref_chosen": -301.11474609375,
      "logps/ref_rejected": -299.673095703125,
      "logps/rejected": -526.1768188476562,
      "loss": 4.4356,
      "margin_dpo/margin_mean": 72.27506256103516,
      "margin_dpo/margin_std": 106.54840087890625,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -143.23675537109375,
      "KL/mean": -179.8782196044922,
      "KL/rejected_KL_mean": -216.51971435546875,
      "KL/std": 108.74295043945312,
      "epoch": 0.6931937172774869,
      "fcm_dpo/beta": 0.005723595153540373,
      "fcm_dpo/delta": 0.03176797181367874,
      "fcm_dpo/margin": 73.2829360961914,
      "fcm_dpo/q_t": 0.4092589318752289,
      "grad_norm": 113.3475570678711,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -0.8881016373634338,
      "logits/rejected": -0.8705732822418213,
      "logps/chosen": -420.82818603515625,
      "logps/ref_chosen": -277.59149169921875,
      "logps/ref_rejected": -256.025634765625,
      "logps/rejected": -472.54534912109375,
      "loss": 4.532,
      "margin_dpo/margin_mean": 73.2829360961914,
      "margin_dpo/margin_std": 127.43795776367188,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -154.5018310546875,
      "KL/mean": -191.70379638671875,
      "KL/rejected_KL_mean": -228.90576171875,
      "KL/std": 100.953857421875,
      "epoch": 0.6952879581151833,
      "fcm_dpo/beta": 0.005800623446702957,
      "fcm_dpo/delta": 0.018545489758253098,
      "fcm_dpo/margin": 74.40392303466797,
      "fcm_dpo/q_t": 0.40669822692871094,
      "grad_norm": 90.81681823730469,
      "learning_rate": 1.2978624834891626e-07,
      "logits/chosen": -0.8917899131774902,
      "logits/rejected": -0.8668915629386902,
      "logps/chosen": -424.47552490234375,
      "logps/ref_chosen": -269.97369384765625,
      "logps/ref_rejected": -235.03164672851562,
      "logps/rejected": -463.9373779296875,
      "loss": 4.5021,
      "margin_dpo/margin_mean": 74.4039306640625,
      "margin_dpo/margin_std": 124.41642761230469,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -154.95396423339844,
      "KL/mean": -188.54953002929688,
      "KL/rejected_KL_mean": -222.1450653076172,
      "KL/std": 100.42887115478516,
      "epoch": 0.6973821989528796,
      "fcm_dpo/beta": 0.005991967860609293,
      "fcm_dpo/delta": 0.011700914241373539,
      "fcm_dpo/margin": 67.19111633300781,
      "fcm_dpo/q_t": 0.4115195572376251,
      "grad_norm": 88.00062561035156,
      "learning_rate": 1.281842711051438e-07,
      "logits/chosen": -0.9563963413238525,
      "logits/rejected": -0.9196116924285889,
      "logps/chosen": -451.71697998046875,
      "logps/ref_chosen": -296.76300048828125,
      "logps/ref_rejected": -265.97991943359375,
      "logps/rejected": -488.125,
      "loss": 4.5031,
      "margin_dpo/margin_mean": 67.19110870361328,
      "margin_dpo/margin_std": 108.95948028564453,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -150.38037109375,
      "KL/mean": -187.99310302734375,
      "KL/rejected_KL_mean": -225.6057891845703,
      "KL/std": 103.81452941894531,
      "epoch": 0.6994764397905759,
      "fcm_dpo/beta": 0.005927244201302528,
      "fcm_dpo/delta": -0.03964047506451607,
      "fcm_dpo/margin": 75.22543334960938,
      "fcm_dpo/q_t": 0.4005029797554016,
      "grad_norm": 92.63084411621094,
      "learning_rate": 1.2658882646922033e-07,
      "logits/chosen": -0.8695560693740845,
      "logits/rejected": -0.8428592681884766,
      "logps/chosen": -451.4171447753906,
      "logps/ref_chosen": -301.0367431640625,
      "logps/ref_rejected": -268.87652587890625,
      "logps/rejected": -494.4823303222656,
      "loss": 4.4196,
      "margin_dpo/margin_mean": 75.22543334960938,
      "margin_dpo/margin_std": 113.39985656738281,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -166.61526489257812,
      "KL/mean": -204.59490966796875,
      "KL/rejected_KL_mean": -242.57456970214844,
      "KL/std": 97.44342803955078,
      "epoch": 0.7015706806282722,
      "fcm_dpo/beta": 0.005680191796272993,
      "fcm_dpo/delta": -0.009413354098796844,
      "fcm_dpo/margin": 75.95929718017578,
      "fcm_dpo/q_t": 0.4048939645290375,
      "grad_norm": 97.54159545898438,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -0.8594706654548645,
      "logits/rejected": -0.8540400266647339,
      "logps/chosen": -442.7480773925781,
      "logps/ref_chosen": -276.13275146484375,
      "logps/ref_rejected": -243.44203186035156,
      "logps/rejected": -486.01654052734375,
      "loss": 4.473,
      "margin_dpo/margin_mean": 75.95928955078125,
      "margin_dpo/margin_std": 120.38851928710938,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -164.53030395507812,
      "KL/mean": -202.40966796875,
      "KL/rejected_KL_mean": -240.28900146484375,
      "KL/std": 117.444091796875,
      "epoch": 0.7036649214659686,
      "fcm_dpo/beta": 0.005646462552249432,
      "fcm_dpo/delta": -0.024706192314624786,
      "fcm_dpo/margin": 75.75872039794922,
      "fcm_dpo/q_t": 0.410119891166687,
      "grad_norm": 105.4400634765625,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -0.8824126720428467,
      "logits/rejected": -0.8186700940132141,
      "logps/chosen": -410.79290771484375,
      "logps/ref_chosen": -246.2626495361328,
      "logps/ref_rejected": -261.0617980957031,
      "logps/rejected": -501.3508605957031,
      "loss": 4.4996,
      "margin_dpo/margin_mean": 75.75872039794922,
      "margin_dpo/margin_std": 125.51907348632812,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -155.38511657714844,
      "KL/mean": -194.76144409179688,
      "KL/rejected_KL_mean": -234.13778686523438,
      "KL/std": 100.49849700927734,
      "epoch": 0.7057591623036649,
      "fcm_dpo/beta": 0.005702502094209194,
      "fcm_dpo/delta": 7.306225597858429e-05,
      "fcm_dpo/margin": 78.75270080566406,
      "fcm_dpo/q_t": 0.4005512595176697,
      "grad_norm": 104.1450424194336,
      "learning_rate": 1.2184254201795363e-07,
      "logits/chosen": -0.8764570355415344,
      "logits/rejected": -0.8483041524887085,
      "logps/chosen": -422.37890625,
      "logps/ref_chosen": -266.9937744140625,
      "logps/ref_rejected": -253.015625,
      "logps/rejected": -487.15338134765625,
      "loss": 4.3417,
      "margin_dpo/margin_mean": 78.75270080566406,
      "margin_dpo/margin_std": 112.03500366210938,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -148.59970092773438,
      "KL/mean": -189.82252502441406,
      "KL/rejected_KL_mean": -231.04537963867188,
      "KL/std": 104.5492172241211,
      "epoch": 0.7078534031413612,
      "fcm_dpo/beta": 0.00562167726457119,
      "fcm_dpo/delta": -0.015084036625921726,
      "fcm_dpo/margin": 82.44564819335938,
      "fcm_dpo/q_t": 0.39697396755218506,
      "grad_norm": 130.8253631591797,
      "learning_rate": 1.202740798300168e-07,
      "logits/chosen": -0.9026167392730713,
      "logits/rejected": -0.883217453956604,
      "logps/chosen": -425.1922912597656,
      "logps/ref_chosen": -276.5925598144531,
      "logps/ref_rejected": -233.979248046875,
      "logps/rejected": -465.0246276855469,
      "loss": 4.2694,
      "margin_dpo/margin_mean": 82.4456558227539,
      "margin_dpo/margin_std": 109.38689422607422,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -161.6890869140625,
      "KL/mean": -204.8871307373047,
      "KL/rejected_KL_mean": -248.08517456054688,
      "KL/std": 106.50354766845703,
      "epoch": 0.7099476439790576,
      "fcm_dpo/beta": 0.005547752603888512,
      "fcm_dpo/delta": -0.03141479194164276,
      "fcm_dpo/margin": 86.39606475830078,
      "fcm_dpo/q_t": 0.39369451999664307,
      "grad_norm": 63.611759185791016,
      "learning_rate": 1.1871257444948096e-07,
      "logits/chosen": -0.9234580397605896,
      "logits/rejected": -0.9119750261306763,
      "logps/chosen": -465.2168884277344,
      "logps/ref_chosen": -303.5277404785156,
      "logps/ref_rejected": -283.11676025390625,
      "logps/rejected": -531.201904296875,
      "loss": 4.3147,
      "margin_dpo/margin_mean": 86.39606475830078,
      "margin_dpo/margin_std": 124.3836669921875,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -163.5699005126953,
      "KL/mean": -202.22467041015625,
      "KL/rejected_KL_mean": -240.87948608398438,
      "KL/std": 118.14247131347656,
      "epoch": 0.7120418848167539,
      "fcm_dpo/beta": 0.005402259062975645,
      "fcm_dpo/delta": -0.0016853753477334976,
      "fcm_dpo/margin": 77.30957794189453,
      "fcm_dpo/q_t": 0.4089118242263794,
      "grad_norm": 70.7248306274414,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": -0.8506317138671875,
      "logits/rejected": -0.845632016658783,
      "logps/chosen": -425.0956115722656,
      "logps/ref_chosen": -261.5257568359375,
      "logps/ref_rejected": -259.39862060546875,
      "logps/rejected": -500.278076171875,
      "loss": 4.6101,
      "margin_dpo/margin_mean": 77.3095703125,
      "margin_dpo/margin_std": 140.07342529296875,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -193.9016571044922,
      "KL/mean": -225.4265594482422,
      "KL/rejected_KL_mean": -256.9514465332031,
      "KL/std": 123.10039520263672,
      "epoch": 0.7141361256544503,
      "fcm_dpo/beta": 0.005434906110167503,
      "fcm_dpo/delta": 0.007884472608566284,
      "fcm_dpo/margin": 63.049774169921875,
      "fcm_dpo/q_t": 0.424948513507843,
      "grad_norm": 112.01563262939453,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -0.8769209980964661,
      "logits/rejected": -0.8482310771942139,
      "logps/chosen": -509.8052062988281,
      "logps/ref_chosen": -315.903564453125,
      "logps/ref_rejected": -308.02392578125,
      "logps/rejected": -564.975341796875,
      "loss": 4.8681,
      "margin_dpo/margin_mean": 63.049774169921875,
      "margin_dpo/margin_std": 137.02130126953125,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -161.8148956298828,
      "KL/mean": -200.04930114746094,
      "KL/rejected_KL_mean": -238.2837371826172,
      "KL/std": 105.99901580810547,
      "epoch": 0.7162303664921466,
      "fcm_dpo/beta": 0.005616472102701664,
      "fcm_dpo/delta": 0.020419172942638397,
      "fcm_dpo/margin": 76.46883392333984,
      "fcm_dpo/q_t": 0.40354785323143005,
      "grad_norm": 84.48583221435547,
      "learning_rate": 1.1407063464793965e-07,
      "logits/chosen": -0.8723053336143494,
      "logits/rejected": -0.8676621317863464,
      "logps/chosen": -430.9935607910156,
      "logps/ref_chosen": -269.17864990234375,
      "logps/ref_rejected": -260.8977355957031,
      "logps/rejected": -499.1814880371094,
      "loss": 4.3757,
      "margin_dpo/margin_mean": 76.46883392333984,
      "margin_dpo/margin_std": 108.46856689453125,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -167.002685546875,
      "KL/mean": -200.95213317871094,
      "KL/rejected_KL_mean": -234.9015655517578,
      "KL/std": 108.25473022460938,
      "epoch": 0.7183246073298429,
      "fcm_dpo/beta": 0.0056951227597892284,
      "fcm_dpo/delta": 0.06580042839050293,
      "fcm_dpo/margin": 67.89884185791016,
      "fcm_dpo/q_t": 0.4139096438884735,
      "grad_norm": 82.44038391113281,
      "learning_rate": 1.125377900869913e-07,
      "logits/chosen": -0.8717567920684814,
      "logits/rejected": -0.8485027551651001,
      "logps/chosen": -477.72271728515625,
      "logps/ref_chosen": -310.719970703125,
      "logps/ref_rejected": -263.5224914550781,
      "logps/rejected": -498.424072265625,
      "loss": 4.5317,
      "margin_dpo/margin_mean": 67.89884948730469,
      "margin_dpo/margin_std": 111.75759887695312,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -166.8184356689453,
      "KL/mean": -202.58326721191406,
      "KL/rejected_KL_mean": -238.3480682373047,
      "KL/std": 98.68063354492188,
      "epoch": 0.7204188481675393,
      "fcm_dpo/beta": 0.005950647871941328,
      "fcm_dpo/delta": -0.0052766092121601105,
      "fcm_dpo/margin": 71.52964782714844,
      "fcm_dpo/q_t": 0.4054498076438904,
      "grad_norm": 121.65364074707031,
      "learning_rate": 1.110123172071844e-07,
      "logits/chosen": -0.8611950874328613,
      "logits/rejected": -0.8410104513168335,
      "logps/chosen": -468.6183166503906,
      "logps/ref_chosen": -301.7999267578125,
      "logps/ref_rejected": -257.9061584472656,
      "logps/rejected": -496.2542419433594,
      "loss": 4.5137,
      "margin_dpo/margin_mean": 71.52964782714844,
      "margin_dpo/margin_std": 117.92987060546875,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -164.86741638183594,
      "KL/mean": -195.7162628173828,
      "KL/rejected_KL_mean": -226.56507873535156,
      "KL/std": 104.853271484375,
      "epoch": 0.7225130890052356,
      "fcm_dpo/beta": 0.0057700155302882195,
      "fcm_dpo/delta": -0.03494897112250328,
      "fcm_dpo/margin": 61.69768142700195,
      "fcm_dpo/q_t": 0.42071518301963806,
      "grad_norm": 117.05513000488281,
      "learning_rate": 1.09494297815e-07,
      "logits/chosen": -0.868345320224762,
      "logits/rejected": -0.868226170539856,
      "logps/chosen": -447.8858642578125,
      "logps/ref_chosen": -283.0184326171875,
      "logps/ref_rejected": -266.8457336425781,
      "logps/rejected": -493.4108581542969,
      "loss": 4.6688,
      "margin_dpo/margin_mean": 61.69768142700195,
      "margin_dpo/margin_std": 106.84193420410156,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -170.68296813964844,
      "KL/mean": -211.1671905517578,
      "KL/rejected_KL_mean": -251.65142822265625,
      "KL/std": 111.68208312988281,
      "epoch": 0.724607329842932,
      "fcm_dpo/beta": 0.005454138852655888,
      "fcm_dpo/delta": -0.06955993920564651,
      "fcm_dpo/margin": 80.96846008300781,
      "fcm_dpo/q_t": 0.40043899416923523,
      "grad_norm": 76.98741912841797,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -0.9455903768539429,
      "logits/rejected": -0.8937309384346008,
      "logps/chosen": -439.12420654296875,
      "logps/ref_chosen": -268.44122314453125,
      "logps/ref_rejected": -227.8225860595703,
      "logps/rejected": -479.4739990234375,
      "loss": 4.4146,
      "margin_dpo/margin_mean": 80.96846008300781,
      "margin_dpo/margin_std": 117.54512023925781,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -160.2595977783203,
      "KL/mean": -193.86691284179688,
      "KL/rejected_KL_mean": -227.47421264648438,
      "KL/std": 101.53842163085938,
      "epoch": 0.7267015706806282,
      "fcm_dpo/beta": 0.005477546714246273,
      "fcm_dpo/delta": 0.042753636837005615,
      "fcm_dpo/margin": 67.21463012695312,
      "fcm_dpo/q_t": 0.4163162410259247,
      "grad_norm": 81.83856201171875,
      "learning_rate": 1.0648094471651722e-07,
      "logits/chosen": -0.7989067435264587,
      "logits/rejected": -0.8253777027130127,
      "logps/chosen": -433.963134765625,
      "logps/ref_chosen": -273.70355224609375,
      "logps/ref_rejected": -243.65521240234375,
      "logps/rejected": -471.1294250488281,
      "loss": 4.5675,
      "margin_dpo/margin_mean": 67.21463775634766,
      "margin_dpo/margin_std": 106.69586181640625,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -163.5076141357422,
      "KL/mean": -192.10580444335938,
      "KL/rejected_KL_mean": -220.7039794921875,
      "KL/std": 106.62578582763672,
      "epoch": 0.7287958115183246,
      "fcm_dpo/beta": 0.005734581500291824,
      "fcm_dpo/delta": 0.05284817889332771,
      "fcm_dpo/margin": 57.19636154174805,
      "fcm_dpo/q_t": 0.4303225874900818,
      "grad_norm": 82.78878021240234,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": -0.9022307395935059,
      "logits/rejected": -0.8868052363395691,
      "logps/chosen": -449.1490478515625,
      "logps/ref_chosen": -285.64141845703125,
      "logps/ref_rejected": -265.6270446777344,
      "logps/rejected": -486.3310546875,
      "loss": 4.802,
      "margin_dpo/margin_mean": 57.19636154174805,
      "margin_dpo/margin_std": 119.81888580322266,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -158.2762908935547,
      "KL/mean": -202.1715087890625,
      "KL/rejected_KL_mean": -246.06674194335938,
      "KL/std": 112.10665130615234,
      "epoch": 0.7308900523560209,
      "fcm_dpo/beta": 0.005708886310458183,
      "fcm_dpo/delta": -0.05489795655012131,
      "fcm_dpo/margin": 87.79044342041016,
      "fcm_dpo/q_t": 0.3896053731441498,
      "grad_norm": 100.4817123413086,
      "learning_rate": 1.0349837717080347e-07,
      "logits/chosen": -0.8397247791290283,
      "logits/rejected": -0.8294237852096558,
      "logps/chosen": -486.5937805175781,
      "logps/ref_chosen": -328.3175048828125,
      "logps/ref_rejected": -292.37872314453125,
      "logps/rejected": -538.4454956054688,
      "loss": 4.2905,
      "margin_dpo/margin_mean": 87.79044342041016,
      "margin_dpo/margin_std": 124.65754699707031,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -173.89703369140625,
      "KL/mean": -207.83709716796875,
      "KL/rejected_KL_mean": -241.77716064453125,
      "KL/std": 106.3682861328125,
      "epoch": 0.7329842931937173,
      "fcm_dpo/beta": 0.0055555738508701324,
      "fcm_dpo/delta": -0.002974544186145067,
      "fcm_dpo/margin": 67.88014221191406,
      "fcm_dpo/q_t": 0.4164275527000427,
      "grad_norm": 88.74214935302734,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": -0.840487003326416,
      "logits/rejected": -0.8536363840103149,
      "logps/chosen": -466.70166015625,
      "logps/ref_chosen": -292.8046569824219,
      "logps/ref_rejected": -250.35504150390625,
      "logps/rejected": -492.1322021484375,
      "loss": 4.6438,
      "margin_dpo/margin_mean": 67.88014221191406,
      "margin_dpo/margin_std": 123.80751037597656,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -160.29066467285156,
      "KL/mean": -183.7502899169922,
      "KL/rejected_KL_mean": -207.2098846435547,
      "KL/std": 102.392822265625,
      "epoch": 0.7350785340314137,
      "fcm_dpo/beta": 0.005624302197247744,
      "fcm_dpo/delta": 0.03526240959763527,
      "fcm_dpo/margin": 46.91920471191406,
      "fcm_dpo/q_t": 0.44256314635276794,
      "grad_norm": 93.0438232421875,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -0.914904773235321,
      "logits/rejected": -0.9003003239631653,
      "logps/chosen": -472.1797180175781,
      "logps/ref_chosen": -311.8890380859375,
      "logps/ref_rejected": -263.59033203125,
      "logps/rejected": -470.80023193359375,
      "loss": 5.0418,
      "margin_dpo/margin_mean": 46.9192008972168,
      "margin_dpo/margin_std": 125.01647186279297,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -147.45492553710938,
      "KL/mean": -192.49136352539062,
      "KL/rejected_KL_mean": -237.5277862548828,
      "KL/std": 110.10826873779297,
      "epoch": 0.7371727748691099,
      "fcm_dpo/beta": 0.005537643563002348,
      "fcm_dpo/delta": -0.052963994443416595,
      "fcm_dpo/margin": 90.07286834716797,
      "fcm_dpo/q_t": 0.3882845938205719,
      "grad_norm": 88.90042114257812,
      "learning_rate": 9.908364643332398e-08,
      "logits/chosen": -0.8344815373420715,
      "logits/rejected": -0.8047367334365845,
      "logps/chosen": -402.3627624511719,
      "logps/ref_chosen": -254.9078826904297,
      "logps/ref_rejected": -257.1688232421875,
      "logps/rejected": -494.69659423828125,
      "loss": 4.2249,
      "margin_dpo/margin_mean": 90.07286834716797,
      "margin_dpo/margin_std": 117.22493743896484,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -157.25930786132812,
      "KL/mean": -194.7118682861328,
      "KL/rejected_KL_mean": -232.16445922851562,
      "KL/std": 105.713623046875,
      "epoch": 0.7392670157068063,
      "fcm_dpo/beta": 0.005408046767115593,
      "fcm_dpo/delta": 0.003531409427523613,
      "fcm_dpo/margin": 74.90514373779297,
      "fcm_dpo/q_t": 0.40997636318206787,
      "grad_norm": 74.88296508789062,
      "learning_rate": 9.76281510992176e-08,
      "logits/chosen": -0.8565041422843933,
      "logits/rejected": -0.8508064150810242,
      "logps/chosen": -427.6353454589844,
      "logps/ref_chosen": -270.3760681152344,
      "logps/ref_rejected": -264.65234375,
      "logps/rejected": -496.81683349609375,
      "loss": 4.4993,
      "margin_dpo/margin_mean": 74.90514373779297,
      "margin_dpo/margin_std": 118.6182861328125,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -157.4679412841797,
      "KL/mean": -185.90573120117188,
      "KL/rejected_KL_mean": -214.34352111816406,
      "KL/std": 107.95223236083984,
      "epoch": 0.7413612565445026,
      "fcm_dpo/beta": 0.005602296441793442,
      "fcm_dpo/delta": 0.06664598733186722,
      "fcm_dpo/margin": 56.87559127807617,
      "fcm_dpo/q_t": 0.42999356985092163,
      "grad_norm": 100.81631469726562,
      "learning_rate": 9.618082700494318e-08,
      "logits/chosen": -0.8458303809165955,
      "logits/rejected": -0.8789796829223633,
      "logps/chosen": -415.1164855957031,
      "logps/ref_chosen": -257.6485595703125,
      "logps/ref_rejected": -246.94203186035156,
      "logps/rejected": -461.2855224609375,
      "loss": 4.8059,
      "margin_dpo/margin_mean": 56.87559509277344,
      "margin_dpo/margin_std": 118.4468765258789,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -150.63006591796875,
      "KL/mean": -193.5918426513672,
      "KL/rejected_KL_mean": -236.5535888671875,
      "KL/std": 112.25565338134766,
      "epoch": 0.743455497382199,
      "fcm_dpo/beta": 0.005481460131704807,
      "fcm_dpo/delta": -0.06419708579778671,
      "fcm_dpo/margin": 85.92353057861328,
      "fcm_dpo/q_t": 0.39600738883018494,
      "grad_norm": 69.32930755615234,
      "learning_rate": 9.474175176609956e-08,
      "logits/chosen": -0.8934347629547119,
      "logits/rejected": -0.8912361860275269,
      "logps/chosen": -443.9834289550781,
      "logps/ref_chosen": -293.35333251953125,
      "logps/ref_rejected": -275.6051940917969,
      "logps/rejected": -512.1588134765625,
      "loss": 4.385,
      "margin_dpo/margin_mean": 85.92353057861328,
      "margin_dpo/margin_std": 129.98941040039062,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -142.90467834472656,
      "KL/mean": -172.47491455078125,
      "KL/rejected_KL_mean": -202.045166015625,
      "KL/std": 93.50942993164062,
      "epoch": 0.7455497382198953,
      "fcm_dpo/beta": 0.005551180802285671,
      "fcm_dpo/delta": 0.04267197847366333,
      "fcm_dpo/margin": 59.14048767089844,
      "fcm_dpo/q_t": 0.42504405975341797,
      "grad_norm": 77.28180694580078,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -0.8072715401649475,
      "logits/rejected": -0.8343250155448914,
      "logps/chosen": -347.1601867675781,
      "logps/ref_chosen": -204.25550842285156,
      "logps/ref_rejected": -213.467529296875,
      "logps/rejected": -415.5126953125,
      "loss": 4.6334,
      "margin_dpo/margin_mean": 59.14048767089844,
      "margin_dpo/margin_std": 94.28951263427734,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -158.0202178955078,
      "KL/mean": -195.85548400878906,
      "KL/rejected_KL_mean": -233.69076538085938,
      "KL/std": 110.6160888671875,
      "epoch": 0.7476439790575916,
      "fcm_dpo/beta": 0.005523748230189085,
      "fcm_dpo/delta": -0.04239559546113014,
      "fcm_dpo/margin": 75.67053985595703,
      "fcm_dpo/q_t": 0.40887686610221863,
      "grad_norm": 86.33390045166016,
      "learning_rate": 9.18886561011557e-08,
      "logits/chosen": -0.7886257171630859,
      "logits/rejected": -0.7880325317382812,
      "logps/chosen": -424.3907775878906,
      "logps/ref_chosen": -266.3705749511719,
      "logps/ref_rejected": -239.04490661621094,
      "logps/rejected": -472.73565673828125,
      "loss": 4.5013,
      "margin_dpo/margin_mean": 75.67053985595703,
      "margin_dpo/margin_std": 122.1236801147461,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -142.3251495361328,
      "KL/mean": -186.70712280273438,
      "KL/rejected_KL_mean": -231.08912658691406,
      "KL/std": 101.25473022460938,
      "epoch": 0.749738219895288,
      "fcm_dpo/beta": 0.005430258344858885,
      "fcm_dpo/delta": -0.03410031273961067,
      "fcm_dpo/margin": 88.76396942138672,
      "fcm_dpo/q_t": 0.3917004466056824,
      "grad_norm": 81.43157958984375,
      "learning_rate": 9.047478867791731e-08,
      "logits/chosen": -0.8815658092498779,
      "logits/rejected": -0.8604514002799988,
      "logps/chosen": -441.47259521484375,
      "logps/ref_chosen": -299.1474609375,
      "logps/ref_rejected": -257.2531433105469,
      "logps/rejected": -488.34228515625,
      "loss": 4.2714,
      "margin_dpo/margin_mean": 88.76396942138672,
      "margin_dpo/margin_std": 120.1227035522461,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -148.87965393066406,
      "KL/mean": -185.2836456298828,
      "KL/rejected_KL_mean": -221.68765258789062,
      "KL/std": 99.21031951904297,
      "epoch": 0.7518324607329843,
      "fcm_dpo/beta": 0.005362433847039938,
      "fcm_dpo/delta": -0.010527530685067177,
      "fcm_dpo/margin": 72.80799102783203,
      "fcm_dpo/q_t": 0.4118102192878723,
      "grad_norm": 108.26348876953125,
      "learning_rate": 8.906947610762825e-08,
      "logits/chosen": -0.8431890606880188,
      "logits/rejected": -0.8552847504615784,
      "logps/chosen": -451.87750244140625,
      "logps/ref_chosen": -302.99786376953125,
      "logps/ref_rejected": -260.4137268066406,
      "logps/rejected": -482.10137939453125,
      "loss": 4.4892,
      "margin_dpo/margin_mean": 72.80799102783203,
      "margin_dpo/margin_std": 108.55307006835938,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -156.75962829589844,
      "KL/mean": -190.37344360351562,
      "KL/rejected_KL_mean": -223.98727416992188,
      "KL/std": 105.21798706054688,
      "epoch": 0.7539267015706806,
      "fcm_dpo/beta": 0.0053200554102659225,
      "fcm_dpo/delta": 0.010399473831057549,
      "fcm_dpo/margin": 67.2276840209961,
      "fcm_dpo/q_t": 0.4184189736843109,
      "grad_norm": 95.97364807128906,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": -0.8648691177368164,
      "logits/rejected": -0.8524286150932312,
      "logps/chosen": -466.37103271484375,
      "logps/ref_chosen": -309.6114501953125,
      "logps/ref_rejected": -256.64031982421875,
      "logps/rejected": -480.62762451171875,
      "loss": 4.6943,
      "margin_dpo/margin_mean": 67.2276840209961,
      "margin_dpo/margin_std": 120.57766723632812,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -129.46585083007812,
      "KL/mean": -174.60940551757812,
      "KL/rejected_KL_mean": -219.75299072265625,
      "KL/std": 108.40211486816406,
      "epoch": 0.7560209424083769,
      "fcm_dpo/beta": 0.0052897087298333645,
      "fcm_dpo/delta": -0.029126279056072235,
      "fcm_dpo/margin": 90.28715515136719,
      "fcm_dpo/q_t": 0.3957591950893402,
      "grad_norm": 82.8307876586914,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -0.8035414218902588,
      "logits/rejected": -0.7840466499328613,
      "logps/chosen": -392.8455810546875,
      "logps/ref_chosen": -263.3797607421875,
      "logps/ref_rejected": -271.18157958984375,
      "logps/rejected": -490.93450927734375,
      "loss": 4.3404,
      "margin_dpo/margin_mean": 90.28715515136719,
      "margin_dpo/margin_std": 135.23036193847656,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -142.95352172851562,
      "KL/mean": -180.92263793945312,
      "KL/rejected_KL_mean": -218.8917999267578,
      "KL/std": 91.27169799804688,
      "epoch": 0.7581151832460733,
      "fcm_dpo/beta": 0.005256773438304663,
      "fcm_dpo/delta": 0.05308995768427849,
      "fcm_dpo/margin": 75.93827056884766,
      "fcm_dpo/q_t": 0.4092256426811218,
      "grad_norm": 93.33192443847656,
      "learning_rate": 8.490561882286135e-08,
      "logits/chosen": -0.8282926082611084,
      "logits/rejected": -0.8166139721870422,
      "logps/chosen": -446.2119140625,
      "logps/ref_chosen": -303.2583923339844,
      "logps/ref_rejected": -243.22891235351562,
      "logps/rejected": -462.12066650390625,
      "loss": 4.4057,
      "margin_dpo/margin_mean": 75.93827056884766,
      "margin_dpo/margin_std": 103.966064453125,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -148.06101989746094,
      "KL/mean": -186.46878051757812,
      "KL/rejected_KL_mean": -224.87655639648438,
      "KL/std": 108.91134643554688,
      "epoch": 0.7602094240837697,
      "fcm_dpo/beta": 0.005439347587525845,
      "fcm_dpo/delta": 0.00030844658613204956,
      "fcm_dpo/margin": 76.81553649902344,
      "fcm_dpo/q_t": 0.4090897738933563,
      "grad_norm": 97.79991149902344,
      "learning_rate": 8.353527464267104e-08,
      "logits/chosen": -0.8552191853523254,
      "logits/rejected": -0.8098867535591125,
      "logps/chosen": -451.40826416015625,
      "logps/ref_chosen": -303.34722900390625,
      "logps/ref_rejected": -262.05419921875,
      "logps/rejected": -486.9307556152344,
      "loss": 4.53,
      "margin_dpo/margin_mean": 76.81553649902344,
      "margin_dpo/margin_std": 128.73184204101562,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -168.41848754882812,
      "KL/mean": -203.2845458984375,
      "KL/rejected_KL_mean": -238.15060424804688,
      "KL/std": 111.24627685546875,
      "epoch": 0.762303664921466,
      "fcm_dpo/beta": 0.00552753359079361,
      "fcm_dpo/delta": 0.06728857755661011,
      "fcm_dpo/margin": 69.73213958740234,
      "fcm_dpo/q_t": 0.4178435802459717,
      "grad_norm": 86.63250732421875,
      "learning_rate": 8.217385746050742e-08,
      "logits/chosen": -0.8349016308784485,
      "logits/rejected": -0.843680202960968,
      "logps/chosen": -453.9622497558594,
      "logps/ref_chosen": -285.54376220703125,
      "logps/ref_rejected": -284.84619140625,
      "logps/rejected": -522.996826171875,
      "loss": 4.6819,
      "margin_dpo/margin_mean": 69.73213958740234,
      "margin_dpo/margin_std": 134.32196044921875,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -159.46041870117188,
      "KL/mean": -194.58639526367188,
      "KL/rejected_KL_mean": -229.71237182617188,
      "KL/std": 112.55696105957031,
      "epoch": 0.7643979057591623,
      "fcm_dpo/beta": 0.005666092038154602,
      "fcm_dpo/delta": -0.019497856497764587,
      "fcm_dpo/margin": 70.25199127197266,
      "fcm_dpo/q_t": 0.4130297005176544,
      "grad_norm": 106.32125091552734,
      "learning_rate": 8.082144028504231e-08,
      "logits/chosen": -0.8369499444961548,
      "logits/rejected": -0.8392655849456787,
      "logps/chosen": -434.24822998046875,
      "logps/ref_chosen": -274.7878112792969,
      "logps/ref_rejected": -256.5738220214844,
      "logps/rejected": -486.2862243652344,
      "loss": 4.581,
      "margin_dpo/margin_mean": 70.25199127197266,
      "margin_dpo/margin_std": 121.36874389648438,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -145.95523071289062,
      "KL/mean": -186.51687622070312,
      "KL/rejected_KL_mean": -227.07852172851562,
      "KL/std": 101.78260803222656,
      "epoch": 0.7664921465968586,
      "fcm_dpo/beta": 0.0056350575760006905,
      "fcm_dpo/delta": -0.007890645414590836,
      "fcm_dpo/margin": 81.12327575683594,
      "fcm_dpo/q_t": 0.39784321188926697,
      "grad_norm": 72.16694641113281,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -0.8017479181289673,
      "logits/rejected": -0.8110736608505249,
      "logps/chosen": -432.6048889160156,
      "logps/ref_chosen": -286.6496276855469,
      "logps/ref_rejected": -251.97140502929688,
      "logps/rejected": -479.0499267578125,
      "loss": 4.3657,
      "margin_dpo/margin_mean": 81.12327575683594,
      "margin_dpo/margin_std": 121.07316589355469,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -144.89678955078125,
      "KL/mean": -186.06268310546875,
      "KL/rejected_KL_mean": -227.22857666015625,
      "KL/std": 99.79731750488281,
      "epoch": 0.768586387434555,
      "fcm_dpo/beta": 0.005410144105553627,
      "fcm_dpo/delta": -0.028827723115682602,
      "fcm_dpo/margin": 82.33177185058594,
      "fcm_dpo/q_t": 0.4000735878944397,
      "grad_norm": 92.13324737548828,
      "learning_rate": 7.814389557179016e-08,
      "logits/chosen": -0.8316141963005066,
      "logits/rejected": -0.8104003071784973,
      "logps/chosen": -446.841796875,
      "logps/ref_chosen": -301.9449768066406,
      "logps/ref_rejected": -265.5677185058594,
      "logps/rejected": -492.7962951660156,
      "loss": 4.3332,
      "margin_dpo/margin_mean": 82.33177185058594,
      "margin_dpo/margin_std": 113.719482421875,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -137.6929931640625,
      "KL/mean": -187.13192749023438,
      "KL/rejected_KL_mean": -236.5708770751953,
      "KL/std": 103.59458923339844,
      "epoch": 0.7706806282722513,
      "fcm_dpo/beta": 0.005297235678881407,
      "fcm_dpo/delta": -0.07807178795337677,
      "fcm_dpo/margin": 98.87789154052734,
      "fcm_dpo/q_t": 0.38079503178596497,
      "grad_norm": 69.9876708984375,
      "learning_rate": 7.681891162260015e-08,
      "logits/chosen": -0.8012065887451172,
      "logits/rejected": -0.8091939091682434,
      "logps/chosen": -432.3194885253906,
      "logps/ref_chosen": -294.62652587890625,
      "logps/ref_rejected": -258.7628479003906,
      "logps/rejected": -495.3337707519531,
      "loss": 4.0389,
      "margin_dpo/margin_mean": 98.87789154052734,
      "margin_dpo/margin_std": 107.66801452636719,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -148.0738525390625,
      "KL/mean": -184.71408081054688,
      "KL/rejected_KL_mean": -221.3543243408203,
      "KL/std": 100.5986557006836,
      "epoch": 0.7727748691099476,
      "fcm_dpo/beta": 0.005211456213146448,
      "fcm_dpo/delta": 0.0386260487139225,
      "fcm_dpo/margin": 73.28047180175781,
      "fcm_dpo/q_t": 0.4126470386981964,
      "grad_norm": 85.48666381835938,
      "learning_rate": 7.550321484960251e-08,
      "logits/chosen": -0.8856424689292908,
      "logits/rejected": -0.8686147332191467,
      "logps/chosen": -430.57958984375,
      "logps/ref_chosen": -282.5057373046875,
      "logps/ref_rejected": -266.41607666015625,
      "logps/rejected": -487.7704162597656,
      "loss": 4.4638,
      "margin_dpo/margin_mean": 73.28047943115234,
      "margin_dpo/margin_std": 105.00106048583984,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -138.93460083007812,
      "KL/mean": -183.23684692382812,
      "KL/rejected_KL_mean": -227.53909301757812,
      "KL/std": 103.05982971191406,
      "epoch": 0.774869109947644,
      "fcm_dpo/beta": 0.00519456434994936,
      "fcm_dpo/delta": -0.011941131204366684,
      "fcm_dpo/margin": 88.60447692871094,
      "fcm_dpo/q_t": 0.39707690477371216,
      "grad_norm": 68.53267669677734,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": -0.8690569996833801,
      "logits/rejected": -0.893211841583252,
      "logps/chosen": -389.9410400390625,
      "logps/ref_chosen": -251.00640869140625,
      "logps/ref_rejected": -238.12542724609375,
      "logps/rejected": -465.66448974609375,
      "loss": 4.3478,
      "margin_dpo/margin_mean": 88.60447692871094,
      "margin_dpo/margin_std": 129.37429809570312,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -157.29238891601562,
      "KL/mean": -190.2088165283203,
      "KL/rejected_KL_mean": -223.125244140625,
      "KL/std": 104.46833038330078,
      "epoch": 0.7769633507853403,
      "fcm_dpo/beta": 0.005483569577336311,
      "fcm_dpo/delta": 0.09168285876512527,
      "fcm_dpo/margin": 65.83285522460938,
      "fcm_dpo/q_t": 0.4189152121543884,
      "grad_norm": 94.57374572753906,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -0.8149279356002808,
      "logits/rejected": -0.80399489402771,
      "logps/chosen": -453.9515380859375,
      "logps/ref_chosen": -296.6591491699219,
      "logps/ref_rejected": -251.14675903320312,
      "logps/rejected": -474.27197265625,
      "loss": 4.6146,
      "margin_dpo/margin_mean": 65.8328628540039,
      "margin_dpo/margin_std": 111.59983825683594,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -154.52407836914062,
      "KL/mean": -198.15037536621094,
      "KL/rejected_KL_mean": -241.77670288085938,
      "KL/std": 112.22760009765625,
      "epoch": 0.7790575916230367,
      "fcm_dpo/beta": 0.005509304814040661,
      "fcm_dpo/delta": -0.0340895913541317,
      "fcm_dpo/margin": 87.25262451171875,
      "fcm_dpo/q_t": 0.3939465880393982,
      "grad_norm": 89.53658294677734,
      "learning_rate": 7.161255064312283e-08,
      "logits/chosen": -0.787936270236969,
      "logits/rejected": -0.7807087898254395,
      "logps/chosen": -485.8955383300781,
      "logps/ref_chosen": -331.3714599609375,
      "logps/ref_rejected": -285.56805419921875,
      "logps/rejected": -527.3447265625,
      "loss": 4.322,
      "margin_dpo/margin_mean": 87.25262451171875,
      "margin_dpo/margin_std": 124.37115478515625,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -142.75440979003906,
      "KL/mean": -183.96206665039062,
      "KL/rejected_KL_mean": -225.169677734375,
      "KL/std": 100.71784973144531,
      "epoch": 0.7811518324607329,
      "fcm_dpo/beta": 0.005426083691418171,
      "fcm_dpo/delta": 0.0022701737470924854,
      "fcm_dpo/margin": 82.415283203125,
      "fcm_dpo/q_t": 0.39797142148017883,
      "grad_norm": 87.75348663330078,
      "learning_rate": 7.033470310611945e-08,
      "logits/chosen": -0.8869479894638062,
      "logits/rejected": -0.8584622144699097,
      "logps/chosen": -464.6973876953125,
      "logps/ref_chosen": -321.9429931640625,
      "logps/ref_rejected": -271.2288513183594,
      "logps/rejected": -496.3985595703125,
      "loss": 4.3172,
      "margin_dpo/margin_mean": 82.415283203125,
      "margin_dpo/margin_std": 109.29535675048828,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -149.73318481445312,
      "KL/mean": -185.51405334472656,
      "KL/rejected_KL_mean": -221.29493713378906,
      "KL/std": 107.82977294921875,
      "epoch": 0.7832460732984293,
      "fcm_dpo/beta": 0.005488203838467598,
      "fcm_dpo/delta": 0.0260069128125906,
      "fcm_dpo/margin": 71.56175994873047,
      "fcm_dpo/q_t": 0.4132099449634552,
      "grad_norm": 74.5303726196289,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": -0.8744044303894043,
      "logits/rejected": -0.8719938397407532,
      "logps/chosen": -468.9017333984375,
      "logps/ref_chosen": -319.1685485839844,
      "logps/ref_rejected": -284.6263732910156,
      "logps/rejected": -505.9212951660156,
      "loss": 4.516,
      "margin_dpo/margin_mean": 71.56175994873047,
      "margin_dpo/margin_std": 114.37178039550781,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -159.27877807617188,
      "KL/mean": -190.62782287597656,
      "KL/rejected_KL_mean": -221.9768524169922,
      "KL/std": 93.83641815185547,
      "epoch": 0.7853403141361257,
      "fcm_dpo/beta": 0.005679625552147627,
      "fcm_dpo/delta": 0.016149766743183136,
      "fcm_dpo/margin": 62.698062896728516,
      "fcm_dpo/q_t": 0.4209960699081421,
      "grad_norm": 99.32843780517578,
      "learning_rate": 6.780798075635675e-08,
      "logits/chosen": -0.8694410920143127,
      "logits/rejected": -0.8502828478813171,
      "logps/chosen": -474.1545715332031,
      "logps/ref_chosen": -314.87579345703125,
      "logps/ref_rejected": -259.1965026855469,
      "logps/rejected": -481.17333984375,
      "loss": 4.6621,
      "margin_dpo/margin_mean": 62.69805908203125,
      "margin_dpo/margin_std": 113.64825439453125,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -158.72039794921875,
      "KL/mean": -194.63528442382812,
      "KL/rejected_KL_mean": -230.55020141601562,
      "KL/std": 104.314453125,
      "epoch": 0.787434554973822,
      "fcm_dpo/beta": 0.005847534164786339,
      "fcm_dpo/delta": 0.02977427840232849,
      "fcm_dpo/margin": 71.82978820800781,
      "fcm_dpo/q_t": 0.4089190661907196,
      "grad_norm": 110.7322998046875,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -0.8434525728225708,
      "logits/rejected": -0.848453164100647,
      "logps/chosen": -446.3936462402344,
      "logps/ref_chosen": -287.6732482910156,
      "logps/ref_rejected": -256.6697082519531,
      "logps/rejected": -487.2198486328125,
      "loss": 4.5656,
      "margin_dpo/margin_mean": 71.82978820800781,
      "margin_dpo/margin_std": 125.38734436035156,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -168.8662109375,
      "KL/mean": -194.5668182373047,
      "KL/rejected_KL_mean": -220.26748657226562,
      "KL/std": 102.88775634765625,
      "epoch": 0.7895287958115184,
      "fcm_dpo/beta": 0.005977586843073368,
      "fcm_dpo/delta": 0.04635694622993469,
      "fcm_dpo/margin": 51.40126037597656,
      "fcm_dpo/q_t": 0.4314709007740021,
      "grad_norm": 82.71524047851562,
      "learning_rate": 6.532033950290885e-08,
      "logits/chosen": -0.8344903588294983,
      "logits/rejected": -0.8335461020469666,
      "logps/chosen": -474.127685546875,
      "logps/ref_chosen": -305.261474609375,
      "logps/ref_rejected": -271.8887023925781,
      "logps/rejected": -492.15618896484375,
      "loss": 4.9181,
      "margin_dpo/margin_mean": 51.40125274658203,
      "margin_dpo/margin_std": 119.6769790649414,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -159.54525756835938,
      "KL/mean": -191.91549682617188,
      "KL/rejected_KL_mean": -224.28573608398438,
      "KL/std": 108.22175598144531,
      "epoch": 0.7916230366492146,
      "fcm_dpo/beta": 0.006102831568568945,
      "fcm_dpo/delta": 0.02815961092710495,
      "fcm_dpo/margin": 64.7405014038086,
      "fcm_dpo/q_t": 0.4136991500854492,
      "grad_norm": 105.0207748413086,
      "learning_rate": 6.409134137148736e-08,
      "logits/chosen": -0.8303889632225037,
      "logits/rejected": -0.8179850578308105,
      "logps/chosen": -441.0748291015625,
      "logps/ref_chosen": -281.5295715332031,
      "logps/ref_rejected": -296.980224609375,
      "logps/rejected": -521.2659912109375,
      "loss": 4.5825,
      "margin_dpo/margin_mean": 64.74049377441406,
      "margin_dpo/margin_std": 113.59513092041016,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -146.8624725341797,
      "KL/mean": -181.2415008544922,
      "KL/rejected_KL_mean": -215.62054443359375,
      "KL/std": 90.73746490478516,
      "epoch": 0.793717277486911,
      "fcm_dpo/beta": 0.006207392085343599,
      "fcm_dpo/delta": 0.023632923141121864,
      "fcm_dpo/margin": 68.75806427001953,
      "fcm_dpo/q_t": 0.40591737627983093,
      "grad_norm": 115.84989166259766,
      "learning_rate": 6.28723129572247e-08,
      "logits/chosen": -0.8956962823867798,
      "logits/rejected": -0.8774758577346802,
      "logps/chosen": -411.94329833984375,
      "logps/ref_chosen": -265.0807800292969,
      "logps/ref_rejected": -230.58932495117188,
      "logps/rejected": -446.2098693847656,
      "loss": 4.4972,
      "margin_dpo/margin_mean": 68.75807189941406,
      "margin_dpo/margin_std": 114.05142211914062,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -155.13485717773438,
      "KL/mean": -193.4832000732422,
      "KL/rejected_KL_mean": -231.83157348632812,
      "KL/std": 104.35556030273438,
      "epoch": 0.7958115183246073,
      "fcm_dpo/beta": 0.0061458623968064785,
      "fcm_dpo/delta": -0.07802033424377441,
      "fcm_dpo/margin": 76.69670867919922,
      "fcm_dpo/q_t": 0.3976585268974304,
      "grad_norm": 117.2214126586914,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": -0.8615453243255615,
      "logits/rejected": -0.8410003185272217,
      "logps/chosen": -461.04327392578125,
      "logps/ref_chosen": -305.90838623046875,
      "logps/ref_rejected": -286.5906677246094,
      "logps/rejected": -518.4222412109375,
      "loss": 4.3999,
      "margin_dpo/margin_mean": 76.69670867919922,
      "margin_dpo/margin_std": 115.11908721923828,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -152.8490447998047,
      "KL/mean": -192.10150146484375,
      "KL/rejected_KL_mean": -231.3539581298828,
      "KL/std": 112.06571197509766,
      "epoch": 0.7979057591623037,
      "fcm_dpo/beta": 0.005857712123543024,
      "fcm_dpo/delta": -0.04640359804034233,
      "fcm_dpo/margin": 78.50491333007812,
      "fcm_dpo/q_t": 0.4003297984600067,
      "grad_norm": 94.79520416259766,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -0.8238821625709534,
      "logits/rejected": -0.7888757586479187,
      "logps/chosen": -405.71966552734375,
      "logps/ref_chosen": -252.87066650390625,
      "logps/ref_rejected": -261.1927490234375,
      "logps/rejected": -492.54669189453125,
      "loss": 4.3868,
      "margin_dpo/margin_mean": 78.50491333007812,
      "margin_dpo/margin_std": 118.86153411865234,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -142.07090759277344,
      "KL/mean": -187.0964813232422,
      "KL/rejected_KL_mean": -232.12203979492188,
      "KL/std": 98.94548034667969,
      "epoch": 0.8,
      "fcm_dpo/beta": 0.0056334505788981915,
      "fcm_dpo/delta": -0.06081090867519379,
      "fcm_dpo/margin": 90.05111694335938,
      "fcm_dpo/q_t": 0.3859347701072693,
      "grad_norm": 74.7948989868164,
      "learning_rate": 5.9275697051098275e-08,
      "logits/chosen": -0.8558260202407837,
      "logits/rejected": -0.8512413501739502,
      "logps/chosen": -431.28240966796875,
      "logps/ref_chosen": -289.2114562988281,
      "logps/ref_rejected": -278.45751953125,
      "logps/rejected": -510.57952880859375,
      "loss": 4.1722,
      "margin_dpo/margin_mean": 90.05110931396484,
      "margin_dpo/margin_std": 112.73042297363281,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -143.9033966064453,
      "KL/mean": -184.68914794921875,
      "KL/rejected_KL_mean": -225.47491455078125,
      "KL/std": 105.05400085449219,
      "epoch": 0.8020942408376963,
      "fcm_dpo/beta": 0.005427788943052292,
      "fcm_dpo/delta": -0.0274592787027359,
      "fcm_dpo/margin": 81.57150268554688,
      "fcm_dpo/q_t": 0.40093404054641724,
      "grad_norm": 75.9984359741211,
      "learning_rate": 5.809719583454414e-08,
      "logits/chosen": -0.8348918557167053,
      "logits/rejected": -0.8166983127593994,
      "logps/chosen": -417.53424072265625,
      "logps/ref_chosen": -273.630859375,
      "logps/ref_rejected": -261.44024658203125,
      "logps/rejected": -486.9151306152344,
      "loss": 4.3787,
      "margin_dpo/margin_mean": 81.57150268554688,
      "margin_dpo/margin_std": 118.5652084350586,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -150.695556640625,
      "KL/mean": -186.2789306640625,
      "KL/rejected_KL_mean": -221.86228942871094,
      "KL/std": 92.42000579833984,
      "epoch": 0.8041884816753927,
      "fcm_dpo/beta": 0.005389348603785038,
      "fcm_dpo/delta": 0.027727246284484863,
      "fcm_dpo/margin": 71.1667709350586,
      "fcm_dpo/q_t": 0.41352203488349915,
      "grad_norm": 76.84312438964844,
      "learning_rate": 5.6928985782982524e-08,
      "logits/chosen": -0.8655426502227783,
      "logits/rejected": -0.8627970814704895,
      "logps/chosen": -425.2655334472656,
      "logps/ref_chosen": -274.5699462890625,
      "logps/ref_rejected": -285.8253479003906,
      "logps/rejected": -507.68768310546875,
      "loss": 4.5168,
      "margin_dpo/margin_mean": 71.1667709350586,
      "margin_dpo/margin_std": 112.27885437011719,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -154.55618286132812,
      "KL/mean": -188.289794921875,
      "KL/rejected_KL_mean": -222.02337646484375,
      "KL/std": 99.65026092529297,
      "epoch": 0.806282722513089,
      "fcm_dpo/beta": 0.0055604358203709126,
      "fcm_dpo/delta": 0.03444130718708038,
      "fcm_dpo/margin": 67.46717834472656,
      "fcm_dpo/q_t": 0.4155120849609375,
      "grad_norm": 74.75515747070312,
      "learning_rate": 5.57711295439732e-08,
      "logits/chosen": -0.8039891123771667,
      "logits/rejected": -0.8020448684692383,
      "logps/chosen": -438.70684814453125,
      "logps/ref_chosen": -284.150634765625,
      "logps/ref_rejected": -244.87921142578125,
      "logps/rejected": -466.90252685546875,
      "loss": 4.5197,
      "margin_dpo/margin_mean": 67.46717834472656,
      "margin_dpo/margin_std": 104.03783416748047,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -143.68930053710938,
      "KL/mean": -190.35922241210938,
      "KL/rejected_KL_mean": -237.02915954589844,
      "KL/std": 104.50996398925781,
      "epoch": 0.8083769633507853,
      "fcm_dpo/beta": 0.005284358747303486,
      "fcm_dpo/delta": -0.10094635933637619,
      "fcm_dpo/margin": 93.33988189697266,
      "fcm_dpo/q_t": 0.38872694969177246,
      "grad_norm": 85.22160339355469,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -0.7912386655807495,
      "logits/rejected": -0.7904696464538574,
      "logps/chosen": -463.86553955078125,
      "logps/ref_chosen": -320.1762390136719,
      "logps/ref_rejected": -302.05023193359375,
      "logps/rejected": -539.0794067382812,
      "loss": 4.1947,
      "margin_dpo/margin_mean": 93.33988189697266,
      "margin_dpo/margin_std": 112.16749572753906,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -155.53041076660156,
      "KL/mean": -193.96739196777344,
      "KL/rejected_KL_mean": -232.40438842773438,
      "KL/std": 99.8486328125,
      "epoch": 0.8104712041884817,
      "fcm_dpo/beta": 0.005203930661082268,
      "fcm_dpo/delta": 0.0009816847741603851,
      "fcm_dpo/margin": 76.87397766113281,
      "fcm_dpo/q_t": 0.40828564763069153,
      "grad_norm": 67.68667602539062,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": -0.827106773853302,
      "logits/rejected": -0.8312069773674011,
      "logps/chosen": -427.810546875,
      "logps/ref_chosen": -272.2801513671875,
      "logps/ref_rejected": -265.1615905761719,
      "logps/rejected": -497.56597900390625,
      "loss": 4.4864,
      "margin_dpo/margin_mean": 76.87397766113281,
      "margin_dpo/margin_std": 116.54380798339844,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -167.99729919433594,
      "KL/mean": -195.81471252441406,
      "KL/rejected_KL_mean": -223.63211059570312,
      "KL/std": 102.584228515625,
      "epoch": 0.812565445026178,
      "fcm_dpo/beta": 0.005073403939604759,
      "fcm_dpo/delta": -0.020425882190465927,
      "fcm_dpo/margin": 55.634796142578125,
      "fcm_dpo/q_t": 0.4362047016620636,
      "grad_norm": 87.78727722167969,
      "learning_rate": 5.2360301829254745e-08,
      "logits/chosen": -0.8104696273803711,
      "logits/rejected": -0.7995538711547852,
      "logps/chosen": -440.5286865234375,
      "logps/ref_chosen": -272.5313415527344,
      "logps/ref_rejected": -239.55735778808594,
      "logps/rejected": -463.1894836425781,
      "loss": 4.8899,
      "margin_dpo/margin_mean": 55.634796142578125,
      "margin_dpo/margin_std": 120.81311798095703,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -164.89100646972656,
      "KL/mean": -201.8444061279297,
      "KL/rejected_KL_mean": -238.7978057861328,
      "KL/std": 104.46894836425781,
      "epoch": 0.8146596858638744,
      "fcm_dpo/beta": 0.004981250502169132,
      "fcm_dpo/delta": -0.03677816316485405,
      "fcm_dpo/margin": 73.90679931640625,
      "fcm_dpo/q_t": 0.41805189847946167,
      "grad_norm": 88.72787475585938,
      "learning_rate": 5.1244476161413806e-08,
      "logits/chosen": -0.8534497618675232,
      "logits/rejected": -0.8520168662071228,
      "logps/chosen": -445.9802551269531,
      "logps/ref_chosen": -281.0892639160156,
      "logps/ref_rejected": -246.50045776367188,
      "logps/rejected": -485.2982177734375,
      "loss": 4.6045,
      "margin_dpo/margin_mean": 73.90679931640625,
      "margin_dpo/margin_std": 123.74664306640625,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -160.1143035888672,
      "KL/mean": -198.15748596191406,
      "KL/rejected_KL_mean": -236.20068359375,
      "KL/std": 103.58088684082031,
      "epoch": 0.8167539267015707,
      "fcm_dpo/beta": 0.005076872184872627,
      "fcm_dpo/delta": 0.033378101885318756,
      "fcm_dpo/margin": 76.08638000488281,
      "fcm_dpo/q_t": 0.4124943017959595,
      "grad_norm": 67.05355072021484,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": -0.863025426864624,
      "logits/rejected": -0.8708958625793457,
      "logps/chosen": -444.1017761230469,
      "logps/ref_chosen": -283.98748779296875,
      "logps/ref_rejected": -283.465087890625,
      "logps/rejected": -519.6658325195312,
      "loss": 4.4899,
      "margin_dpo/margin_mean": 76.08638000488281,
      "margin_dpo/margin_std": 115.08943939208984,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -173.00987243652344,
      "KL/mean": -213.27383422851562,
      "KL/rejected_KL_mean": -253.53778076171875,
      "KL/std": 112.33917236328125,
      "epoch": 0.818848167539267,
      "fcm_dpo/beta": 0.004998537711799145,
      "fcm_dpo/delta": 0.0024930015206336975,
      "fcm_dpo/margin": 80.52791595458984,
      "fcm_dpo/q_t": 0.4091907739639282,
      "grad_norm": 92.08856201171875,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -0.8173843622207642,
      "logits/rejected": -0.8072810769081116,
      "logps/chosen": -456.8712463378906,
      "logps/ref_chosen": -283.86138916015625,
      "logps/ref_rejected": -263.5093688964844,
      "logps/rejected": -517.0471801757812,
      "loss": 4.4525,
      "margin_dpo/margin_mean": 80.52790832519531,
      "margin_dpo/margin_std": 117.97828674316406,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -153.20596313476562,
      "KL/mean": -201.96731567382812,
      "KL/rejected_KL_mean": -250.72866821289062,
      "KL/std": 102.16930389404297,
      "epoch": 0.8209424083769633,
      "fcm_dpo/beta": 0.004816275089979172,
      "fcm_dpo/delta": -0.0636444166302681,
      "fcm_dpo/margin": 97.5226821899414,
      "fcm_dpo/q_t": 0.3912141025066376,
      "grad_norm": 71.26329040527344,
      "learning_rate": 4.796118758344353e-08,
      "logits/chosen": -0.7978020906448364,
      "logits/rejected": -0.8156266212463379,
      "logps/chosen": -463.27606201171875,
      "logps/ref_chosen": -310.070068359375,
      "logps/ref_rejected": -252.89817810058594,
      "logps/rejected": -503.6268310546875,
      "loss": 4.2022,
      "margin_dpo/margin_mean": 97.5226821899414,
      "margin_dpo/margin_std": 111.68810272216797,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -155.3448486328125,
      "KL/mean": -190.8988800048828,
      "KL/rejected_KL_mean": -226.45294189453125,
      "KL/std": 99.34536743164062,
      "epoch": 0.8230366492146597,
      "fcm_dpo/beta": 0.004986890591681004,
      "fcm_dpo/delta": 0.0614260658621788,
      "fcm_dpo/margin": 71.10809326171875,
      "fcm_dpo/q_t": 0.4194105565547943,
      "grad_norm": 63.78581237792969,
      "learning_rate": 4.688834983610082e-08,
      "logits/chosen": -0.8468508720397949,
      "logits/rejected": -0.831695556640625,
      "logps/chosen": -442.0604553222656,
      "logps/ref_chosen": -286.7156677246094,
      "logps/ref_rejected": -230.00357055664062,
      "logps/rejected": -456.45648193359375,
      "loss": 4.5747,
      "margin_dpo/margin_mean": 71.10809326171875,
      "margin_dpo/margin_std": 116.2547607421875,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -156.9377899169922,
      "KL/mean": -191.566650390625,
      "KL/rejected_KL_mean": -226.19552612304688,
      "KL/std": 108.4596176147461,
      "epoch": 0.8251308900523561,
      "fcm_dpo/beta": 0.005143392831087112,
      "fcm_dpo/delta": 0.04705846309661865,
      "fcm_dpo/margin": 69.25774383544922,
      "fcm_dpo/q_t": 0.4230070412158966,
      "grad_norm": 75.06909942626953,
      "learning_rate": 4.582640435014459e-08,
      "logits/chosen": -0.8808258175849915,
      "logits/rejected": -0.876298725605011,
      "logps/chosen": -482.9312438964844,
      "logps/ref_chosen": -325.9934387207031,
      "logps/ref_rejected": -317.42706298828125,
      "logps/rejected": -543.6226196289062,
      "loss": 4.6787,
      "margin_dpo/margin_mean": 69.25774383544922,
      "margin_dpo/margin_std": 126.5280532836914,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -153.951904296875,
      "KL/mean": -197.8990020751953,
      "KL/rejected_KL_mean": -241.84609985351562,
      "KL/std": 102.86681365966797,
      "epoch": 0.8272251308900523,
      "fcm_dpo/beta": 0.005283154547214508,
      "fcm_dpo/delta": -0.015460766851902008,
      "fcm_dpo/margin": 87.89419555664062,
      "fcm_dpo/q_t": 0.3990100920200348,
      "grad_norm": 72.6529312133789,
      "learning_rate": 4.477540807448832e-08,
      "logits/chosen": -0.828191339969635,
      "logits/rejected": -0.8377105593681335,
      "logps/chosen": -422.85272216796875,
      "logps/ref_chosen": -268.90081787109375,
      "logps/ref_rejected": -272.85809326171875,
      "logps/rejected": -514.7042236328125,
      "loss": 4.3011,
      "margin_dpo/margin_mean": 87.8941879272461,
      "margin_dpo/margin_std": 122.65936279296875,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -161.9087677001953,
      "KL/mean": -198.92108154296875,
      "KL/rejected_KL_mean": -235.93336486816406,
      "KL/std": 104.56163787841797,
      "epoch": 0.8293193717277487,
      "fcm_dpo/beta": 0.005232794210314751,
      "fcm_dpo/delta": 0.013093803077936172,
      "fcm_dpo/margin": 74.02458953857422,
      "fcm_dpo/q_t": 0.4126887917518616,
      "grad_norm": 91.1370620727539,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -0.834671139717102,
      "logits/rejected": -0.8187417984008789,
      "logps/chosen": -453.1070861816406,
      "logps/ref_chosen": -291.19830322265625,
      "logps/ref_rejected": -253.2803955078125,
      "logps/rejected": -489.2137451171875,
      "loss": 4.5134,
      "margin_dpo/margin_mean": 74.02458953857422,
      "margin_dpo/margin_std": 115.50848388671875,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -159.6127166748047,
      "KL/mean": -193.7106475830078,
      "KL/rejected_KL_mean": -227.80859375,
      "KL/std": 103.82569122314453,
      "epoch": 0.831413612565445,
      "fcm_dpo/beta": 0.005162069573998451,
      "fcm_dpo/delta": -0.05037570744752884,
      "fcm_dpo/margin": 68.19586944580078,
      "fcm_dpo/q_t": 0.4208472669124603,
      "grad_norm": 67.379638671875,
      "learning_rate": 4.270648801084295e-08,
      "logits/chosen": -0.8660804033279419,
      "logits/rejected": -0.8465254306793213,
      "logps/chosen": -469.4350891113281,
      "logps/ref_chosen": -309.8224182128906,
      "logps/ref_rejected": -291.9057922363281,
      "logps/rejected": -519.7144165039062,
      "loss": 4.7013,
      "margin_dpo/margin_mean": 68.19586944580078,
      "margin_dpo/margin_std": 122.6355972290039,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -175.7735595703125,
      "KL/mean": -206.456298828125,
      "KL/rejected_KL_mean": -237.13902282714844,
      "KL/std": 112.29230499267578,
      "epoch": 0.8335078534031414,
      "fcm_dpo/beta": 0.005021877586841583,
      "fcm_dpo/delta": 0.04257064312696457,
      "fcm_dpo/margin": 61.3654670715332,
      "fcm_dpo/q_t": 0.42978501319885254,
      "grad_norm": 99.95638275146484,
      "learning_rate": 4.168867517275806e-08,
      "logits/chosen": -0.7550973892211914,
      "logits/rejected": -0.7929383516311646,
      "logps/chosen": -473.58709716796875,
      "logps/ref_chosen": -297.8135070800781,
      "logps/ref_rejected": -270.5025634765625,
      "logps/rejected": -507.6416015625,
      "loss": 4.9284,
      "margin_dpo/margin_mean": 61.36547088623047,
      "margin_dpo/margin_std": 139.70736694335938,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -171.309814453125,
      "KL/mean": -207.6065673828125,
      "KL/rejected_KL_mean": -243.90328979492188,
      "KL/std": 106.35570526123047,
      "epoch": 0.8356020942408376,
      "fcm_dpo/beta": 0.005257927812635899,
      "fcm_dpo/delta": 0.024527132511138916,
      "fcm_dpo/margin": 72.59346771240234,
      "fcm_dpo/q_t": 0.4156295359134674,
      "grad_norm": 99.00328826904297,
      "learning_rate": 4.0682033438831584e-08,
      "logits/chosen": -0.8564267158508301,
      "logits/rejected": -0.8187931776046753,
      "logps/chosen": -464.1565856933594,
      "logps/ref_chosen": -292.8467712402344,
      "logps/ref_rejected": -268.3638916015625,
      "logps/rejected": -512.2672119140625,
      "loss": 4.5855,
      "margin_dpo/margin_mean": 72.59347534179688,
      "margin_dpo/margin_std": 124.75273132324219,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -163.8533172607422,
      "KL/mean": -200.3409881591797,
      "KL/rejected_KL_mean": -236.8286590576172,
      "KL/std": 102.72570037841797,
      "epoch": 0.837696335078534,
      "fcm_dpo/beta": 0.005257011856883764,
      "fcm_dpo/delta": -0.003097481094300747,
      "fcm_dpo/margin": 72.9753189086914,
      "fcm_dpo/q_t": 0.41297653317451477,
      "grad_norm": 83.04850769042969,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": -0.8974708318710327,
      "logits/rejected": -0.8973955512046814,
      "logps/chosen": -427.52972412109375,
      "logps/ref_chosen": -263.6763916015625,
      "logps/ref_rejected": -258.67266845703125,
      "logps/rejected": -495.5013427734375,
      "loss": 4.5768,
      "margin_dpo/margin_mean": 72.97531127929688,
      "margin_dpo/margin_std": 118.35626220703125,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_KL/chosen_KL_mean": -164.89222717285156,
      "eval_KL/mean": -203.09072875976562,
      "eval_KL/rejected_KL_mean": -241.28921508789062,
      "eval_KL/std": 105.05392456054688,
      "eval_fcm_dpo/beta": 0.0053019337356090546,
      "eval_logits/chosen": -0.8540005087852478,
      "eval_logits/rejected": -0.8400713205337524,
      "eval_logps/chosen": -452.7190246582031,
      "eval_logps/ref_chosen": -287.8268127441406,
      "eval_logps/ref_rejected": -266.9300231933594,
      "eval_logps/rejected": -508.21923828125,
      "eval_loss": 0.5654463768005371,
      "eval_margin_dpo/margin_mean": 76.39701080322266,
      "eval_margin_dpo/margin_std": 117.35706329345703,
      "eval_runtime": 78.488,
      "eval_samples_per_second": 25.482,
      "eval_steps_per_second": 3.185,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -162.49705505371094,
      "KL/mean": -202.2840118408203,
      "KL/rejected_KL_mean": -242.07098388671875,
      "KL/std": 106.80648803710938,
      "epoch": 0.8397905759162304,
      "fcm_dpo/beta": 0.005323378369212151,
      "fcm_dpo/delta": -0.005477304104715586,
      "fcm_dpo/margin": 79.57393646240234,
      "fcm_dpo/q_t": 0.40523409843444824,
      "grad_norm": 89.19438934326172,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -0.8266849517822266,
      "logits/rejected": -0.8267785906791687,
      "logps/chosen": -480.78240966796875,
      "logps/ref_chosen": -318.2853088378906,
      "logps/ref_rejected": -293.75225830078125,
      "logps/rejected": -535.8231811523438,
      "loss": 4.4469,
      "margin_dpo/margin_mean": 79.57393646240234,
      "margin_dpo/margin_std": 122.01748657226562,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -157.3339080810547,
      "KL/mean": -195.322998046875,
      "KL/rejected_KL_mean": -233.31210327148438,
      "KL/std": 107.65657806396484,
      "epoch": 0.8418848167539267,
      "fcm_dpo/beta": 0.005396377760916948,
      "fcm_dpo/delta": 0.04138898849487305,
      "fcm_dpo/margin": 75.97816467285156,
      "fcm_dpo/q_t": 0.4098457396030426,
      "grad_norm": 97.6332778930664,
      "learning_rate": 3.772967168071517e-08,
      "logits/chosen": -0.893134593963623,
      "logits/rejected": -0.8696551322937012,
      "logps/chosen": -466.76177978515625,
      "logps/ref_chosen": -309.4278564453125,
      "logps/ref_rejected": -282.0279846191406,
      "logps/rejected": -515.340087890625,
      "loss": 4.4642,
      "margin_dpo/margin_mean": 75.97816467285156,
      "margin_dpo/margin_std": 120.94024658203125,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -156.88931274414062,
      "KL/mean": -208.34542846679688,
      "KL/rejected_KL_mean": -259.80157470703125,
      "KL/std": 111.80844116210938,
      "epoch": 0.8439790575916231,
      "fcm_dpo/beta": 0.005283900536596775,
      "fcm_dpo/delta": -0.09864435344934464,
      "fcm_dpo/margin": 102.91226959228516,
      "fcm_dpo/q_t": 0.37841156125068665,
      "grad_norm": 84.92820739746094,
      "learning_rate": 3.676824816087978e-08,
      "logits/chosen": -0.8652935028076172,
      "logits/rejected": -0.8444188237190247,
      "logps/chosen": -465.9178161621094,
      "logps/ref_chosen": -309.0284729003906,
      "logps/ref_rejected": -272.9622497558594,
      "logps/rejected": -532.7638549804688,
      "loss": 4.0275,
      "margin_dpo/margin_mean": 102.91226196289062,
      "margin_dpo/margin_std": 116.59791564941406,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -171.84939575195312,
      "KL/mean": -210.167236328125,
      "KL/rejected_KL_mean": -248.48509216308594,
      "KL/std": 111.88416290283203,
      "epoch": 0.8460732984293193,
      "fcm_dpo/beta": 0.005103566683828831,
      "fcm_dpo/delta": 0.030192140489816666,
      "fcm_dpo/margin": 76.63570404052734,
      "fcm_dpo/q_t": 0.4123821556568146,
      "grad_norm": 80.14241790771484,
      "learning_rate": 3.581825961277074e-08,
      "logits/chosen": -0.8965409994125366,
      "logits/rejected": -0.8733291625976562,
      "logps/chosen": -469.133056640625,
      "logps/ref_chosen": -297.2837219238281,
      "logps/ref_rejected": -256.99041748046875,
      "logps/rejected": -505.47552490234375,
      "loss": 4.5587,
      "margin_dpo/margin_mean": 76.63569641113281,
      "margin_dpo/margin_std": 128.69839477539062,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -162.85931396484375,
      "KL/mean": -203.57318115234375,
      "KL/rejected_KL_mean": -244.28707885742188,
      "KL/std": 108.88365173339844,
      "epoch": 0.8481675392670157,
      "fcm_dpo/beta": 0.005240336991846561,
      "fcm_dpo/delta": 0.02371572144329548,
      "fcm_dpo/margin": 81.4277572631836,
      "fcm_dpo/q_t": 0.4049849510192871,
      "grad_norm": 56.64188766479492,
      "learning_rate": 3.487975698139084e-08,
      "logits/chosen": -0.8054448366165161,
      "logits/rejected": -0.8122345805168152,
      "logps/chosen": -420.8246154785156,
      "logps/ref_chosen": -257.96533203125,
      "logps/ref_rejected": -255.811279296875,
      "logps/rejected": -500.0983581542969,
      "loss": 4.4074,
      "margin_dpo/margin_mean": 81.42776489257812,
      "margin_dpo/margin_std": 121.74008178710938,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -175.34703063964844,
      "KL/mean": -206.49766540527344,
      "KL/rejected_KL_mean": -237.6482696533203,
      "KL/std": 104.31028747558594,
      "epoch": 0.8502617801047121,
      "fcm_dpo/beta": 0.005294849164783955,
      "fcm_dpo/delta": 0.00797463208436966,
      "fcm_dpo/margin": 62.30122375488281,
      "fcm_dpo/q_t": 0.42453256249427795,
      "grad_norm": 101.14784240722656,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -0.8528972268104553,
      "logits/rejected": -0.8299845457077026,
      "logps/chosen": -460.5281066894531,
      "logps/ref_chosen": -285.1810607910156,
      "logps/ref_rejected": -264.41351318359375,
      "logps/rejected": -502.06182861328125,
      "loss": 4.6778,
      "margin_dpo/margin_mean": 62.30121612548828,
      "margin_dpo/margin_std": 109.26322937011719,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -178.4529571533203,
      "KL/mean": -218.6519012451172,
      "KL/rejected_KL_mean": -258.85089111328125,
      "KL/std": 111.62689971923828,
      "epoch": 0.8523560209424084,
      "fcm_dpo/beta": 0.005344776436686516,
      "fcm_dpo/delta": 0.02056296542286873,
      "fcm_dpo/margin": 80.39791870117188,
      "fcm_dpo/q_t": 0.4063396453857422,
      "grad_norm": 80.80738067626953,
      "learning_rate": 3.303741016635614e-08,
      "logits/chosen": -0.8416044116020203,
      "logits/rejected": -0.8644787669181824,
      "logps/chosen": -443.6910705566406,
      "logps/ref_chosen": -265.23809814453125,
      "logps/ref_rejected": -219.0631561279297,
      "logps/rejected": -477.9140625,
      "loss": 4.4668,
      "margin_dpo/margin_mean": 80.39791870117188,
      "margin_dpo/margin_std": 128.4066925048828,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -159.2659912109375,
      "KL/mean": -199.16773986816406,
      "KL/rejected_KL_mean": -239.0694580078125,
      "KL/std": 104.24473571777344,
      "epoch": 0.8544502617801047,
      "fcm_dpo/beta": 0.005535793490707874,
      "fcm_dpo/delta": 0.006213851273059845,
      "fcm_dpo/margin": 79.80348205566406,
      "fcm_dpo/q_t": 0.40345457196235657,
      "grad_norm": 91.17036437988281,
      "learning_rate": 3.2133664782169944e-08,
      "logits/chosen": -0.877729594707489,
      "logits/rejected": -0.8713379502296448,
      "logps/chosen": -456.23858642578125,
      "logps/ref_chosen": -296.9726257324219,
      "logps/ref_rejected": -295.4786376953125,
      "logps/rejected": -534.548095703125,
      "loss": 4.4156,
      "margin_dpo/margin_mean": 79.80347442626953,
      "margin_dpo/margin_std": 118.69510650634766,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -168.13027954101562,
      "KL/mean": -208.9474639892578,
      "KL/rejected_KL_mean": -249.76466369628906,
      "KL/std": 115.43540954589844,
      "epoch": 0.856544502617801,
      "fcm_dpo/beta": 0.0053654080256819725,
      "fcm_dpo/delta": -0.029886584728956223,
      "fcm_dpo/margin": 81.6343994140625,
      "fcm_dpo/q_t": 0.40363845229148865,
      "grad_norm": 66.33670806884766,
      "learning_rate": 3.12416029083514e-08,
      "logits/chosen": -0.8552312254905701,
      "logits/rejected": -0.8411694765090942,
      "logps/chosen": -455.5096435546875,
      "logps/ref_chosen": -287.37933349609375,
      "logps/ref_rejected": -275.80291748046875,
      "logps/rejected": -525.5675659179688,
      "loss": 4.505,
      "margin_dpo/margin_mean": 81.6343994140625,
      "margin_dpo/margin_std": 138.33746337890625,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -163.54208374023438,
      "KL/mean": -201.466796875,
      "KL/rejected_KL_mean": -239.39149475097656,
      "KL/std": 104.40740966796875,
      "epoch": 0.8586387434554974,
      "fcm_dpo/beta": 0.005275522358715534,
      "fcm_dpo/delta": 0.016677306964993477,
      "fcm_dpo/margin": 75.84941864013672,
      "fcm_dpo/q_t": 0.41182762384414673,
      "grad_norm": 79.84806060791016,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": -0.8642200231552124,
      "logits/rejected": -0.8718496561050415,
      "logps/chosen": -445.322265625,
      "logps/ref_chosen": -281.7801818847656,
      "logps/ref_rejected": -266.7550354003906,
      "logps/rejected": -506.14654541015625,
      "loss": 4.5451,
      "margin_dpo/margin_mean": 75.84941101074219,
      "margin_dpo/margin_std": 126.93839263916016,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -158.9381866455078,
      "KL/mean": -203.3583221435547,
      "KL/rejected_KL_mean": -247.7784423828125,
      "KL/std": 112.47294616699219,
      "epoch": 0.8607329842931937,
      "fcm_dpo/beta": 0.005286515690386295,
      "fcm_dpo/delta": -0.022061126306653023,
      "fcm_dpo/margin": 88.84026336669922,
      "fcm_dpo/q_t": 0.39569759368896484,
      "grad_norm": 73.40019989013672,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -0.8600984811782837,
      "logits/rejected": -0.8178382515907288,
      "logps/chosen": -440.52545166015625,
      "logps/ref_chosen": -281.5872497558594,
      "logps/ref_rejected": -254.78916931152344,
      "logps/rejected": -502.567626953125,
      "loss": 4.2457,
      "margin_dpo/margin_mean": 88.84025573730469,
      "margin_dpo/margin_std": 114.49974822998047,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -177.669677734375,
      "KL/mean": -212.2499237060547,
      "KL/rejected_KL_mean": -246.8301544189453,
      "KL/std": 109.3361587524414,
      "epoch": 0.86282722513089,
      "fcm_dpo/beta": 0.005120260175317526,
      "fcm_dpo/delta": -0.007789885625243187,
      "fcm_dpo/margin": 69.16047668457031,
      "fcm_dpo/q_t": 0.4218669831752777,
      "grad_norm": 74.58943939208984,
      "learning_rate": 2.863599358669755e-08,
      "logits/chosen": -0.8310728669166565,
      "logits/rejected": -0.8415468335151672,
      "logps/chosen": -454.20379638671875,
      "logps/ref_chosen": -276.5341796875,
      "logps/ref_rejected": -273.8751220703125,
      "logps/rejected": -520.7052612304688,
      "loss": 4.65,
      "margin_dpo/margin_mean": 69.16047668457031,
      "margin_dpo/margin_std": 120.7418212890625,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -167.18284606933594,
      "KL/mean": -199.99412536621094,
      "KL/rejected_KL_mean": -232.80543518066406,
      "KL/std": 109.3698501586914,
      "epoch": 0.8649214659685864,
      "fcm_dpo/beta": 0.0054544368758797646,
      "fcm_dpo/delta": 0.09568115323781967,
      "fcm_dpo/margin": 65.62258911132812,
      "fcm_dpo/q_t": 0.42092451453208923,
      "grad_norm": 99.38496398925781,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": -0.8510812520980835,
      "logits/rejected": -0.8496342897415161,
      "logps/chosen": -438.4573974609375,
      "logps/ref_chosen": -271.2745666503906,
      "logps/ref_rejected": -270.16912841796875,
      "logps/rejected": -502.9745788574219,
      "loss": 4.634,
      "margin_dpo/margin_mean": 65.6225814819336,
      "margin_dpo/margin_std": 116.8655014038086,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -165.31759643554688,
      "KL/mean": -208.4487762451172,
      "KL/rejected_KL_mean": -251.5799560546875,
      "KL/std": 105.42410278320312,
      "epoch": 0.8670157068062827,
      "fcm_dpo/beta": 0.005637951660901308,
      "fcm_dpo/delta": -0.03902636468410492,
      "fcm_dpo/margin": 86.26231384277344,
      "fcm_dpo/q_t": 0.3933212459087372,
      "grad_norm": 80.98966979980469,
      "learning_rate": 2.6958198472749717e-08,
      "logits/chosen": -0.8857486248016357,
      "logits/rejected": -0.8864164352416992,
      "logps/chosen": -462.4326477050781,
      "logps/ref_chosen": -297.11505126953125,
      "logps/ref_rejected": -271.7034606933594,
      "logps/rejected": -523.2833862304688,
      "loss": 4.2615,
      "margin_dpo/margin_mean": 86.26231384277344,
      "margin_dpo/margin_std": 116.04374694824219,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -160.324951171875,
      "KL/mean": -199.44869995117188,
      "KL/rejected_KL_mean": -238.57247924804688,
      "KL/std": 100.03067016601562,
      "epoch": 0.8691099476439791,
      "fcm_dpo/beta": 0.005566161125898361,
      "fcm_dpo/delta": 0.014400789514183998,
      "fcm_dpo/margin": 78.24754333496094,
      "fcm_dpo/q_t": 0.40194499492645264,
      "grad_norm": 80.29901123046875,
      "learning_rate": 2.613722016414943e-08,
      "logits/chosen": -0.8861839771270752,
      "logits/rejected": -0.8721035122871399,
      "logps/chosen": -458.0176086425781,
      "logps/ref_chosen": -297.6926574707031,
      "logps/ref_rejected": -279.0503234863281,
      "logps/rejected": -517.622802734375,
      "loss": 4.3507,
      "margin_dpo/margin_mean": 78.24754333496094,
      "margin_dpo/margin_std": 110.00711059570312,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -151.757568359375,
      "KL/mean": -198.45835876464844,
      "KL/rejected_KL_mean": -245.1591339111328,
      "KL/std": 103.57637786865234,
      "epoch": 0.8712041884816754,
      "fcm_dpo/beta": 0.005360407754778862,
      "fcm_dpo/delta": -0.05554232373833656,
      "fcm_dpo/margin": 93.40155029296875,
      "fcm_dpo/q_t": 0.38735538721084595,
      "grad_norm": 76.16287994384766,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -0.8847571611404419,
      "logits/rejected": -0.8882590532302856,
      "logps/chosen": -463.5831298828125,
      "logps/ref_chosen": -311.8255615234375,
      "logps/ref_rejected": -268.6170654296875,
      "logps/rejected": -513.7762451171875,
      "loss": 4.2014,
      "margin_dpo/margin_mean": 93.40155029296875,
      "margin_dpo/margin_std": 119.23721313476562,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -167.6260986328125,
      "KL/mean": -205.4844512939453,
      "KL/rejected_KL_mean": -243.3428192138672,
      "KL/std": 107.53627014160156,
      "epoch": 0.8732984293193717,
      "fcm_dpo/beta": 0.005166263319551945,
      "fcm_dpo/delta": -0.05792851746082306,
      "fcm_dpo/margin": 75.71672058105469,
      "fcm_dpo/q_t": 0.413451224565506,
      "grad_norm": 86.92569732666016,
      "learning_rate": 2.4531322174210973e-08,
      "logits/chosen": -0.829699695110321,
      "logits/rejected": -0.82613605260849,
      "logps/chosen": -478.06292724609375,
      "logps/ref_chosen": -310.43682861328125,
      "logps/ref_rejected": -277.15283203125,
      "logps/rejected": -520.4956665039062,
      "loss": 4.5638,
      "margin_dpo/margin_mean": 75.71672058105469,
      "margin_dpo/margin_std": 124.438232421875,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -176.69129943847656,
      "KL/mean": -214.87091064453125,
      "KL/rejected_KL_mean": -253.05055236816406,
      "KL/std": 114.29906463623047,
      "epoch": 0.875392670157068,
      "fcm_dpo/beta": 0.004977349657565355,
      "fcm_dpo/delta": -0.04639672487974167,
      "fcm_dpo/margin": 76.35926818847656,
      "fcm_dpo/q_t": 0.41482898592948914,
      "grad_norm": 68.1673583984375,
      "learning_rate": 2.3746488612308295e-08,
      "logits/chosen": -0.826282262802124,
      "logits/rejected": -0.80389803647995,
      "logps/chosen": -455.18719482421875,
      "logps/ref_chosen": -278.49591064453125,
      "logps/ref_rejected": -276.56671142578125,
      "logps/rejected": -529.6172485351562,
      "loss": 4.5625,
      "margin_dpo/margin_mean": 76.35926818847656,
      "margin_dpo/margin_std": 123.32653045654297,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -177.27310180664062,
      "KL/mean": -217.8011016845703,
      "KL/rejected_KL_mean": -258.3291320800781,
      "KL/std": 112.32427978515625,
      "epoch": 0.8774869109947644,
      "fcm_dpo/beta": 0.004920286126434803,
      "fcm_dpo/delta": 0.015617555007338524,
      "fcm_dpo/margin": 81.0560302734375,
      "fcm_dpo/q_t": 0.40922775864601135,
      "grad_norm": 92.53379821777344,
      "learning_rate": 2.297378833957761e-08,
      "logits/chosen": -0.8714659810066223,
      "logits/rejected": -0.8457875847816467,
      "logps/chosen": -476.17333984375,
      "logps/ref_chosen": -298.9002380371094,
      "logps/ref_rejected": -246.1540985107422,
      "logps/rejected": -504.48321533203125,
      "loss": 4.513,
      "margin_dpo/margin_mean": 81.0560302734375,
      "margin_dpo/margin_std": 129.5267333984375,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -172.4613800048828,
      "KL/mean": -218.99417114257812,
      "KL/rejected_KL_mean": -265.5269775390625,
      "KL/std": 117.44099426269531,
      "epoch": 0.8795811518324608,
      "fcm_dpo/beta": 0.004903985187411308,
      "fcm_dpo/delta": -0.007674246095120907,
      "fcm_dpo/margin": 93.06561279296875,
      "fcm_dpo/q_t": 0.3993038237094879,
      "grad_norm": 89.03120422363281,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": -0.8212546110153198,
      "logits/rejected": -0.7954023480415344,
      "logps/chosen": -437.0222473144531,
      "logps/ref_chosen": -264.5608825683594,
      "logps/ref_rejected": -245.67031860351562,
      "logps/rejected": -511.19732666015625,
      "loss": 4.4111,
      "margin_dpo/margin_mean": 93.06560516357422,
      "margin_dpo/margin_std": 146.12716674804688,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -153.9785614013672,
      "KL/mean": -195.23495483398438,
      "KL/rejected_KL_mean": -236.49136352539062,
      "KL/std": 102.88247680664062,
      "epoch": 0.881675392670157,
      "fcm_dpo/beta": 0.004906922113150358,
      "fcm_dpo/delta": 0.005978137254714966,
      "fcm_dpo/margin": 82.51280975341797,
      "fcm_dpo/q_t": 0.4064605236053467,
      "grad_norm": 108.0601577758789,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -0.894304096698761,
      "logits/rejected": -0.8736079335212708,
      "logps/chosen": -451.6835632324219,
      "logps/ref_chosen": -297.70501708984375,
      "logps/ref_rejected": -243.74771118164062,
      "logps/rejected": -480.2391052246094,
      "loss": 4.3985,
      "margin_dpo/margin_mean": 82.51280975341797,
      "margin_dpo/margin_std": 113.22681427001953,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -164.56703186035156,
      "KL/mean": -204.88162231445312,
      "KL/rejected_KL_mean": -245.19622802734375,
      "KL/std": 110.78426361083984,
      "epoch": 0.8837696335078534,
      "fcm_dpo/beta": 0.005090317688882351,
      "fcm_dpo/delta": 0.040271662175655365,
      "fcm_dpo/margin": 80.62918853759766,
      "fcm_dpo/q_t": 0.40936410427093506,
      "grad_norm": 63.56366729736328,
      "learning_rate": 2.07288983654679e-08,
      "logits/chosen": -0.7540973424911499,
      "logits/rejected": -0.7999140620231628,
      "logps/chosen": -452.92578125,
      "logps/ref_chosen": -288.3587646484375,
      "logps/ref_rejected": -256.4377746582031,
      "logps/rejected": -501.6340026855469,
      "loss": 4.4943,
      "margin_dpo/margin_mean": 80.62918853759766,
      "margin_dpo/margin_std": 131.133544921875,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -168.51727294921875,
      "KL/mean": -210.96209716796875,
      "KL/rejected_KL_mean": -253.40689086914062,
      "KL/std": 105.04615783691406,
      "epoch": 0.8858638743455497,
      "fcm_dpo/beta": 0.005041640251874924,
      "fcm_dpo/delta": -0.01042180135846138,
      "fcm_dpo/margin": 84.88963317871094,
      "fcm_dpo/q_t": 0.4035080671310425,
      "grad_norm": 81.02098083496094,
      "learning_rate": 2.0005139085293942e-08,
      "logits/chosen": -0.8793633580207825,
      "logits/rejected": -0.8597881197929382,
      "logps/chosen": -464.5243225097656,
      "logps/ref_chosen": -296.00701904296875,
      "logps/ref_rejected": -261.3480529785156,
      "logps/rejected": -514.7548828125,
      "loss": 4.4021,
      "margin_dpo/margin_mean": 84.88963317871094,
      "margin_dpo/margin_std": 124.96765899658203,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -155.43612670898438,
      "KL/mean": -196.52139282226562,
      "KL/rejected_KL_mean": -237.60667419433594,
      "KL/std": 100.2783432006836,
      "epoch": 0.8879581151832461,
      "fcm_dpo/beta": 0.005119968205690384,
      "fcm_dpo/delta": 0.02945992723107338,
      "fcm_dpo/margin": 82.17054748535156,
      "fcm_dpo/q_t": 0.40422746539115906,
      "grad_norm": 95.59243774414062,
      "learning_rate": 1.9293713731512673e-08,
      "logits/chosen": -0.8579764366149902,
      "logits/rejected": -0.8521823883056641,
      "logps/chosen": -464.85797119140625,
      "logps/ref_chosen": -309.421875,
      "logps/ref_rejected": -249.14886474609375,
      "logps/rejected": -486.75555419921875,
      "loss": 4.3527,
      "margin_dpo/margin_mean": 82.17054748535156,
      "margin_dpo/margin_std": 108.03909301757812,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -165.78961181640625,
      "KL/mean": -201.8004150390625,
      "KL/rejected_KL_mean": -237.8112335205078,
      "KL/std": 107.87975311279297,
      "epoch": 0.8900523560209425,
      "fcm_dpo/beta": 0.005168507341295481,
      "fcm_dpo/delta": -0.003047725185751915,
      "fcm_dpo/margin": 72.0216293334961,
      "fcm_dpo/q_t": 0.41708609461784363,
      "grad_norm": 105.23089599609375,
      "learning_rate": 1.8594660455706763e-08,
      "logits/chosen": -0.8384729623794556,
      "logits/rejected": -0.8426160216331482,
      "logps/chosen": -446.2987060546875,
      "logps/ref_chosen": -280.50909423828125,
      "logps/ref_rejected": -276.8252258300781,
      "logps/rejected": -514.636474609375,
      "loss": 4.5912,
      "margin_dpo/margin_mean": 72.0216293334961,
      "margin_dpo/margin_std": 121.20518493652344,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -168.3882598876953,
      "KL/mean": -210.25863647460938,
      "KL/rejected_KL_mean": -252.1290283203125,
      "KL/std": 113.96662139892578,
      "epoch": 0.8921465968586387,
      "fcm_dpo/beta": 0.005218572914600372,
      "fcm_dpo/delta": 0.012883099727332592,
      "fcm_dpo/margin": 83.74078369140625,
      "fcm_dpo/q_t": 0.40226829051971436,
      "grad_norm": 74.78345489501953,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -0.8745719194412231,
      "logits/rejected": -0.8579055070877075,
      "logps/chosen": -461.1734924316406,
      "logps/ref_chosen": -292.78521728515625,
      "logps/ref_rejected": -255.62698364257812,
      "logps/rejected": -507.75604248046875,
      "loss": 4.3839,
      "margin_dpo/margin_mean": 83.74079132080078,
      "margin_dpo/margin_std": 120.8238754272461,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -162.7379150390625,
      "KL/mean": -216.88710021972656,
      "KL/rejected_KL_mean": -271.0362854003906,
      "KL/std": 120.43949127197266,
      "epoch": 0.8942408376963351,
      "fcm_dpo/beta": 0.005039297975599766,
      "fcm_dpo/delta": -0.13903112709522247,
      "fcm_dpo/margin": 108.29835510253906,
      "fcm_dpo/q_t": 0.3800096809864044,
      "grad_norm": 88.75221252441406,
      "learning_rate": 1.7233819424956247e-08,
      "logits/chosen": -0.8478140830993652,
      "logits/rejected": -0.814448356628418,
      "logps/chosen": -451.50665283203125,
      "logps/ref_chosen": -288.7687072753906,
      "logps/ref_rejected": -268.4986572265625,
      "logps/rejected": -539.5349731445312,
      "loss": 4.1674,
      "margin_dpo/margin_mean": 108.29835510253906,
      "margin_dpo/margin_std": 139.79736328125,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -183.75950622558594,
      "KL/mean": -233.40191650390625,
      "KL/rejected_KL_mean": -283.04437255859375,
      "KL/std": 121.09063720703125,
      "epoch": 0.8963350785340314,
      "fcm_dpo/beta": 0.004665692802518606,
      "fcm_dpo/delta": -0.01456773467361927,
      "fcm_dpo/margin": 99.28483581542969,
      "fcm_dpo/q_t": 0.39457541704177856,
      "grad_norm": 78.49534606933594,
      "learning_rate": 1.6572104647786245e-08,
      "logits/chosen": -0.8163129687309265,
      "logits/rejected": -0.8462315797805786,
      "logps/chosen": -479.28045654296875,
      "logps/ref_chosen": -295.5209655761719,
      "logps/ref_rejected": -275.71026611328125,
      "logps/rejected": -558.754638671875,
      "loss": 4.2682,
      "margin_dpo/margin_mean": 99.28485107421875,
      "margin_dpo/margin_std": 129.79824829101562,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -171.76539611816406,
      "KL/mean": -214.47125244140625,
      "KL/rejected_KL_mean": -257.1771240234375,
      "KL/std": 115.47428131103516,
      "epoch": 0.8984293193717278,
      "fcm_dpo/beta": 0.0046165199019014835,
      "fcm_dpo/delta": -0.005352715030312538,
      "fcm_dpo/margin": 85.41175842285156,
      "fcm_dpo/q_t": 0.4087330102920532,
      "grad_norm": 88.50299072265625,
      "learning_rate": 1.5922907900227017e-08,
      "logits/chosen": -0.8064049482345581,
      "logits/rejected": -0.8116366863250732,
      "logps/chosen": -446.15771484375,
      "logps/ref_chosen": -274.392333984375,
      "logps/ref_rejected": -258.574462890625,
      "logps/rejected": -515.7515869140625,
      "loss": 4.5508,
      "margin_dpo/margin_mean": 85.4117660522461,
      "margin_dpo/margin_std": 138.73605346679688,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -169.39431762695312,
      "KL/mean": -206.14993286132812,
      "KL/rejected_KL_mean": -242.90557861328125,
      "KL/std": 105.4596176147461,
      "epoch": 0.900523560209424,
      "fcm_dpo/beta": 0.00470721535384655,
      "fcm_dpo/delta": 0.022220980376005173,
      "fcm_dpo/margin": 73.51124572753906,
      "fcm_dpo/q_t": 0.42204442620277405,
      "grad_norm": 97.17189025878906,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": -0.8815496563911438,
      "logits/rejected": -0.852321982383728,
      "logps/chosen": -458.1334228515625,
      "logps/ref_chosen": -288.7391357421875,
      "logps/ref_rejected": -268.6106262207031,
      "logps/rejected": -511.5162048339844,
      "loss": 4.6216,
      "margin_dpo/margin_mean": 73.51124572753906,
      "margin_dpo/margin_std": 123.12567138671875,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -167.99652099609375,
      "KL/mean": -200.49012756347656,
      "KL/rejected_KL_mean": -232.98370361328125,
      "KL/std": 101.47885131835938,
      "epoch": 0.9026178010471204,
      "fcm_dpo/beta": 0.004743383731693029,
      "fcm_dpo/delta": 0.010848462581634521,
      "fcm_dpo/margin": 64.9871826171875,
      "fcm_dpo/q_t": 0.4300733506679535,
      "grad_norm": 93.34741973876953,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -0.8661752939224243,
      "logits/rejected": -0.8336012959480286,
      "logps/chosen": -443.7212829589844,
      "logps/ref_chosen": -275.7247314453125,
      "logps/ref_rejected": -268.91729736328125,
      "logps/rejected": -501.9010009765625,
      "loss": 4.7098,
      "margin_dpo/margin_mean": 64.9871826171875,
      "margin_dpo/margin_std": 113.40322875976562,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -157.7758026123047,
      "KL/mean": -201.10763549804688,
      "KL/rejected_KL_mean": -244.43948364257812,
      "KL/std": 113.5224838256836,
      "epoch": 0.9047120418848168,
      "fcm_dpo/beta": 0.004710648208856583,
      "fcm_dpo/delta": 0.007381734438240528,
      "fcm_dpo/margin": 86.66366577148438,
      "fcm_dpo/q_t": 0.4094912111759186,
      "grad_norm": 64.65988159179688,
      "learning_rate": 1.40507706120426e-08,
      "logits/chosen": -0.8955056071281433,
      "logits/rejected": -0.8802809119224548,
      "logps/chosen": -449.19586181640625,
      "logps/ref_chosen": -291.42010498046875,
      "logps/ref_rejected": -255.48202514648438,
      "logps/rejected": -499.9215087890625,
      "loss": 4.4387,
      "margin_dpo/margin_mean": 86.66366577148438,
      "margin_dpo/margin_std": 129.47885131835938,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -175.79312133789062,
      "KL/mean": -212.71182250976562,
      "KL/rejected_KL_mean": -249.63052368164062,
      "KL/std": 105.85745239257812,
      "epoch": 0.9068062827225131,
      "fcm_dpo/beta": 0.004913164768368006,
      "fcm_dpo/delta": 0.0906573235988617,
      "fcm_dpo/margin": 73.83741760253906,
      "fcm_dpo/q_t": 0.4172857701778412,
      "grad_norm": 90.05935668945312,
      "learning_rate": 1.345198738661285e-08,
      "logits/chosen": -0.8436307907104492,
      "logits/rejected": -0.8390262126922607,
      "logps/chosen": -422.0200500488281,
      "logps/ref_chosen": -246.2268829345703,
      "logps/ref_rejected": -253.65924072265625,
      "logps/rejected": -503.289794921875,
      "loss": 4.5374,
      "margin_dpo/margin_mean": 73.83741760253906,
      "margin_dpo/margin_std": 115.48959350585938,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -182.51544189453125,
      "KL/mean": -220.29696655273438,
      "KL/rejected_KL_mean": -258.0785217285156,
      "KL/std": 110.51648712158203,
      "epoch": 0.9089005235602095,
      "fcm_dpo/beta": 0.005197789520025253,
      "fcm_dpo/delta": 0.021634036675095558,
      "fcm_dpo/margin": 75.56307220458984,
      "fcm_dpo/q_t": 0.4116612374782562,
      "grad_norm": 78.48800659179688,
      "learning_rate": 1.2865889513213628e-08,
      "logits/chosen": -0.850239634513855,
      "logits/rejected": -0.8575378656387329,
      "logps/chosen": -477.97723388671875,
      "logps/ref_chosen": -295.4618225097656,
      "logps/ref_rejected": -256.2254333496094,
      "logps/rejected": -514.3040161132812,
      "loss": 4.4999,
      "margin_dpo/margin_mean": 75.56307220458984,
      "margin_dpo/margin_std": 119.42990112304688,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -168.10305786132812,
      "KL/mean": -211.28720092773438,
      "KL/rejected_KL_mean": -254.4713134765625,
      "KL/std": 113.85639190673828,
      "epoch": 0.9109947643979057,
      "fcm_dpo/beta": 0.005123570561408997,
      "fcm_dpo/delta": -0.02720644138753414,
      "fcm_dpo/margin": 86.36825561523438,
      "fcm_dpo/q_t": 0.4020588994026184,
      "grad_norm": 103.86099243164062,
      "learning_rate": 1.2292508422495157e-08,
      "logits/chosen": -0.8403117060661316,
      "logits/rejected": -0.8277177214622498,
      "logps/chosen": -428.8414306640625,
      "logps/ref_chosen": -260.7384033203125,
      "logps/ref_rejected": -248.5688018798828,
      "logps/rejected": -503.04010009765625,
      "loss": 4.3267,
      "margin_dpo/margin_mean": 86.36825561523438,
      "margin_dpo/margin_std": 117.33106231689453,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -179.7392120361328,
      "KL/mean": -213.6044921875,
      "KL/rejected_KL_mean": -247.4697723388672,
      "KL/std": 122.3073501586914,
      "epoch": 0.9130890052356021,
      "fcm_dpo/beta": 0.005273106973618269,
      "fcm_dpo/delta": 0.09634209424257278,
      "fcm_dpo/margin": 67.73058319091797,
      "fcm_dpo/q_t": 0.42340925335884094,
      "grad_norm": 104.27674102783203,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -0.8317723870277405,
      "logits/rejected": -0.8298194408416748,
      "logps/chosen": -499.0617370605469,
      "logps/ref_chosen": -319.3224792480469,
      "logps/ref_rejected": -299.30322265625,
      "logps/rejected": -546.7730102539062,
      "loss": 4.706,
      "margin_dpo/margin_mean": 67.73057556152344,
      "margin_dpo/margin_std": 134.4886474609375,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -165.16844177246094,
      "KL/mean": -213.1919403076172,
      "KL/rejected_KL_mean": -261.2154235839844,
      "KL/std": 125.07614135742188,
      "epoch": 0.9151832460732985,
      "fcm_dpo/beta": 0.005230634473264217,
      "fcm_dpo/delta": -0.09450878202915192,
      "fcm_dpo/margin": 96.04700469970703,
      "fcm_dpo/q_t": 0.3893167972564697,
      "grad_norm": 84.06829833984375,
      "learning_rate": 1.118401890024001e-08,
      "logits/chosen": -0.8561391830444336,
      "logits/rejected": -0.8397589325904846,
      "logps/chosen": -443.9971923828125,
      "logps/ref_chosen": -278.82879638671875,
      "logps/ref_rejected": -272.55303955078125,
      "logps/rejected": -533.7684326171875,
      "loss": 4.2486,
      "margin_dpo/margin_mean": 96.04700469970703,
      "margin_dpo/margin_std": 130.83738708496094,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -174.8148956298828,
      "KL/mean": -197.31129455566406,
      "KL/rejected_KL_mean": -219.8076934814453,
      "KL/std": 109.91526794433594,
      "epoch": 0.9172774869109948,
      "fcm_dpo/beta": 0.00512558501213789,
      "fcm_dpo/delta": 0.03356669470667839,
      "fcm_dpo/margin": 44.992794036865234,
      "fcm_dpo/q_t": 0.44835174083709717,
      "grad_norm": 87.14521789550781,
      "learning_rate": 1.06489699136324e-08,
      "logits/chosen": -0.8403440713882446,
      "logits/rejected": -0.8568493723869324,
      "logps/chosen": -434.1340026855469,
      "logps/ref_chosen": -259.31903076171875,
      "logps/ref_rejected": -240.99581909179688,
      "logps/rejected": -460.80352783203125,
      "loss": 5.0917,
      "margin_dpo/margin_mean": 44.992794036865234,
      "margin_dpo/margin_std": 123.96754455566406,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -172.35157775878906,
      "KL/mean": -213.42881774902344,
      "KL/rejected_KL_mean": -254.5060577392578,
      "KL/std": 107.76215362548828,
      "epoch": 0.9193717277486911,
      "fcm_dpo/beta": 0.005223650950938463,
      "fcm_dpo/delta": 0.021350806578993797,
      "fcm_dpo/margin": 82.15447235107422,
      "fcm_dpo/q_t": 0.403455913066864,
      "grad_norm": 76.09151458740234,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": -0.8302324414253235,
      "logits/rejected": -0.8326482772827148,
      "logps/chosen": -429.4759521484375,
      "logps/ref_chosen": -257.1243896484375,
      "logps/ref_rejected": -243.20416259765625,
      "logps/rejected": -497.710205078125,
      "loss": 4.3867,
      "margin_dpo/margin_mean": 82.15447235107422,
      "margin_dpo/margin_std": 118.64126586914062,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -183.18096923828125,
      "KL/mean": -213.66024780273438,
      "KL/rejected_KL_mean": -244.1395263671875,
      "KL/std": 102.81112670898438,
      "epoch": 0.9214659685863874,
      "fcm_dpo/beta": 0.005540065001696348,
      "fcm_dpo/delta": 0.06673873215913773,
      "fcm_dpo/margin": 60.95854187011719,
      "fcm_dpo/q_t": 0.42290928959846497,
      "grad_norm": 76.16566467285156,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": -0.8716725707054138,
      "logits/rejected": -0.8630011081695557,
      "logps/chosen": -490.7124938964844,
      "logps/ref_chosen": -307.5315246582031,
      "logps/ref_rejected": -264.3540954589844,
      "logps/rejected": -508.49359130859375,
      "loss": 4.6891,
      "margin_dpo/margin_mean": 60.95854187011719,
      "margin_dpo/margin_std": 109.7963638305664,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -169.93527221679688,
      "KL/mean": -213.7938995361328,
      "KL/rejected_KL_mean": -257.6525573730469,
      "KL/std": 109.45353698730469,
      "epoch": 0.9235602094240838,
      "fcm_dpo/beta": 0.0055548963136971,
      "fcm_dpo/delta": -0.03971199691295624,
      "fcm_dpo/margin": 87.71729278564453,
      "fcm_dpo/q_t": 0.3912283778190613,
      "grad_norm": 86.50141906738281,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -0.8363861441612244,
      "logits/rejected": -0.8239439725875854,
      "logps/chosen": -479.917236328125,
      "logps/ref_chosen": -309.9819641113281,
      "logps/ref_rejected": -297.4968566894531,
      "logps/rejected": -555.1494140625,
      "loss": 4.2295,
      "margin_dpo/margin_mean": 87.71730041503906,
      "margin_dpo/margin_std": 113.45219421386719,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -185.58535766601562,
      "KL/mean": -219.0474090576172,
      "KL/rejected_KL_mean": -252.50949096679688,
      "KL/std": 108.39557647705078,
      "epoch": 0.9256544502617801,
      "fcm_dpo/beta": 0.005506675224751234,
      "fcm_dpo/delta": 0.05083406716585159,
      "fcm_dpo/margin": 66.92411804199219,
      "fcm_dpo/q_t": 0.4190221428871155,
      "grad_norm": 91.00436401367188,
      "learning_rate": 8.637407257200496e-09,
      "logits/chosen": -0.9067466855049133,
      "logits/rejected": -0.862942636013031,
      "logps/chosen": -464.56451416015625,
      "logps/ref_chosen": -278.9791564941406,
      "logps/ref_rejected": -242.87310791015625,
      "logps/rejected": -495.3825988769531,
      "loss": 4.7011,
      "margin_dpo/margin_mean": 66.92411804199219,
      "margin_dpo/margin_std": 129.05897521972656,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -170.37591552734375,
      "KL/mean": -209.1938934326172,
      "KL/rejected_KL_mean": -248.01187133789062,
      "KL/std": 99.9699935913086,
      "epoch": 0.9277486910994764,
      "fcm_dpo/beta": 0.0057164886966347694,
      "fcm_dpo/delta": 0.003951897844672203,
      "fcm_dpo/margin": 77.63595581054688,
      "fcm_dpo/q_t": 0.4004153609275818,
      "grad_norm": 87.5594482421875,
      "learning_rate": 8.166809758815895e-09,
      "logits/chosen": -0.8068374991416931,
      "logits/rejected": -0.8282541036605835,
      "logps/chosen": -443.9349670410156,
      "logps/ref_chosen": -273.5590515136719,
      "logps/ref_rejected": -264.0199279785156,
      "logps/rejected": -512.03173828125,
      "loss": 4.4281,
      "margin_dpo/margin_mean": 77.63595581054688,
      "margin_dpo/margin_std": 117.02406311035156,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -169.11415100097656,
      "KL/mean": -211.88565063476562,
      "KL/rejected_KL_mean": -254.65716552734375,
      "KL/std": 117.47710418701172,
      "epoch": 0.9298429319371728,
      "fcm_dpo/beta": 0.005521510727703571,
      "fcm_dpo/delta": -0.02465350367128849,
      "fcm_dpo/margin": 85.54300689697266,
      "fcm_dpo/q_t": 0.3977935016155243,
      "grad_norm": 85.76791381835938,
      "learning_rate": 7.709181040498253e-09,
      "logits/chosen": -0.8242456316947937,
      "logits/rejected": -0.8122622966766357,
      "logps/chosen": -467.25830078125,
      "logps/ref_chosen": -298.1441955566406,
      "logps/ref_rejected": -268.0572814941406,
      "logps/rejected": -522.7144775390625,
      "loss": 4.3898,
      "margin_dpo/margin_mean": 85.54300689697266,
      "margin_dpo/margin_std": 134.19903564453125,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -170.99850463867188,
      "KL/mean": -207.6737060546875,
      "KL/rejected_KL_mean": -244.34890747070312,
      "KL/std": 112.4179458618164,
      "epoch": 0.9319371727748691,
      "fcm_dpo/beta": 0.005374241154640913,
      "fcm_dpo/delta": -0.09346143901348114,
      "fcm_dpo/margin": 73.35040283203125,
      "fcm_dpo/q_t": 0.41321587562561035,
      "grad_norm": 79.82279205322266,
      "learning_rate": 7.2645456434869965e-09,
      "logits/chosen": -0.8946458101272583,
      "logits/rejected": -0.9047534465789795,
      "logps/chosen": -425.5391845703125,
      "logps/ref_chosen": -254.54067993164062,
      "logps/ref_rejected": -264.2445983886719,
      "logps/rejected": -508.5934753417969,
      "loss": 4.6118,
      "margin_dpo/margin_mean": 73.35039520263672,
      "margin_dpo/margin_std": 122.0877685546875,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -167.30377197265625,
      "KL/mean": -207.5486602783203,
      "KL/rejected_KL_mean": -247.7935028076172,
      "KL/std": 104.70956420898438,
      "epoch": 0.9340314136125655,
      "fcm_dpo/beta": 0.00517143914476037,
      "fcm_dpo/delta": 0.034428343176841736,
      "fcm_dpo/margin": 80.48971557617188,
      "fcm_dpo/q_t": 0.4070422053337097,
      "grad_norm": 80.14286804199219,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -0.8223922848701477,
      "logits/rejected": -0.8251609802246094,
      "logps/chosen": -474.0262451171875,
      "logps/ref_chosen": -306.72247314453125,
      "logps/ref_rejected": -266.3735656738281,
      "logps/rejected": -514.1670532226562,
      "loss": 4.4592,
      "margin_dpo/margin_mean": 80.48971557617188,
      "margin_dpo/margin_std": 123.26667022705078,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -161.01161193847656,
      "KL/mean": -208.1389617919922,
      "KL/rejected_KL_mean": -255.26634216308594,
      "KL/std": 108.80945587158203,
      "epoch": 0.9361256544502617,
      "fcm_dpo/beta": 0.005069321021437645,
      "fcm_dpo/delta": -0.07068045437335968,
      "fcm_dpo/margin": 94.25475311279297,
      "fcm_dpo/q_t": 0.39169907569885254,
      "grad_norm": 66.20989990234375,
      "learning_rate": 6.414349493100129e-09,
      "logits/chosen": -0.8259595632553101,
      "logits/rejected": -0.8232617378234863,
      "logps/chosen": -421.5289001464844,
      "logps/ref_chosen": -260.51727294921875,
      "logps/ref_rejected": -236.47061157226562,
      "logps/rejected": -491.7369689941406,
      "loss": 4.2145,
      "margin_dpo/margin_mean": 94.25474548339844,
      "margin_dpo/margin_std": 117.8897705078125,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -175.0517120361328,
      "KL/mean": -217.67620849609375,
      "KL/rejected_KL_mean": -260.30072021484375,
      "KL/std": 109.43276977539062,
      "epoch": 0.9382198952879581,
      "fcm_dpo/beta": 0.0050066811963915825,
      "fcm_dpo/delta": 0.023107346147298813,
      "fcm_dpo/margin": 85.24903106689453,
      "fcm_dpo/q_t": 0.4045635461807251,
      "grad_norm": 100.51019287109375,
      "learning_rate": 6.0088343331638756e-09,
      "logits/chosen": -0.8242851495742798,
      "logits/rejected": -0.818529486656189,
      "logps/chosen": -443.8387451171875,
      "logps/ref_chosen": -268.78704833984375,
      "logps/ref_rejected": -262.1703796386719,
      "logps/rejected": -522.4711303710938,
      "loss": 4.362,
      "margin_dpo/margin_mean": 85.24903106689453,
      "margin_dpo/margin_std": 115.99938201904297,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -161.47848510742188,
      "KL/mean": -203.19589233398438,
      "KL/rejected_KL_mean": -244.91329956054688,
      "KL/std": 104.93698120117188,
      "epoch": 0.9403141361256544,
      "fcm_dpo/beta": 0.00520076509565115,
      "fcm_dpo/delta": 0.015470391139388084,
      "fcm_dpo/margin": 83.43479919433594,
      "fcm_dpo/q_t": 0.39993420243263245,
      "grad_norm": 136.72283935546875,
      "learning_rate": 5.616403678967624e-09,
      "logits/chosen": -0.9107736349105835,
      "logits/rejected": -0.8956658244132996,
      "logps/chosen": -492.429931640625,
      "logps/ref_chosen": -330.9514465332031,
      "logps/ref_rejected": -239.76974487304688,
      "logps/rejected": -484.68304443359375,
      "loss": 4.4133,
      "margin_dpo/margin_mean": 83.43480682373047,
      "margin_dpo/margin_std": 122.92559051513672,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -181.75177001953125,
      "KL/mean": -218.86537170410156,
      "KL/rejected_KL_mean": -255.97898864746094,
      "KL/std": 112.59514617919922,
      "epoch": 0.9424083769633508,
      "fcm_dpo/beta": 0.005191947799175978,
      "fcm_dpo/delta": 0.02817104011774063,
      "fcm_dpo/margin": 74.22718811035156,
      "fcm_dpo/q_t": 0.41280612349510193,
      "grad_norm": 93.79536437988281,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": -0.8111035227775574,
      "logits/rejected": -0.8107234835624695,
      "logps/chosen": -466.0171813964844,
      "logps/ref_chosen": -284.26544189453125,
      "logps/ref_rejected": -250.5401611328125,
      "logps/rejected": -506.5191650390625,
      "loss": 4.4909,
      "margin_dpo/margin_mean": 74.2271957397461,
      "margin_dpo/margin_std": 110.18287658691406,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -186.6180877685547,
      "KL/mean": -220.67398071289062,
      "KL/rejected_KL_mean": -254.72991943359375,
      "KL/std": 110.34291076660156,
      "epoch": 0.9445026178010472,
      "fcm_dpo/beta": 0.0051805381663143635,
      "fcm_dpo/delta": 0.011119775474071503,
      "fcm_dpo/margin": 68.11183166503906,
      "fcm_dpo/q_t": 0.4210098385810852,
      "grad_norm": 106.1446533203125,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -0.8346443772315979,
      "logits/rejected": -0.8073043823242188,
      "logps/chosen": -488.93902587890625,
      "logps/ref_chosen": -302.3209228515625,
      "logps/ref_rejected": -254.09747314453125,
      "logps/rejected": -508.827392578125,
      "loss": 4.666,
      "margin_dpo/margin_mean": 68.11183166503906,
      "margin_dpo/margin_std": 119.73832702636719,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -167.25689697265625,
      "KL/mean": -212.2622833251953,
      "KL/rejected_KL_mean": -257.2677001953125,
      "KL/std": 122.58613586425781,
      "epoch": 0.9465968586387434,
      "fcm_dpo/beta": 0.005065259523689747,
      "fcm_dpo/delta": -0.049960654228925705,
      "fcm_dpo/margin": 90.01078796386719,
      "fcm_dpo/q_t": 0.4009445905685425,
      "grad_norm": 89.22917175292969,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": -0.8890603184700012,
      "logits/rejected": -0.8659049868583679,
      "logps/chosen": -466.6490783691406,
      "logps/ref_chosen": -299.39215087890625,
      "logps/ref_rejected": -284.3475036621094,
      "logps/rejected": -541.615234375,
      "loss": 4.3766,
      "margin_dpo/margin_mean": 90.01078033447266,
      "margin_dpo/margin_std": 134.02072143554688,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -183.13320922851562,
      "KL/mean": -223.8314971923828,
      "KL/rejected_KL_mean": -264.5297546386719,
      "KL/std": 111.88533782958984,
      "epoch": 0.9486910994764398,
      "fcm_dpo/beta": 0.005183791276067495,
      "fcm_dpo/delta": 0.029111308977007866,
      "fcm_dpo/margin": 81.39653015136719,
      "fcm_dpo/q_t": 0.40667787194252014,
      "grad_norm": 86.04676055908203,
      "learning_rate": 4.1779364682113794e-09,
      "logits/chosen": -0.8265345096588135,
      "logits/rejected": -0.8217592835426331,
      "logps/chosen": -507.784912109375,
      "logps/ref_chosen": -324.6517028808594,
      "logps/ref_rejected": -304.1527099609375,
      "logps/rejected": -568.6824951171875,
      "loss": 4.444,
      "margin_dpo/margin_mean": 81.39653015136719,
      "margin_dpo/margin_std": 128.67710876464844,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -174.5858154296875,
      "KL/mean": -221.30422973632812,
      "KL/rejected_KL_mean": -268.02264404296875,
      "KL/std": 124.37655639648438,
      "epoch": 0.9507853403141361,
      "fcm_dpo/beta": 0.00515640527009964,
      "fcm_dpo/delta": -0.03455362468957901,
      "fcm_dpo/margin": 93.43683624267578,
      "fcm_dpo/q_t": 0.3930882215499878,
      "grad_norm": 70.44085693359375,
      "learning_rate": 3.851229943335393e-09,
      "logits/chosen": -0.8620548248291016,
      "logits/rejected": -0.8730037808418274,
      "logps/chosen": -474.1975402832031,
      "logps/ref_chosen": -299.6117248535156,
      "logps/ref_rejected": -303.74224853515625,
      "logps/rejected": -571.764892578125,
      "loss": 4.2981,
      "margin_dpo/margin_mean": 93.43683624267578,
      "margin_dpo/margin_std": 133.02867126464844,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -167.13040161132812,
      "KL/mean": -200.8979949951172,
      "KL/rejected_KL_mean": -234.6656036376953,
      "KL/std": 111.39952850341797,
      "epoch": 0.9528795811518325,
      "fcm_dpo/beta": 0.005253899842500687,
      "fcm_dpo/delta": 0.024034246802330017,
      "fcm_dpo/margin": 67.53520202636719,
      "fcm_dpo/q_t": 0.42268288135528564,
      "grad_norm": 79.2885971069336,
      "learning_rate": 3.5377236299748147e-09,
      "logits/chosen": -0.8365157246589661,
      "logits/rejected": -0.8462912440299988,
      "logps/chosen": -440.7420654296875,
      "logps/ref_chosen": -273.6116943359375,
      "logps/ref_rejected": -274.4293518066406,
      "logps/rejected": -509.094970703125,
      "loss": 4.6768,
      "margin_dpo/margin_mean": 67.53520202636719,
      "margin_dpo/margin_std": 126.26296997070312,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -193.4385986328125,
      "KL/mean": -236.28993225097656,
      "KL/rejected_KL_mean": -279.1412048339844,
      "KL/std": 129.02645874023438,
      "epoch": 0.9549738219895288,
      "fcm_dpo/beta": 0.005159331485629082,
      "fcm_dpo/delta": -0.021475916728377342,
      "fcm_dpo/margin": 85.70262145996094,
      "fcm_dpo/q_t": 0.40772879123687744,
      "grad_norm": 90.39838409423828,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -0.7484225034713745,
      "logits/rejected": -0.7598169445991516,
      "logps/chosen": -515.6105346679688,
      "logps/ref_chosen": -322.17193603515625,
      "logps/ref_rejected": -294.54461669921875,
      "logps/rejected": -573.6858520507812,
      "loss": 4.5266,
      "margin_dpo/margin_mean": 85.70262145996094,
      "margin_dpo/margin_std": 150.2220916748047,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -184.51071166992188,
      "KL/mean": -229.59371948242188,
      "KL/rejected_KL_mean": -274.6767578125,
      "KL/std": 120.49661254882812,
      "epoch": 0.9570680628272251,
      "fcm_dpo/beta": 0.005027763079851866,
      "fcm_dpo/delta": -0.004381144419312477,
      "fcm_dpo/margin": 90.16605377197266,
      "fcm_dpo/q_t": 0.3963942527770996,
      "grad_norm": 90.38420867919922,
      "learning_rate": 2.9503781785795713e-09,
      "logits/chosen": -0.8214358687400818,
      "logits/rejected": -0.8249754309654236,
      "logps/chosen": -492.30694580078125,
      "logps/ref_chosen": -307.7962341308594,
      "logps/ref_rejected": -274.5501403808594,
      "logps/rejected": -549.2269287109375,
      "loss": 4.4102,
      "margin_dpo/margin_mean": 90.16605377197266,
      "margin_dpo/margin_std": 138.66323852539062,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -177.21435546875,
      "KL/mean": -216.5757293701172,
      "KL/rejected_KL_mean": -255.93711853027344,
      "KL/std": 121.320556640625,
      "epoch": 0.9591623036649215,
      "fcm_dpo/beta": 0.005124977324157953,
      "fcm_dpo/delta": 0.015909496694803238,
      "fcm_dpo/margin": 78.7227783203125,
      "fcm_dpo/q_t": 0.4107885956764221,
      "grad_norm": 81.07950592041016,
      "learning_rate": 2.6765705380989432e-09,
      "logits/chosen": -0.8464676141738892,
      "logits/rejected": -0.8287412524223328,
      "logps/chosen": -474.2459716796875,
      "logps/ref_chosen": -297.0316467285156,
      "logps/ref_rejected": -276.1112365722656,
      "logps/rejected": -532.04833984375,
      "loss": 4.5135,
      "margin_dpo/margin_mean": 78.7227783203125,
      "margin_dpo/margin_std": 126.8575210571289,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -168.28826904296875,
      "KL/mean": -203.2023162841797,
      "KL/rejected_KL_mean": -238.11634826660156,
      "KL/std": 102.70429992675781,
      "epoch": 0.9612565445026178,
      "fcm_dpo/beta": 0.005178958643227816,
      "fcm_dpo/delta": 0.006201361771672964,
      "fcm_dpo/margin": 69.828125,
      "fcm_dpo/q_t": 0.4197620749473572,
      "grad_norm": 87.29827117919922,
      "learning_rate": 2.416026102552732e-09,
      "logits/chosen": -0.9117629528045654,
      "logits/rejected": -0.9042491912841797,
      "logps/chosen": -461.8135986328125,
      "logps/ref_chosen": -293.5252990722656,
      "logps/ref_rejected": -289.30126953125,
      "logps/rejected": -527.4176025390625,
      "loss": 4.6246,
      "margin_dpo/margin_mean": 69.82811737060547,
      "margin_dpo/margin_std": 120.81681823730469,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -172.2991180419922,
      "KL/mean": -207.47708129882812,
      "KL/rejected_KL_mean": -242.65504455566406,
      "KL/std": 100.72721862792969,
      "epoch": 0.9633507853403142,
      "fcm_dpo/beta": 0.00521495658904314,
      "fcm_dpo/delta": 0.017437200993299484,
      "fcm_dpo/margin": 70.35596466064453,
      "fcm_dpo/q_t": 0.4167800545692444,
      "grad_norm": 94.0357894897461,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": -0.8717383146286011,
      "logits/rejected": -0.8724873065948486,
      "logps/chosen": -491.0794677734375,
      "logps/ref_chosen": -318.7803649902344,
      "logps/ref_rejected": -258.7906799316406,
      "logps/rejected": -501.44573974609375,
      "loss": 4.5815,
      "margin_dpo/margin_mean": 70.35596466064453,
      "margin_dpo/margin_std": 117.5007553100586,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -172.34783935546875,
      "KL/mean": -210.01210021972656,
      "KL/rejected_KL_mean": -247.6763458251953,
      "KL/std": 110.3778076171875,
      "epoch": 0.9654450261780104,
      "fcm_dpo/beta": 0.005268401931971312,
      "fcm_dpo/delta": -0.010377016849815845,
      "fcm_dpo/margin": 75.32850646972656,
      "fcm_dpo/q_t": 0.4127994775772095,
      "grad_norm": 97.776611328125,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -0.8309637308120728,
      "logits/rejected": -0.8563615083694458,
      "logps/chosen": -416.2578125,
      "logps/ref_chosen": -243.9099884033203,
      "logps/ref_rejected": -232.6382293701172,
      "logps/rejected": -480.3145751953125,
      "loss": 4.5993,
      "margin_dpo/margin_mean": 75.32849884033203,
      "margin_dpo/margin_std": 132.525146484375,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -176.3037109375,
      "KL/mean": -221.43557739257812,
      "KL/rejected_KL_mean": -266.56744384765625,
      "KL/std": 114.42298889160156,
      "epoch": 0.9675392670157068,
      "fcm_dpo/beta": 0.00506819412112236,
      "fcm_dpo/delta": -0.05484515428543091,
      "fcm_dpo/margin": 90.26371002197266,
      "fcm_dpo/q_t": 0.3993760347366333,
      "grad_norm": 91.68852233886719,
      "learning_rate": 1.7141081868094209e-09,
      "logits/chosen": -0.8642257452011108,
      "logits/rejected": -0.8165584802627563,
      "logps/chosen": -520.3946533203125,
      "logps/ref_chosen": -344.09100341796875,
      "logps/ref_rejected": -252.45037841796875,
      "logps/rejected": -519.017822265625,
      "loss": 4.3924,
      "margin_dpo/margin_mean": 90.26371002197266,
      "margin_dpo/margin_std": 136.05551147460938,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -173.57421875,
      "KL/mean": -209.10317993164062,
      "KL/rejected_KL_mean": -244.63217163085938,
      "KL/std": 110.4543228149414,
      "epoch": 0.9696335078534032,
      "fcm_dpo/beta": 0.005103372968733311,
      "fcm_dpo/delta": 0.0166908148676157,
      "fcm_dpo/margin": 71.0579833984375,
      "fcm_dpo/q_t": 0.41910725831985474,
      "grad_norm": 69.62117767333984,
      "learning_rate": 1.5067491694100153e-09,
      "logits/chosen": -0.8673644065856934,
      "logits/rejected": -0.8280107378959656,
      "logps/chosen": -470.71661376953125,
      "logps/ref_chosen": -297.1424560546875,
      "logps/ref_rejected": -234.0208282470703,
      "logps/rejected": -478.65301513671875,
      "loss": 4.6439,
      "margin_dpo/margin_mean": 71.0579833984375,
      "margin_dpo/margin_std": 127.77708435058594,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -188.04385375976562,
      "KL/mean": -223.29537963867188,
      "KL/rejected_KL_mean": -258.546875,
      "KL/std": 112.52729034423828,
      "epoch": 0.9717277486910995,
      "fcm_dpo/beta": 0.005199297796934843,
      "fcm_dpo/delta": 0.08676628768444061,
      "fcm_dpo/margin": 70.50304412841797,
      "fcm_dpo/q_t": 0.4186946451663971,
      "grad_norm": 132.28236389160156,
      "learning_rate": 1.3127160909147672e-09,
      "logits/chosen": -0.8498543500900269,
      "logits/rejected": -0.8720042109489441,
      "logps/chosen": -453.7546081542969,
      "logps/ref_chosen": -265.71075439453125,
      "logps/ref_rejected": -256.4108581542969,
      "logps/rejected": -514.957763671875,
      "loss": 4.6357,
      "margin_dpo/margin_mean": 70.50304412841797,
      "margin_dpo/margin_std": 125.67542266845703,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -164.92324829101562,
      "KL/mean": -211.6644287109375,
      "KL/rejected_KL_mean": -258.4056091308594,
      "KL/std": 108.36969757080078,
      "epoch": 0.9738219895287958,
      "fcm_dpo/beta": 0.005149043630808592,
      "fcm_dpo/delta": -0.0949038565158844,
      "fcm_dpo/margin": 93.48236083984375,
      "fcm_dpo/q_t": 0.39296746253967285,
      "grad_norm": 73.30370330810547,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": -0.8986018300056458,
      "logits/rejected": -0.8646162152290344,
      "logps/chosen": -458.0759582519531,
      "logps/ref_chosen": -293.1527404785156,
      "logps/ref_rejected": -293.70947265625,
      "logps/rejected": -552.1151123046875,
      "loss": 4.2983,
      "margin_dpo/margin_mean": 93.48236083984375,
      "margin_dpo/margin_std": 126.78382110595703,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -158.126220703125,
      "KL/mean": -207.4628143310547,
      "KL/rejected_KL_mean": -256.7994384765625,
      "KL/std": 109.99119567871094,
      "epoch": 0.9759162303664921,
      "fcm_dpo/beta": 0.004878416191786528,
      "fcm_dpo/delta": -0.03413254767656326,
      "fcm_dpo/margin": 98.67318725585938,
      "fcm_dpo/q_t": 0.3917115330696106,
      "grad_norm": 68.8930435180664,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -0.8271768689155579,
      "logits/rejected": -0.775895357131958,
      "logps/chosen": -419.6037902832031,
      "logps/ref_chosen": -261.4775695800781,
      "logps/ref_rejected": -248.36282348632812,
      "logps/rejected": -505.16229248046875,
      "loss": 4.1847,
      "margin_dpo/margin_mean": 98.67318725585938,
      "margin_dpo/margin_std": 120.41122436523438,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -177.9638671875,
      "KL/mean": -212.95016479492188,
      "KL/rejected_KL_mean": -247.9364471435547,
      "KL/std": 109.7723617553711,
      "epoch": 0.9780104712041885,
      "fcm_dpo/beta": 0.004856320563703775,
      "fcm_dpo/delta": 0.015015661716461182,
      "fcm_dpo/margin": 69.97257995605469,
      "fcm_dpo/q_t": 0.42448192834854126,
      "grad_norm": 76.92168426513672,
      "learning_rate": 8.106729664475176e-10,
      "logits/chosen": -0.8034157752990723,
      "logits/rejected": -0.7989070415496826,
      "logps/chosen": -444.318115234375,
      "logps/ref_chosen": -266.354248046875,
      "logps/ref_rejected": -277.76324462890625,
      "logps/rejected": -525.69970703125,
      "loss": 4.7362,
      "margin_dpo/margin_mean": 69.97257995605469,
      "margin_dpo/margin_std": 135.70855712890625,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -174.75184631347656,
      "KL/mean": -209.31219482421875,
      "KL/rejected_KL_mean": -243.8725128173828,
      "KL/std": 108.66677856445312,
      "epoch": 0.9801047120418848,
      "fcm_dpo/beta": 0.004959843121469021,
      "fcm_dpo/delta": 0.0029601585119962692,
      "fcm_dpo/margin": 69.12068939208984,
      "fcm_dpo/q_t": 0.4214484691619873,
      "grad_norm": 90.60674285888672,
      "learning_rate": 6.700405431837585e-10,
      "logits/chosen": -0.8957569599151611,
      "logits/rejected": -0.8660439848899841,
      "logps/chosen": -492.7149963378906,
      "logps/ref_chosen": -317.9631652832031,
      "logps/ref_rejected": -261.8744201660156,
      "logps/rejected": -505.7469482421875,
      "loss": 4.6993,
      "margin_dpo/margin_mean": 69.12068939208984,
      "margin_dpo/margin_std": 127.72600555419922,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -162.23385620117188,
      "KL/mean": -208.19677734375,
      "KL/rejected_KL_mean": -254.15966796875,
      "KL/std": 111.96234130859375,
      "epoch": 0.9821989528795811,
      "fcm_dpo/beta": 0.004891404416412115,
      "fcm_dpo/delta": -0.0006460901349782944,
      "fcm_dpo/margin": 91.92579650878906,
      "fcm_dpo/q_t": 0.39855480194091797,
      "grad_norm": 66.42138671875,
      "learning_rate": 5.427789289685347e-10,
      "logits/chosen": -0.821508526802063,
      "logits/rejected": -0.8067971467971802,
      "logps/chosen": -487.12066650390625,
      "logps/ref_chosen": -324.8868103027344,
      "logps/ref_rejected": -264.0421447753906,
      "logps/rejected": -518.2018432617188,
      "loss": 4.3393,
      "margin_dpo/margin_mean": 91.92579650878906,
      "margin_dpo/margin_std": 128.5748748779297,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -164.5446319580078,
      "KL/mean": -206.8803253173828,
      "KL/rejected_KL_mean": -249.2160186767578,
      "KL/std": 109.4319839477539,
      "epoch": 0.9842931937172775,
      "fcm_dpo/beta": 0.004932143725454807,
      "fcm_dpo/delta": -0.031199753284454346,
      "fcm_dpo/margin": 84.67138671875,
      "fcm_dpo/q_t": 0.40564781427383423,
      "grad_norm": 79.43443298339844,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": -0.8226178884506226,
      "logits/rejected": -0.8206679821014404,
      "logps/chosen": -479.2489013671875,
      "logps/ref_chosen": -314.7042541503906,
      "logps/ref_rejected": -259.2276611328125,
      "logps/rejected": -508.44366455078125,
      "loss": 4.4122,
      "margin_dpo/margin_mean": 84.67138671875,
      "margin_dpo/margin_std": 121.67056274414062,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -177.97642517089844,
      "KL/mean": -218.14285278320312,
      "KL/rejected_KL_mean": -258.309326171875,
      "KL/std": 123.82325744628906,
      "epoch": 0.9863874345549738,
      "fcm_dpo/beta": 0.004885872360318899,
      "fcm_dpo/delta": -0.013526059687137604,
      "fcm_dpo/margin": 80.33290100097656,
      "fcm_dpo/q_t": 0.41574254631996155,
      "grad_norm": 88.63224792480469,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -0.8717851638793945,
      "logits/rejected": -0.8601978421211243,
      "logps/chosen": -470.55126953125,
      "logps/ref_chosen": -292.5748291015625,
      "logps/ref_rejected": -298.7525329589844,
      "logps/rejected": -557.0618286132812,
      "loss": 4.5346,
      "margin_dpo/margin_mean": 80.33290100097656,
      "margin_dpo/margin_std": 132.79739379882812,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -159.89474487304688,
      "KL/mean": -205.90017700195312,
      "KL/rejected_KL_mean": -251.90557861328125,
      "KL/std": 107.56350708007812,
      "epoch": 0.9884816753926702,
      "fcm_dpo/beta": 0.004792001098394394,
      "fcm_dpo/delta": 0.00895017385482788,
      "fcm_dpo/margin": 92.01083374023438,
      "fcm_dpo/q_t": 0.4019765853881836,
      "grad_norm": 84.93602752685547,
      "learning_rate": 2.412835998185092e-10,
      "logits/chosen": -0.8595799803733826,
      "logits/rejected": -0.8735888004302979,
      "logps/chosen": -403.2685546875,
      "logps/ref_chosen": -243.37380981445312,
      "logps/ref_rejected": -251.12109375,
      "logps/rejected": -503.02667236328125,
      "loss": 4.3226,
      "margin_dpo/margin_mean": 92.01083374023438,
      "margin_dpo/margin_std": 124.53809356689453,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -168.1905059814453,
      "KL/mean": -214.41455078125,
      "KL/rejected_KL_mean": -260.63861083984375,
      "KL/std": 103.71307373046875,
      "epoch": 0.9905759162303664,
      "fcm_dpo/beta": 0.004873677156865597,
      "fcm_dpo/delta": -0.0007807677611708641,
      "fcm_dpo/margin": 92.44808959960938,
      "fcm_dpo/q_t": 0.39710500836372375,
      "grad_norm": 68.06848907470703,
      "learning_rate": 1.6756629272085544e-10,
      "logits/chosen": -0.8255312442779541,
      "logits/rejected": -0.8275748491287231,
      "logps/chosen": -454.5191650390625,
      "logps/ref_chosen": -286.3286437988281,
      "logps/ref_rejected": -258.6535339355469,
      "logps/rejected": -519.2921752929688,
      "loss": 4.2701,
      "margin_dpo/margin_mean": 92.44808959960938,
      "margin_dpo/margin_std": 117.46378326416016,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -173.24331665039062,
      "KL/mean": -210.619873046875,
      "KL/rejected_KL_mean": -247.99642944335938,
      "KL/std": 109.95854949951172,
      "epoch": 0.9926701570680628,
      "fcm_dpo/beta": 0.004747421480715275,
      "fcm_dpo/delta": 0.008153961971402168,
      "fcm_dpo/margin": 74.75313568115234,
      "fcm_dpo/q_t": 0.4183030128479004,
      "grad_norm": 75.06713104248047,
      "learning_rate": 1.072467408408384e-10,
      "logits/chosen": -0.8549594879150391,
      "logits/rejected": -0.8526111841201782,
      "logps/chosen": -461.33294677734375,
      "logps/ref_chosen": -288.08966064453125,
      "logps/ref_rejected": -266.69696044921875,
      "logps/rejected": -514.6934814453125,
      "loss": 4.554,
      "margin_dpo/margin_mean": 74.75313568115234,
      "margin_dpo/margin_std": 111.50443267822266,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -161.9423065185547,
      "KL/mean": -197.55239868164062,
      "KL/rejected_KL_mean": -233.1625213623047,
      "KL/std": 102.64295196533203,
      "epoch": 0.9947643979057592,
      "fcm_dpo/beta": 0.0048391371965408325,
      "fcm_dpo/delta": 0.021528335288167,
      "fcm_dpo/margin": 71.22021484375,
      "fcm_dpo/q_t": 0.42340517044067383,
      "grad_norm": 87.310546875,
      "learning_rate": 6.032817893297793e-11,
      "logits/chosen": -0.8284741044044495,
      "logits/rejected": -0.8437927961349487,
      "logps/chosen": -417.94537353515625,
      "logps/ref_chosen": -256.0030517578125,
      "logps/ref_rejected": -244.50660705566406,
      "logps/rejected": -477.66912841796875,
      "loss": 4.6324,
      "margin_dpo/margin_mean": 71.22021484375,
      "margin_dpo/margin_std": 121.39439392089844,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -160.92379760742188,
      "KL/mean": -201.7862091064453,
      "KL/rejected_KL_mean": -242.6486053466797,
      "KL/std": 105.0941162109375,
      "epoch": 0.9968586387434555,
      "fcm_dpo/beta": 0.0049579935148358345,
      "fcm_dpo/delta": 0.00904696062207222,
      "fcm_dpo/margin": 81.72483825683594,
      "fcm_dpo/q_t": 0.4120728671550751,
      "grad_norm": 95.99996185302734,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -0.9014330506324768,
      "logits/rejected": -0.8572571873664856,
      "logps/chosen": -482.39129638671875,
      "logps/ref_chosen": -321.467529296875,
      "logps/ref_rejected": -295.0592956542969,
      "logps/rejected": -537.7079467773438,
      "loss": 4.5306,
      "margin_dpo/margin_mean": 81.72484588623047,
      "margin_dpo/margin_std": 130.52923583984375,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -179.8543701171875,
      "KL/mean": -219.45687866210938,
      "KL/rejected_KL_mean": -259.05938720703125,
      "KL/std": 119.88137817382812,
      "epoch": 0.9989528795811519,
      "fcm_dpo/beta": 0.005032530054450035,
      "fcm_dpo/delta": -0.02052845060825348,
      "fcm_dpo/margin": 79.2050552368164,
      "fcm_dpo/q_t": 0.4129430055618286,
      "grad_norm": 88.6489028930664,
      "learning_rate": 6.7033706447061635e-12,
      "logits/chosen": -0.7853702902793884,
      "logits/rejected": -0.7928628325462341,
      "logps/chosen": -456.6483459472656,
      "logps/ref_chosen": -276.7939758300781,
      "logps/ref_rejected": -244.82919311523438,
      "logps/rejected": -503.8885803222656,
      "loss": 4.6334,
      "margin_dpo/margin_mean": 79.2050552368164,
      "margin_dpo/margin_std": 144.53997802734375,
      "step": 477
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 4.705121593155451,
      "train_runtime": 6559.8554,
      "train_samples_per_second": 9.32,
      "train_steps_per_second": 0.073
    }
  ],
  "logging_steps": 1,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}