{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 31.407852172851562,
      "kl": 21.7319393157959,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35708672.0,
      "logits/rejected": -37165390.76923077,
      "logps/chosen": -496.9298780487805,
      "logps/rejected": -362.92918669871796,
      "loss": 0.5378,
      "loss/base_kto": 3.8584771156311035,
      "loss/total_with_kl": 4.3021464347839355,
      "metrics/advantage_var": 266.4680480957031,
      "regularization/advantage_var": 266.4680480957031,
      "regularization/kl": 0.44366922974586487,
      "rewards/chosen": 0.3604372303660323,
      "rewards/margins": 0.13938797049555204,
      "rewards/rejected": 0.22104925987048027,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 13.991652488708496,
      "kl": 11.298606872558594,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36759266.597243495,
      "logits/rejected": -36918110.315789476,
      "logps/chosen": -483.4872702909648,
      "logps/rejected": -404.06678628389153,
      "loss": 0.5262,
      "loss/base_kto": 3.8981258869171143,
      "loss/total_with_kl": 4.209258079528809,
      "metrics/advantage_var": 186.8661346435547,
      "regularization/advantage_var": 186.8661346435547,
      "regularization/kl": 0.3111321032047272,
      "rewards/chosen": 0.1703898968776919,
      "rewards/margins": 0.10665024803134397,
      "rewards/rejected": 0.06373964884634793,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 14.879023551940918,
      "kl": 16.563892364501953,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35328210.131699845,
      "logits/rejected": -36644468.77192982,
      "logps/chosen": -492.49081163859114,
      "logps/rejected": -387.8072169059011,
      "loss": 0.5241,
      "loss/base_kto": 3.8087785243988037,
      "loss/total_with_kl": 4.192681789398193,
      "metrics/advantage_var": 230.5727081298828,
      "regularization/advantage_var": 230.5727081298828,
      "regularization/kl": 0.3839035630226135,
      "rewards/chosen": 0.33150058062846954,
      "rewards/margins": 0.168357088046918,
      "rewards/rejected": 0.16314349258155153,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 10.058321952819824,
      "kl": 4.800376892089844,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37254383.51186441,
      "logits/rejected": -37743794.08695652,
      "logps/chosen": -489.9477224576271,
      "logps/rejected": -394.09415760869564,
      "loss": 0.525,
      "loss/base_kto": 3.8233535289764404,
      "loss/total_with_kl": 4.199948787689209,
      "metrics/advantage_var": 226.18325805664062,
      "regularization/advantage_var": 226.18325805664062,
      "regularization/kl": 0.3765951097011566,
      "rewards/chosen": -0.06402210946810448,
      "rewards/margins": 0.15035100640315913,
      "rewards/rejected": -0.2143731158712636,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 14.036251068115234,
      "kl": 6.662667751312256,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35068959.80124223,
      "logits/rejected": -36745950.18867925,
      "logps/chosen": -482.4549204192547,
      "logps/rejected": -355.26459316037733,
      "loss": 0.5194,
      "loss/base_kto": 3.804851531982422,
      "loss/total_with_kl": 4.155176639556885,
      "metrics/advantage_var": 210.4054718017578,
      "regularization/advantage_var": 210.4054718017578,
      "regularization/kl": 0.3503250777721405,
      "rewards/chosen": 0.019241907581779526,
      "rewards/margins": 0.19215162622122528,
      "rewards/rejected": -0.17290971863944576,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 17.4003849029541,
      "kl": 17.45867919921875,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -36515702.594249204,
      "logits/rejected": -36474366.434250765,
      "logps/chosen": -468.5167731629393,
      "logps/rejected": -374.1554376911315,
      "loss": 0.525,
      "loss/base_kto": 3.8414978981018066,
      "loss/total_with_kl": 4.200013637542725,
      "metrics/advantage_var": 215.3249053955078,
      "regularization/advantage_var": 215.3249053955078,
      "regularization/kl": 0.3585159480571747,
      "rewards/chosen": 0.3180838880447534,
      "rewards/margins": 0.13438673291765096,
      "rewards/rejected": 0.18369715512710244,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 15.743810653686523,
      "kl": 10.514762878417969,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36569637.38897169,
      "logits/rejected": -37995814.25287356,
      "logps/chosen": -496.6527570789866,
      "logps/rejected": -369.03014675697864,
      "loss": 0.5272,
      "loss/base_kto": 3.764463186264038,
      "loss/total_with_kl": 4.2173614501953125,
      "metrics/advantage_var": 272.01068115234375,
      "regularization/advantage_var": 272.01068115234375,
      "regularization/kl": 0.4528977572917938,
      "rewards/chosen": 0.17616865496344075,
      "rewards/margins": 0.2380280113221615,
      "rewards/rejected": -0.06185935635872075,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 13.376973152160645,
      "kl": 11.327672004699707,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -36204002.461538464,
      "logits/rejected": -36326381.26829268,
      "logps/chosen": -476.1418269230769,
      "logps/rejected": -349.2894912347561,
      "loss": 0.5159,
      "loss/base_kto": 3.762291669845581,
      "loss/total_with_kl": 4.127110958099365,
      "metrics/advantage_var": 219.1105194091797,
      "regularization/advantage_var": 219.1105194091797,
      "regularization/kl": 0.3648190200328827,
      "rewards/chosen": 0.1949013685568785,
      "rewards/margins": 0.23164148260609219,
      "rewards/rejected": -0.036740114049213686,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 12.790648460388184,
      "kl": 3.0852344036102295,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35509029.822706066,
      "logits/rejected": -36370239.09576138,
      "logps/chosen": -454.6269440124417,
      "logps/rejected": -361.42177688383043,
      "loss": 0.5176,
      "loss/base_kto": 3.7811639308929443,
      "loss/total_with_kl": 4.141047954559326,
      "metrics/advantage_var": 216.1467742919922,
      "regularization/advantage_var": 216.1467742919922,
      "regularization/kl": 0.3598843812942505,
      "rewards/chosen": 0.03764512935656226,
      "rewards/margins": 0.22475955149448293,
      "rewards/rejected": -0.18711442213792068,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 13.26974868774414,
      "kl": 9.794063568115234,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -35194075.08082409,
      "logits/rejected": -38104267.537750386,
      "logps/chosen": -497.2285063391442,
      "logps/rejected": -373.55662557781204,
      "loss": 0.5216,
      "loss/base_kto": 3.756938934326172,
      "loss/total_with_kl": 4.173141002655029,
      "metrics/advantage_var": 249.9713134765625,
      "regularization/advantage_var": 249.9713134765625,
      "regularization/kl": 0.41620221734046936,
      "rewards/chosen": 0.1574949817687698,
      "rewards/margins": 0.2115917268174926,
      "rewards/rejected": -0.05409674504872279,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 14.107484817504883,
      "kl": 6.901716709136963,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -36784849.491311215,
      "logits/rejected": -36821523.78361669,
      "logps/chosen": -480.4296998420221,
      "logps/rejected": -375.8837905718702,
      "loss": 0.5297,
      "loss/base_kto": 3.738966464996338,
      "loss/total_with_kl": 4.237467288970947,
      "metrics/advantage_var": 299.39990234375,
      "regularization/advantage_var": 299.39990234375,
      "regularization/kl": 0.4985008239746094,
      "rewards/chosen": 0.1235669032092343,
      "rewards/margins": 0.2720938333918077,
      "rewards/rejected": -0.1485269301825734,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 16.072452545166016,
      "kl": 14.483355522155762,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36794460.080620155,
      "logits/rejected": -36787279.01732284,
      "logps/chosen": -472.3289728682171,
      "logps/rejected": -384.60944881889765,
      "loss": 0.5219,
      "loss/base_kto": 3.738342046737671,
      "loss/total_with_kl": 4.175151348114014,
      "metrics/advantage_var": 262.34765625,
      "regularization/advantage_var": 262.34765625,
      "regularization/kl": 0.43680888414382935,
      "rewards/chosen": 0.26270316663638565,
      "rewards/margins": 0.2332827044560753,
      "rewards/rejected": 0.029420462180310348,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 31.844099044799805,
      "kl": 18.43171501159668,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36242617.35311573,
      "logits/rejected": -38018287.94719472,
      "logps/chosen": -478.20938427299706,
      "logps/rejected": -370.2881342821782,
      "loss": 0.5163,
      "loss/base_kto": 3.638079881668091,
      "loss/total_with_kl": 4.130200386047363,
      "metrics/advantage_var": 295.5679626464844,
      "regularization/advantage_var": 295.5679626464844,
      "regularization/kl": 0.4921206533908844,
      "rewards/chosen": 0.44125633352941857,
      "rewards/margins": 0.3134749534016597,
      "rewards/rejected": 0.12778138012775886,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 23.6060848236084,
      "kl": 20.614234924316406,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -38277793.39969834,
      "logits/rejected": -39959278.158833064,
      "logps/chosen": -489.9046003016591,
      "logps/rejected": -354.2591166936791,
      "loss": 0.5232,
      "loss/base_kto": 3.713111639022827,
      "loss/total_with_kl": 4.185714244842529,
      "metrics/advantage_var": 283.8451843261719,
      "regularization/advantage_var": 283.8451843261719,
      "regularization/kl": 0.4726021885871887,
      "rewards/chosen": 0.3984208833398143,
      "rewards/margins": 0.2593852828290579,
      "rewards/rejected": 0.13903560051075642,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 13.035717964172363,
      "kl": 25.790876388549805,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -37029192.39178515,
      "logits/rejected": -38328598.55332303,
      "logps/chosen": -489.5135268562401,
      "logps/rejected": -380.8914702472952,
      "loss": 0.5228,
      "loss/base_kto": 3.7213408946990967,
      "loss/total_with_kl": 4.182716369628906,
      "metrics/advantage_var": 277.1021423339844,
      "regularization/advantage_var": 277.1021423339844,
      "regularization/kl": 0.46137505769729614,
      "rewards/chosen": 0.5010644511965591,
      "rewards/margins": 0.25748866081486954,
      "rewards/rejected": 0.24357579038168953,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 15.439168930053711,
      "kl": 19.15919303894043,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -38340044.6443769,
      "logits/rejected": -39156298.08360129,
      "logps/chosen": -493.11816109422494,
      "logps/rejected": -358.1865454180064,
      "loss": 0.5118,
      "loss/base_kto": 3.6858232021331787,
      "loss/total_with_kl": 4.094377040863037,
      "metrics/advantage_var": 245.37767028808594,
      "regularization/advantage_var": 245.37767028808594,
      "regularization/kl": 0.408553808927536,
      "rewards/chosen": 0.3816912340900456,
      "rewards/margins": 0.2520246695411483,
      "rewards/rejected": 0.12966656454889722,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 15.302995681762695,
      "kl": 19.191791534423828,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36681801.9466248,
      "logits/rejected": -38200022.19595645,
      "logps/chosen": -474.67268445839875,
      "logps/rejected": -362.0720985614308,
      "loss": 0.5166,
      "loss/base_kto": 3.708709478378296,
      "loss/total_with_kl": 4.133121490478516,
      "metrics/advantage_var": 254.9022674560547,
      "regularization/advantage_var": 254.9022674560547,
      "regularization/kl": 0.4244123101234436,
      "rewards/chosen": 0.2618728949099171,
      "rewards/margins": 0.26020920784020074,
      "rewards/rejected": 0.0016636870697163868,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 14.084619522094727,
      "kl": 21.147933959960938,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -37405050.67488444,
      "logits/rejected": -38261949.87004755,
      "logps/chosen": -484.3794780431433,
      "logps/rejected": -385.4020156497623,
      "loss": 0.5159,
      "loss/base_kto": 3.6717727184295654,
      "loss/total_with_kl": 4.127252101898193,
      "metrics/advantage_var": 273.56134033203125,
      "regularization/advantage_var": 273.56134033203125,
      "regularization/kl": 0.45547962188720703,
      "rewards/chosen": 0.3963017397558744,
      "rewards/margins": 0.26100734088735594,
      "rewards/rejected": 0.13529439886851846,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 12.959440231323242,
      "kl": 19.34718894958496,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35951478.12405446,
      "logits/rejected": -39251895.21163166,
      "logps/chosen": -501.0877458396369,
      "logps/rejected": -371.3808057350565,
      "loss": 0.5161,
      "loss/base_kto": 3.6851253509521484,
      "loss/total_with_kl": 4.128602504730225,
      "metrics/advantage_var": 266.3529052734375,
      "regularization/advantage_var": 266.3529052734375,
      "regularization/kl": 0.4434775412082672,
      "rewards/chosen": 0.3878343729316377,
      "rewards/margins": 0.267031476650779,
      "rewards/rejected": 0.12080289628085875,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 14.978385925292969,
      "kl": 18.645681381225586,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36797240.93312597,
      "logits/rejected": -38115401.9466248,
      "logps/chosen": -460.13826788491446,
      "logps/rejected": -361.6690296310832,
      "loss": 0.5169,
      "loss/base_kto": 3.7090961933135986,
      "loss/total_with_kl": 4.134805202484131,
      "metrics/advantage_var": 255.6807861328125,
      "regularization/advantage_var": 255.6807861328125,
      "regularization/kl": 0.425708532333374,
      "rewards/chosen": 0.3230946067703271,
      "rewards/margins": 0.2698136521249097,
      "rewards/rejected": 0.053280954645417364,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 19.60897445678711,
      "kl": 9.722540855407715,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -35991659.61247948,
      "logits/rejected": -37533399.94038748,
      "logps/chosen": -472.529197454844,
      "logps/rejected": -352.2409649776453,
      "loss": 0.5154,
      "loss/base_kto": 3.6974093914031982,
      "loss/total_with_kl": 4.123062610626221,
      "metrics/advantage_var": 255.6476593017578,
      "regularization/advantage_var": 255.6476593017578,
      "regularization/kl": 0.4256533682346344,
      "rewards/chosen": 0.21491746323057778,
      "rewards/margins": 0.28975284113552374,
      "rewards/rejected": -0.07483537790494597,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 16.646005630493164,
      "kl": 8.006230354309082,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -37325941.915151514,
      "logits/rejected": -37923820.18064516,
      "logps/chosen": -508.9840909090909,
      "logps/rejected": -372.57459677419354,
      "loss": 0.5222,
      "loss/base_kto": 3.70198130607605,
      "loss/total_with_kl": 4.177700042724609,
      "metrics/advantage_var": 285.7170104980469,
      "regularization/advantage_var": 285.7170104980469,
      "regularization/kl": 0.47571879625320435,
      "rewards/chosen": 0.11891341931892163,
      "rewards/margins": 0.2911074264541283,
      "rewards/rejected": -0.17219400713520666,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 14.671125411987305,
      "kl": 9.332090377807617,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -37215868.35510888,
      "logits/rejected": -37923514.65885798,
      "logps/chosen": -471.71869765494137,
      "logps/rejected": -402.11786237188875,
      "loss": 0.5121,
      "loss/base_kto": 3.6447296142578125,
      "loss/total_with_kl": 4.097075939178467,
      "metrics/advantage_var": 271.67950439453125,
      "regularization/advantage_var": 271.67950439453125,
      "regularization/kl": 0.4523463249206543,
      "rewards/chosen": 0.09727847995470516,
      "rewards/margins": 0.316666735217848,
      "rewards/rejected": -0.21938825526314285,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 20.42093849182129,
      "kl": 8.731324195861816,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36552382.12634823,
      "logits/rejected": -37327782.74484944,
      "logps/chosen": -496.2094568567026,
      "logps/rejected": -372.47043383518223,
      "loss": 0.5237,
      "loss/base_kto": 3.6933434009552,
      "loss/total_with_kl": 4.189201354980469,
      "metrics/advantage_var": 297.812744140625,
      "regularization/advantage_var": 297.812744140625,
      "regularization/kl": 0.49585819244384766,
      "rewards/chosen": 0.14480564957958156,
      "rewards/margins": 0.28422389942017434,
      "rewards/rejected": -0.1394182498405928,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 15.883015632629395,
      "kl": 6.376058101654053,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -34663966.51655629,
      "logits/rejected": -36241871.52662722,
      "logps/chosen": -490.091059602649,
      "logps/rejected": -375.5445173816568,
      "loss": 0.5119,
      "loss/base_kto": 3.6528472900390625,
      "loss/total_with_kl": 4.094952583312988,
      "metrics/advantage_var": 265.52880859375,
      "regularization/advantage_var": 265.52880859375,
      "regularization/kl": 0.4421055018901825,
      "rewards/chosen": 0.04661942791465103,
      "rewards/margins": 0.31981497679044846,
      "rewards/rejected": -0.27319554887579744,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 12.917749404907227,
      "kl": 8.632678031921387,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36128419.469879515,
      "logits/rejected": -36053909.61038961,
      "logps/chosen": -509.4839043674699,
      "logps/rejected": -378.4122615665584,
      "loss": 0.5256,
      "loss/base_kto": 3.6626203060150146,
      "loss/total_with_kl": 4.205124855041504,
      "metrics/advantage_var": 325.8285827636719,
      "regularization/advantage_var": 325.8285827636719,
      "regularization/kl": 0.5425044894218445,
      "rewards/chosen": 0.13677340817738728,
      "rewards/margins": 0.3419107352972889,
      "rewards/rejected": -0.20513732711990157,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 10.976865768432617,
      "kl": 10.955889701843262,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36438100.65500795,
      "logits/rejected": -36357725.59139785,
      "logps/chosen": -496.5558426073132,
      "logps/rejected": -369.4063460061444,
      "loss": 0.5163,
      "loss/base_kto": 3.6595475673675537,
      "loss/total_with_kl": 4.130549430847168,
      "metrics/advantage_var": 282.8838806152344,
      "regularization/advantage_var": 282.8838806152344,
      "regularization/kl": 0.4710017144680023,
      "rewards/chosen": 0.18982798420188915,
      "rewards/margins": 0.29242409722812285,
      "rewards/rejected": -0.10259611302623368,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 14.063850402832031,
      "kl": 8.425477981567383,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35098068.720893145,
      "logits/rejected": -35861402.07044411,
      "logps/chosen": -480.1312799043062,
      "logps/rejected": -362.8375765696784,
      "loss": 0.5184,
      "loss/base_kto": 3.654953718185425,
      "loss/total_with_kl": 4.147418022155762,
      "metrics/advantage_var": 295.7740173339844,
      "regularization/advantage_var": 295.7740173339844,
      "regularization/kl": 0.49246373772621155,
      "rewards/chosen": 0.08376480175547622,
      "rewards/margins": 0.3313756445902293,
      "rewards/rejected": -0.24761084283475307,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 22.535619735717773,
      "kl": 5.692698955535889,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -33987992.63849765,
      "logits/rejected": -36481305.160686426,
      "logps/chosen": -473.60450899843505,
      "logps/rejected": -393.6680479719189,
      "loss": 0.5261,
      "loss/base_kto": 3.7334465980529785,
      "loss/total_with_kl": 4.208803176879883,
      "metrics/advantage_var": 285.4994201660156,
      "regularization/advantage_var": 285.4994201660156,
      "regularization/kl": 0.4753565490245819,
      "rewards/chosen": -0.007258453279593741,
      "rewards/margins": 0.25475685521360536,
      "rewards/rejected": -0.2620153084931991,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 14.898407936096191,
      "kl": 7.375546455383301,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37071323.136,
      "logits/rejected": -37126512.95267176,
      "logps/chosen": -468.7284,
      "logps/rejected": -398.7224952290076,
      "loss": 0.5138,
      "loss/base_kto": 3.680459976196289,
      "loss/total_with_kl": 4.110528469085693,
      "metrics/advantage_var": 258.29962158203125,
      "regularization/advantage_var": 258.29962158203125,
      "regularization/kl": 0.43006888031959534,
      "rewards/chosen": 0.068622119140625,
      "rewards/margins": 0.30442510883826335,
      "rewards/rejected": -0.23580298969763835,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 14.685869216918945,
      "kl": 7.301432132720947,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -34996881.21880065,
      "logits/rejected": -36247459.330316745,
      "logps/chosen": -494.2022893030794,
      "logps/rejected": -366.40346907993967,
      "loss": 0.5117,
      "loss/base_kto": 3.652690887451172,
      "loss/total_with_kl": 4.093989849090576,
      "metrics/advantage_var": 265.0446472167969,
      "regularization/advantage_var": 265.0446472167969,
      "regularization/kl": 0.4412992596626282,
      "rewards/chosen": 0.06706471497274488,
      "rewards/margins": 0.3209760282210009,
      "rewards/rejected": -0.25391131324825605,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 17.012447357177734,
      "kl": 6.321325778961182,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -34797348.79510704,
      "logits/rejected": -36428875.24600639,
      "logps/chosen": -486.8264525993884,
      "logps/rejected": -369.2567392172524,
      "loss": 0.514,
      "loss/base_kto": 3.7007968425750732,
      "loss/total_with_kl": 4.11183500289917,
      "metrics/advantage_var": 246.86985778808594,
      "regularization/advantage_var": 246.86985778808594,
      "regularization/kl": 0.4110383093357086,
      "rewards/chosen": 0.08572891655318234,
      "rewards/margins": 0.29614313271357573,
      "rewards/rejected": -0.21041421616039338,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 21.558069229125977,
      "kl": 6.568594455718994,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36229036.27672956,
      "logits/rejected": -34802853.36645963,
      "logps/chosen": -477.03586871069183,
      "logps/rejected": -385.36818225931677,
      "loss": 0.5199,
      "loss/base_kto": 3.7086777687072754,
      "loss/total_with_kl": 4.158957004547119,
      "metrics/advantage_var": 270.4382019042969,
      "regularization/advantage_var": 270.4382019042969,
      "regularization/kl": 0.45027956366539,
      "rewards/chosen": 0.024344616716013016,
      "rewards/margins": 0.27192017520664774,
      "rewards/rejected": -0.2475755584906347,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 15.986832618713379,
      "kl": 10.523635864257812,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35453926.559006214,
      "logits/rejected": -36066287.89937107,
      "logps/chosen": -488.52135093167703,
      "logps/rejected": -365.40109080188677,
      "loss": 0.5114,
      "loss/base_kto": 3.6973044872283936,
      "loss/total_with_kl": 4.0909857749938965,
      "metrics/advantage_var": 236.445068359375,
      "regularization/advantage_var": 236.445068359375,
      "regularization/kl": 0.39368098974227905,
      "rewards/chosen": 0.19559511931046195,
      "rewards/margins": 0.2771852259626798,
      "rewards/rejected": -0.08159010665221784,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 16.754920959472656,
      "kl": 12.260619163513184,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -36413636.8620155,
      "logits/rejected": -36617994.88503937,
      "logps/chosen": -489.39670542635656,
      "logps/rejected": -366.496062992126,
      "loss": 0.5106,
      "loss/base_kto": 3.6487183570861816,
      "loss/total_with_kl": 4.084438800811768,
      "metrics/advantage_var": 261.6939697265625,
      "regularization/advantage_var": 261.6939697265625,
      "regularization/kl": 0.43572044372558594,
      "rewards/chosen": 0.2697226354318072,
      "rewards/margins": 0.3251430919651641,
      "rewards/rejected": -0.055420456533356914,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 13.55805778503418,
      "kl": 15.827859878540039,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36800989.97730956,
      "logits/rejected": -37554994.582202114,
      "logps/chosen": -479.0614870340357,
      "logps/rejected": -384.43071266968326,
      "loss": 0.5181,
      "loss/base_kto": 3.6306519508361816,
      "loss/total_with_kl": 4.144779205322266,
      "metrics/advantage_var": 308.7852478027344,
      "regularization/advantage_var": 308.7852478027344,
      "regularization/kl": 0.5141274333000183,
      "rewards/chosen": 0.2789377122875937,
      "rewards/margins": 0.3369197413465881,
      "rewards/rejected": -0.05798202905899439,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 13.294617652893066,
      "kl": 7.158412933349609,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36268843.47169811,
      "logits/rejected": -36132946.68322981,
      "logps/chosen": -471.4134728773585,
      "logps/rejected": -367.129173136646,
      "loss": 0.5121,
      "loss/base_kto": 3.6745097637176514,
      "loss/total_with_kl": 4.096756935119629,
      "metrics/advantage_var": 253.6019287109375,
      "regularization/advantage_var": 253.6019287109375,
      "regularization/kl": 0.4222472310066223,
      "rewards/chosen": 0.002667512533799657,
      "rewards/margins": 0.2960335360061769,
      "rewards/rejected": -0.2933660234723772,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 13.791971206665039,
      "kl": 12.70384407043457,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35769428.69172932,
      "logits/rejected": -37067341.42439024,
      "logps/chosen": -503.9091165413534,
      "logps/rejected": -361.11981707317074,
      "loss": 0.508,
      "loss/base_kto": 3.6423072814941406,
      "loss/total_with_kl": 4.063610553741455,
      "metrics/advantage_var": 253.0349884033203,
      "regularization/advantage_var": 253.0349884033203,
      "regularization/kl": 0.42130327224731445,
      "rewards/chosen": 0.24071848016036185,
      "rewards/margins": 0.32390460959856054,
      "rewards/rejected": -0.08318612943819868,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 23.122209548950195,
      "kl": 18.913740158081055,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -36850221.969040245,
      "logits/rejected": -37901305.51898734,
      "logps/chosen": -468.94465944272446,
      "logps/rejected": -361.54361155063293,
      "loss": 0.5082,
      "loss/base_kto": 3.410432815551758,
      "loss/total_with_kl": 4.065714359283447,
      "metrics/advantage_var": 393.5623779296875,
      "regularization/advantage_var": 393.5623779296875,
      "regularization/kl": 0.6552813649177551,
      "rewards/chosen": 0.5387757009027913,
      "rewards/margins": 0.5983267829827149,
      "rewards/rejected": -0.05955108207992361,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 17.803544998168945,
      "kl": 12.378803253173828,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -35954840.07920792,
      "logits/rejected": -37374268.01186944,
      "logps/chosen": -469.9499793729373,
      "logps/rejected": -365.42447143916917,
      "loss": 0.5176,
      "loss/base_kto": 3.1306824684143066,
      "loss/total_with_kl": 4.140599250793457,
      "metrics/advantage_var": 606.5565795898438,
      "regularization/advantage_var": 606.5565795898438,
      "regularization/kl": 1.0099166631698608,
      "rewards/chosen": 0.5161125762234426,
      "rewards/margins": 0.9470148026120739,
      "rewards/rejected": -0.4309022263886313,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 30.049779891967773,
      "kl": 10.264975547790527,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -33420564.625592418,
      "logits/rejected": -35505707.52395672,
      "logps/chosen": -463.80835308056874,
      "logps/rejected": -381.7633066074189,
      "loss": 0.5134,
      "loss/base_kto": 3.1224896907806396,
      "loss/total_with_kl": 4.1072797775268555,
      "metrics/advantage_var": 591.4656372070312,
      "regularization/advantage_var": 591.4656372070312,
      "regularization/kl": 0.9847902655601501,
      "rewards/chosen": 0.5197738478722354,
      "rewards/margins": 0.9656515126652705,
      "rewards/rejected": -0.44587766479303514,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 21.001802444458008,
      "kl": 9.1188325881958,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35266192.41025641,
      "logits/rejected": -37204579.90243903,
      "logps/chosen": -486.75320512820514,
      "logps/rejected": -379.7540015243902,
      "loss": 0.5272,
      "loss/base_kto": 3.172616958618164,
      "loss/total_with_kl": 4.2177653312683105,
      "metrics/advantage_var": 627.7164916992188,
      "regularization/advantage_var": 627.7164916992188,
      "regularization/kl": 1.0451478958129883,
      "rewards/chosen": 0.4424891838660607,
      "rewards/margins": 0.9121579211379975,
      "rewards/rejected": -0.4696687372719369,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 19.123069763183594,
      "kl": 8.992833137512207,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -34572778.09191759,
      "logits/rejected": -35946439.198767334,
      "logps/chosen": -473.49583993660855,
      "logps/rejected": -376.42221205701077,
      "loss": 0.5479,
      "loss/base_kto": 3.2300808429718018,
      "loss/total_with_kl": 4.383556365966797,
      "metrics/advantage_var": 692.7782592773438,
      "regularization/advantage_var": 692.7782592773438,
      "regularization/kl": 1.1534757614135742,
      "rewards/chosen": 0.33582137843886195,
      "rewards/margins": 0.8600832762503989,
      "rewards/rejected": -0.524261897811537,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 25.682153701782227,
      "kl": 12.340991020202637,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -33759355.1392405,
      "logits/rejected": -35178097.777777776,
      "logps/chosen": -456.51864121835445,
      "logps/rejected": -369.23042052469134,
      "loss": 0.5296,
      "loss/base_kto": 3.1937670707702637,
      "loss/total_with_kl": 4.236794948577881,
      "metrics/advantage_var": 626.4430541992188,
      "regularization/advantage_var": 626.4430541992188,
      "regularization/kl": 1.0430277585983276,
      "rewards/chosen": 0.5073145130012608,
      "rewards/margins": 0.8919978865349101,
      "rewards/rejected": -0.38468337353364923,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 14.437877655029297,
      "kl": 12.907342910766602,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -34224996.49283668,
      "logits/rejected": -33979712.219931275,
      "logps/chosen": -463.4241583094556,
      "logps/rejected": -374.91771370274915,
      "loss": 0.5354,
      "loss/base_kto": 3.1738390922546387,
      "loss/total_with_kl": 4.2835001945495605,
      "metrics/advantage_var": 666.4631958007812,
      "regularization/advantage_var": 666.4631958007812,
      "regularization/kl": 1.1096612215042114,
      "rewards/chosen": 0.5474070901515491,
      "rewards/margins": 0.8528692227660519,
      "rewards/rejected": -0.3054621326145028,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 17.568225860595703,
      "kl": 18.759857177734375,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34270990.29027113,
      "logits/rejected": -36013280.24502297,
      "logps/chosen": -467.75956937799043,
      "logps/rejected": -351.4282877105666,
      "loss": 0.5019,
      "loss/base_kto": 3.1603076457977295,
      "loss/total_with_kl": 4.015546798706055,
      "metrics/advantage_var": 513.6570434570312,
      "regularization/advantage_var": 513.6570434570312,
      "regularization/kl": 0.8552390336990356,
      "rewards/chosen": 0.6412341834255383,
      "rewards/margins": 0.8678184149321664,
      "rewards/rejected": -0.22658423150662807,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 20.05158233642578,
      "kl": 17.881261825561523,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35184865.408805035,
      "logits/rejected": -36423568.69565217,
      "logps/chosen": -473.2978085691824,
      "logps/rejected": -369.11201960403724,
      "loss": 0.5222,
      "loss/base_kto": 3.1760265827178955,
      "loss/total_with_kl": 4.1773552894592285,
      "metrics/advantage_var": 601.3986206054688,
      "regularization/advantage_var": 601.3986206054688,
      "regularization/kl": 1.0013288259506226,
      "rewards/chosen": 0.5759879538098222,
      "rewards/margins": 0.8547715269367632,
      "rewards/rejected": -0.278783573126941,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 11.790045738220215,
      "kl": 11.199904441833496,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -34577200.667697065,
      "logits/rejected": -35275314.95734597,
      "logps/chosen": -487.47111669242656,
      "logps/rejected": -350.2688586097946,
      "loss": 0.5104,
      "loss/base_kto": 3.13375186920166,
      "loss/total_with_kl": 4.082924842834473,
      "metrics/advantage_var": 570.07373046875,
      "regularization/advantage_var": 570.07373046875,
      "regularization/kl": 0.9491726160049438,
      "rewards/chosen": 0.5863393698077666,
      "rewards/margins": 0.956593428404602,
      "rewards/rejected": -0.3702540585968355,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 16.616731643676758,
      "kl": 10.115321159362793,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -34927872.3838081,
      "logits/rejected": -35150107.98042414,
      "logps/chosen": -498.2335550974513,
      "logps/rejected": -374.2097777324633,
      "loss": 0.5363,
      "loss/base_kto": 3.1613800525665283,
      "loss/total_with_kl": 4.290786266326904,
      "metrics/advantage_var": 678.3221435546875,
      "regularization/advantage_var": 678.3221435546875,
      "regularization/kl": 1.1294063329696655,
      "rewards/chosen": 0.5670057122317748,
      "rewards/margins": 0.9298887650888175,
      "rewards/rejected": -0.3628830528570427,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 19.794078826904297,
      "kl": 13.229277610778809,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -35329152.580937974,
      "logits/rejected": -37444599.728594504,
      "logps/chosen": -499.31855143721634,
      "logps/rejected": -376.1533218901454,
      "loss": 0.5087,
      "loss/base_kto": 3.1340301036834717,
      "loss/total_with_kl": 4.0699334144592285,
      "metrics/advantage_var": 562.10400390625,
      "regularization/advantage_var": 562.10400390625,
      "regularization/kl": 0.9359031915664673,
      "rewards/chosen": 0.6185328127216102,
      "rewards/margins": 0.9373207649536629,
      "rewards/rejected": -0.3187879522320527,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 21.475278854370117,
      "kl": 19.56073570251465,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35025939.75562701,
      "logits/rejected": -36578649.48328268,
      "logps/chosen": -456.02858721864953,
      "logps/rejected": -377.5395611702128,
      "loss": 0.5235,
      "loss/base_kto": 3.1346421241760254,
      "loss/total_with_kl": 4.187652587890625,
      "metrics/advantage_var": 632.4390258789062,
      "regularization/advantage_var": 632.4390258789062,
      "regularization/kl": 1.0530109405517578,
      "rewards/chosen": 0.6271888266805667,
      "rewards/margins": 0.9491588764140156,
      "rewards/rejected": -0.3219700497334489,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 18.9902400970459,
      "kl": 19.397607803344727,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35581945.488076314,
      "logits/rejected": -36827519.80337942,
      "logps/chosen": -488.9942368839428,
      "logps/rejected": -382.7798099078341,
      "loss": 0.5211,
      "loss/base_kto": 3.155086040496826,
      "loss/total_with_kl": 4.168495178222656,
      "metrics/advantage_var": 608.6544189453125,
      "regularization/advantage_var": 608.6544189453125,
      "regularization/kl": 1.0134094953536987,
      "rewards/chosen": 0.6110664695169664,
      "rewards/margins": 0.8886074587027744,
      "rewards/rejected": -0.2775409891858079,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 13.5298490524292,
      "kl": 18.607656478881836,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -33904899.24050633,
      "logits/rejected": -36248582.32098766,
      "logps/chosen": -476.12351661392404,
      "logps/rejected": -395.6892361111111,
      "loss": 0.5208,
      "loss/base_kto": 3.171834945678711,
      "loss/total_with_kl": 4.166215419769287,
      "metrics/advantage_var": 597.225341796875,
      "regularization/advantage_var": 597.225341796875,
      "regularization/kl": 0.9943801760673523,
      "rewards/chosen": 0.58079422576518,
      "rewards/margins": 0.8998145323877205,
      "rewards/rejected": -0.3190203066225405,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 9.113730430603027,
      "kl": 20.980093002319336,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35865745.84707646,
      "logits/rejected": -36366382.77324633,
      "logps/chosen": -481.3038324587706,
      "logps/rejected": -374.52472471451875,
      "loss": 0.5078,
      "loss/base_kto": 3.164994716644287,
      "loss/total_with_kl": 4.062058448791504,
      "metrics/advantage_var": 538.7771606445312,
      "regularization/advantage_var": 538.7771606445312,
      "regularization/kl": 0.8970640301704407,
      "rewards/chosen": 0.6543488509651424,
      "rewards/margins": 0.8510356749982977,
      "rewards/rejected": -0.19668682403315532,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 14.13674259185791,
      "kl": 31.113117218017578,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -37203584.9653092,
      "logits/rejected": -37281410.28200972,
      "logps/chosen": -460.94546568627453,
      "logps/rejected": -380.5852410858995,
      "loss": 0.5059,
      "loss/base_kto": 3.1098976135253906,
      "loss/total_with_kl": 4.047135829925537,
      "metrics/advantage_var": 562.9060668945312,
      "regularization/advantage_var": 562.9060668945312,
      "regularization/kl": 0.9372385144233704,
      "rewards/chosen": 0.8707901392227564,
      "rewards/margins": 0.936078701471889,
      "rewards/rejected": -0.06528856224913265,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 18.788869857788086,
      "kl": 22.033191680908203,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35469781.66614664,
      "logits/rejected": -36470463.499217525,
      "logps/chosen": -503.5286661466459,
      "logps/rejected": -389.35942390453835,
      "loss": 0.5703,
      "loss/base_kto": 3.1834867000579834,
      "loss/total_with_kl": 4.562434196472168,
      "metrics/advantage_var": 828.1964721679688,
      "regularization/advantage_var": 828.1964721679688,
      "regularization/kl": 1.378947138786316,
      "rewards/chosen": 0.6484626853335852,
      "rewards/margins": 0.8696144521628595,
      "rewards/rejected": -0.22115176682927426,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 21.770421981811523,
      "kl": 16.500782012939453,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34434442.325722985,
      "logits/rejected": -35681284.93097913,
      "logps/chosen": -509.9009703196347,
      "logps/rejected": -366.6454404093098,
      "loss": 0.5181,
      "loss/base_kto": 3.1322789192199707,
      "loss/total_with_kl": 4.14486837387085,
      "metrics/advantage_var": 608.161865234375,
      "regularization/advantage_var": 608.161865234375,
      "regularization/kl": 1.012589454650879,
      "rewards/chosen": 0.66641119226836,
      "rewards/margins": 0.9310437831607783,
      "rewards/rejected": -0.26463259089241825,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 12.632974624633789,
      "kl": 18.189037322998047,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34116828.43358779,
      "logits/rejected": -36532958.0032,
      "logps/chosen": -456.4682251908397,
      "logps/rejected": -372.2326,
      "loss": 0.5149,
      "loss/base_kto": 3.2032310962677,
      "loss/total_with_kl": 4.119079113006592,
      "metrics/advantage_var": 550.0590209960938,
      "regularization/advantage_var": 550.0590209960938,
      "regularization/kl": 0.9158482551574707,
      "rewards/chosen": 0.6247592139790077,
      "rewards/margins": 0.8383302589008828,
      "rewards/rejected": -0.213571044921875,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 15.308002471923828,
      "kl": 21.430456161499023,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34532757.26888218,
      "logits/rejected": -36014557.20388349,
      "logps/chosen": -480.0640577794562,
      "logps/rejected": -388.9739583333333,
      "loss": 0.5413,
      "loss/base_kto": 3.1949360370635986,
      "loss/total_with_kl": 4.330256938934326,
      "metrics/advantage_var": 681.8742065429688,
      "regularization/advantage_var": 681.8742065429688,
      "regularization/kl": 1.1353205442428589,
      "rewards/chosen": 0.6811915740505806,
      "rewards/margins": 0.8478852028493771,
      "rewards/rejected": -0.1666936287987965,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 30.371458053588867,
      "kl": 22.418798446655273,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -35363215.7696,
      "logits/rejected": -35970881.270229004,
      "logps/chosen": -465.29445,
      "logps/rejected": -385.3062261450382,
      "loss": 0.5415,
      "loss/base_kto": 3.2369346618652344,
      "loss/total_with_kl": 4.332038402557373,
      "metrics/advantage_var": 657.719970703125,
      "regularization/advantage_var": 657.719970703125,
      "regularization/kl": 1.0951037406921387,
      "rewards/chosen": 0.60460400390625,
      "rewards/margins": 0.7587009305298784,
      "rewards/rejected": -0.15409692662362834,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 7.786691188812256,
      "kl": 19.352018356323242,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -33677738.2504065,
      "logits/rejected": -36691638.47218045,
      "logps/chosen": -499.4220020325203,
      "logps/rejected": -400.281719924812,
      "loss": 0.5055,
      "loss/base_kto": 3.089252233505249,
      "loss/total_with_kl": 4.043831825256348,
      "metrics/advantage_var": 573.3209838867188,
      "regularization/advantage_var": 573.3209838867188,
      "regularization/kl": 0.9545795321464539,
      "rewards/chosen": 0.6374167341526931,
      "rewards/margins": 0.9257240044035154,
      "rewards/rejected": -0.28830727025082237,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 23.878217697143555,
      "kl": 17.17015266418457,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34343660.36691729,
      "logits/rejected": -34894023.80487805,
      "logps/chosen": -490.42598684210526,
      "logps/rejected": -373.0741869918699,
      "loss": 0.5007,
      "loss/base_kto": 3.183598041534424,
      "loss/total_with_kl": 4.00538969039917,
      "metrics/advantage_var": 493.568603515625,
      "regularization/advantage_var": 493.568603515625,
      "regularization/kl": 0.8217917680740356,
      "rewards/chosen": 0.6170498726063205,
      "rewards/margins": 0.8689512914007819,
      "rewards/rejected": -0.2519014187944614,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 21.530298233032227,
      "kl": 23.011098861694336,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -35550925.606299214,
      "logits/rejected": -37183583.255813956,
      "logps/chosen": -491.2183070866142,
      "logps/rejected": -383.9325581395349,
      "loss": 0.5324,
      "loss/base_kto": 3.167588472366333,
      "loss/total_with_kl": 4.259336948394775,
      "metrics/advantage_var": 655.7047119140625,
      "regularization/advantage_var": 655.7047119140625,
      "regularization/kl": 1.0917482376098633,
      "rewards/chosen": 0.7420297217181349,
      "rewards/margins": 0.9214242884536908,
      "rewards/rejected": -0.17939456673555595,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 10.666282653808594,
      "kl": 21.266172409057617,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -35279183.58346095,
      "logits/rejected": -38027107.215311006,
      "logps/chosen": -515.2427737366003,
      "logps/rejected": -352.39478668261563,
      "loss": 0.5311,
      "loss/base_kto": 3.1706020832061768,
      "loss/total_with_kl": 4.248837947845459,
      "metrics/advantage_var": 647.5892944335938,
      "regularization/advantage_var": 647.5892944335938,
      "regularization/kl": 1.0782362222671509,
      "rewards/chosen": 0.7221505835446975,
      "rewards/margins": 0.878060266309849,
      "rewards/rejected": -0.15590968276515152,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 17.214746475219727,
      "kl": 20.309959411621094,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -35189824.30917875,
      "logits/rejected": -37286748.84370258,
      "logps/chosen": -472.0552536231884,
      "logps/rejected": -365.98112670713203,
      "loss": 0.509,
      "loss/base_kto": 3.1659202575683594,
      "loss/total_with_kl": 4.072198390960693,
      "metrics/advantage_var": 544.3112182617188,
      "regularization/advantage_var": 544.3112182617188,
      "regularization/kl": 0.906278133392334,
      "rewards/chosen": 0.6328442952867099,
      "rewards/margins": 0.871725060561985,
      "rewards/rejected": -0.23888076527527505,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 132.70606994628906,
      "kl": 17.47706413269043,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34890082.093457945,
      "logits/rejected": -35401368.47648903,
      "logps/chosen": -482.0947235202492,
      "logps/rejected": -382.3184267241379,
      "loss": 0.52,
      "loss/base_kto": 3.2115752696990967,
      "loss/total_with_kl": 4.1597418785095215,
      "metrics/advantage_var": 569.4691162109375,
      "regularization/advantage_var": 569.4691162109375,
      "regularization/kl": 0.9481660723686218,
      "rewards/chosen": 0.5828254675939447,
      "rewards/margins": 0.8018098263306586,
      "rewards/rejected": -0.21898435873671385,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 18.174816131591797,
      "kl": 10.889971733093262,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34250043.71941272,
      "logits/rejected": -36426864.07196402,
      "logps/chosen": -470.50881933115824,
      "logps/rejected": -367.67648988006,
      "loss": 0.4985,
      "loss/base_kto": 3.2247750759124756,
      "loss/total_with_kl": 3.9877007007598877,
      "metrics/advantage_var": 458.2135314941406,
      "regularization/advantage_var": 458.2135314941406,
      "regularization/kl": 0.7629255652427673,
      "rewards/chosen": 0.5111739141517129,
      "rewards/margins": 0.8395675786475899,
      "rewards/rejected": -0.32839366449587704,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 24.727251052856445,
      "kl": 7.525174140930176,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -33379049.296636086,
      "logits/rejected": -35816722.8115016,
      "logps/chosen": -473.3429854740061,
      "logps/rejected": -364.55051916932905,
      "loss": 0.5236,
      "loss/base_kto": 3.204340696334839,
      "loss/total_with_kl": 4.188536167144775,
      "metrics/advantage_var": 591.1080322265625,
      "regularization/advantage_var": 591.1080322265625,
      "regularization/kl": 0.9841948747634888,
      "rewards/chosen": 0.44478586179400803,
      "rewards/margins": 0.8876847431151442,
      "rewards/rejected": -0.4428988813211362,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 81.66817474365234,
      "kl": 19.855487823486328,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35810387.82664526,
      "logits/rejected": -35341082.88584475,
      "logps/chosen": -499.32609349919744,
      "logps/rejected": -395.1006230974125,
      "loss": 0.5183,
      "loss/base_kto": 3.1676383018493652,
      "loss/total_with_kl": 4.1465253829956055,
      "metrics/advantage_var": 587.9201049804688,
      "regularization/advantage_var": 587.9201049804688,
      "regularization/kl": 0.9788870215415955,
      "rewards/chosen": 0.6302710666319723,
      "rewards/margins": 0.8804101840543229,
      "rewards/rejected": -0.2501391174223506,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 13.090487480163574,
      "kl": 11.669354438781738,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -33785616.0,
      "logits/rejected": -36079353.6,
      "logps/chosen": -470.088134765625,
      "logps/rejected": -378.515869140625,
      "loss": 0.5388,
      "loss/base_kto": 3.1741302013397217,
      "loss/total_with_kl": 4.31029748916626,
      "metrics/advantage_var": 682.3826904296875,
      "regularization/advantage_var": 682.3826904296875,
      "regularization/kl": 1.1361671686172485,
      "rewards/chosen": 0.55972900390625,
      "rewards/margins": 0.91335186958313,
      "rewards/rejected": -0.3536228656768799,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 11.46224594116211,
      "kl": 14.195319175720215,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -35331752.10794602,
      "logits/rejected": -35831473.90538336,
      "logps/chosen": -469.05406671664167,
      "logps/rejected": -360.18046492659056,
      "loss": 0.6604,
      "loss/base_kto": 3.2137787342071533,
      "loss/total_with_kl": 5.2835774421691895,
      "metrics/advantage_var": 1243.1224365234375,
      "regularization/advantage_var": 1243.1224365234375,
      "regularization/kl": 2.0697989463806152,
      "rewards/chosen": 0.6094958349682581,
      "rewards/margins": 0.8998844805573181,
      "rewards/rejected": -0.29038864558905997,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 10.405191421508789,
      "kl": 19.894010543823242,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34590439.9872814,
      "logits/rejected": -36019840.19662058,
      "logps/chosen": -495.714626391097,
      "logps/rejected": -385.013176843318,
      "loss": 0.4964,
      "loss/base_kto": 3.168362617492676,
      "loss/total_with_kl": 3.9712867736816406,
      "metrics/advantage_var": 482.2367858886719,
      "regularization/advantage_var": 482.2367858886719,
      "regularization/kl": 0.8029242753982544,
      "rewards/chosen": 0.6687352312388215,
      "rewards/margins": 0.8462016398702239,
      "rewards/rejected": -0.17746640863140242,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 13.125397682189941,
      "kl": 15.497633934020996,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -35739088.87363494,
      "logits/rejected": -36400065.50234742,
      "logps/chosen": -479.00409516380654,
      "logps/rejected": -362.5519610719875,
      "loss": 0.5092,
      "loss/base_kto": 3.217585802078247,
      "loss/total_with_kl": 4.0731987953186035,
      "metrics/advantage_var": 513.8816528320312,
      "regularization/advantage_var": 513.8816528320312,
      "regularization/kl": 0.8556130528450012,
      "rewards/chosen": 0.5395711580416341,
      "rewards/margins": 0.808753891568446,
      "rewards/rejected": -0.26918273352681193,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 9.873025894165039,
      "kl": 12.705950736999512,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -34435339.41401274,
      "logits/rejected": -36264476.26993865,
      "logps/chosen": -465.34633757961785,
      "logps/rejected": -391.04637174079755,
      "loss": 0.5051,
      "loss/base_kto": 3.231473207473755,
      "loss/total_with_kl": 4.040536403656006,
      "metrics/advantage_var": 485.9240417480469,
      "regularization/advantage_var": 485.9240417480469,
      "regularization/kl": 0.8090634346008301,
      "rewards/chosen": 0.4578510940454568,
      "rewards/margins": 0.7875057174398203,
      "rewards/rejected": -0.3296546233943635,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 19.27297592163086,
      "kl": 13.081128120422363,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -34923876.243369736,
      "logits/rejected": -37735089.076682314,
      "logps/chosen": -486.90103354134163,
      "logps/rejected": -390.3783499608764,
      "loss": 0.5035,
      "loss/base_kto": 3.196063756942749,
      "loss/total_with_kl": 4.028075695037842,
      "metrics/advantage_var": 499.7066345214844,
      "regularization/advantage_var": 499.7066345214844,
      "regularization/kl": 0.8320116400718689,
      "rewards/chosen": 0.5644023258488934,
      "rewards/margins": 0.8339658604855327,
      "rewards/rejected": -0.2695635346366393,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 47.9726448059082,
      "kl": 14.978861808776855,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33340049.265116278,
      "logits/rejected": -35089638.6015748,
      "logps/chosen": -504.12107558139536,
      "logps/rejected": -365.18033956692915,
      "loss": 0.5166,
      "loss/base_kto": 3.168360471725464,
      "loss/total_with_kl": 4.132617473602295,
      "metrics/advantage_var": 579.13330078125,
      "regularization/advantage_var": 579.13330078125,
      "regularization/kl": 0.9642569422721863,
      "rewards/chosen": 0.5865663513656735,
      "rewards/margins": 0.9104793969334639,
      "rewards/rejected": -0.3239130455677904,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 12.994561195373535,
      "kl": 20.298545837402344,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -33154546.52631579,
      "logits/rejected": -34941696.0,
      "logps/chosen": -489.10043174342104,
      "logps/rejected": -368.0780784970238,
      "loss": 0.5073,
      "loss/base_kto": 3.1604514122009277,
      "loss/total_with_kl": 4.058101177215576,
      "metrics/advantage_var": 539.12890625,
      "regularization/advantage_var": 539.12890625,
      "regularization/kl": 0.8976497054100037,
      "rewards/chosen": 0.6708749469957853,
      "rewards/margins": 0.889276719631109,
      "rewards/rejected": -0.21840177263532365,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 6.703609943389893,
      "kl": 10.383610725402832,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35828997.274167985,
      "logits/rejected": -36020733.625966,
      "logps/chosen": -499.90887480190173,
      "logps/rejected": -372.4459524729521,
      "loss": 0.4814,
      "loss/base_kto": 3.183142900466919,
      "loss/total_with_kl": 3.851154327392578,
      "metrics/advantage_var": 401.2080993652344,
      "regularization/advantage_var": 401.2080993652344,
      "regularization/kl": 0.668011486530304,
      "rewards/chosen": 0.4524446176084588,
      "rewards/margins": 0.8381700841170441,
      "rewards/rejected": -0.3857254665085853,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 20.486181259155273,
      "kl": 12.683451652526855,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34344392.36535433,
      "logits/rejected": -35632847.18139535,
      "logps/chosen": -482.1603346456693,
      "logps/rejected": -389.7322189922481,
      "loss": 0.4673,
      "loss/base_kto": 3.0994527339935303,
      "loss/total_with_kl": 3.7386157512664795,
      "metrics/advantage_var": 383.8816833496094,
      "regularization/advantage_var": 383.8816833496094,
      "regularization/kl": 0.6391629576683044,
      "rewards/chosen": 0.5894631213090551,
      "rewards/margins": 0.9362109410764969,
      "rewards/rejected": -0.34674781976744184,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 12.909422874450684,
      "kl": 12.25561809539795,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -34363460.266666666,
      "logits/rejected": -34902749.31612903,
      "logps/chosen": -479.07670454545456,
      "logps/rejected": -381.37142137096777,
      "loss": 0.4675,
      "loss/base_kto": 3.1592354774475098,
      "loss/total_with_kl": 3.7402336597442627,
      "metrics/advantage_var": 348.9477233886719,
      "regularization/advantage_var": 348.9477233886719,
      "regularization/kl": 0.580997884273529,
      "rewards/chosen": 0.5304400819720644,
      "rewards/margins": 0.8264333208518526,
      "rewards/rejected": -0.2959932388797883,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 8.274309158325195,
      "kl": 16.63254165649414,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -35039798.327974275,
      "logits/rejected": -37158880.87537994,
      "logps/chosen": -479.2469855305466,
      "logps/rejected": -405.74534574468083,
      "loss": 0.4626,
      "loss/base_kto": 3.1412341594696045,
      "loss/total_with_kl": 3.700852632522583,
      "metrics/advantage_var": 336.1070251464844,
      "regularization/advantage_var": 336.1070251464844,
      "regularization/kl": 0.5596182346343994,
      "rewards/chosen": 0.5669709723862038,
      "rewards/margins": 0.8345602747970368,
      "rewards/rejected": -0.267589302410833,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 13.507829666137695,
      "kl": 15.458251953125,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34374997.333333336,
      "logits/rejected": -35833673.884555385,
      "logps/chosen": -454.6964495305164,
      "logps/rejected": -376.2020768330733,
      "loss": 0.46,
      "loss/base_kto": 3.1337621212005615,
      "loss/total_with_kl": 3.6797828674316406,
      "metrics/advantage_var": 327.9403991699219,
      "regularization/advantage_var": 327.9403991699219,
      "regularization/kl": 0.5460208058357239,
      "rewards/chosen": 0.5840901119608275,
      "rewards/margins": 0.86935336392157,
      "rewards/rejected": -0.2852632519607425,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 9.739164352416992,
      "kl": 12.5602388381958,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -36261101.4939759,
      "logits/rejected": -36452877.2987013,
      "logps/chosen": -475.23870481927713,
      "logps/rejected": -375.8420505275974,
      "loss": 0.4679,
      "loss/base_kto": 3.14152193069458,
      "loss/total_with_kl": 3.743072509765625,
      "metrics/advantage_var": 361.2915954589844,
      "regularization/advantage_var": 361.2915954589844,
      "regularization/kl": 0.6015505194664001,
      "rewards/chosen": 0.6037944196218468,
      "rewards/margins": 0.8639737658798368,
      "rewards/rejected": -0.26017934625799005,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 11.08957576751709,
      "kl": 16.796659469604492,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34987716.678071536,
      "logits/rejected": -36955770.97645212,
      "logps/chosen": -497.6564930015552,
      "logps/rejected": -363.1662333202512,
      "loss": 0.47,
      "loss/base_kto": 3.0917279720306396,
      "loss/total_with_kl": 3.7601680755615234,
      "metrics/advantage_var": 401.4655456542969,
      "regularization/advantage_var": 401.4655456542969,
      "regularization/kl": 0.6684401631355286,
      "rewards/chosen": 0.6684237134586412,
      "rewards/margins": 0.9006318935493373,
      "rewards/rejected": -0.23220818009069613,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 12.405346870422363,
      "kl": 13.004144668579102,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -35085379.62264151,
      "logits/rejected": -36464627.27950311,
      "logps/chosen": -478.507419418239,
      "logps/rejected": -370.686286878882,
      "loss": 0.4676,
      "loss/base_kto": 3.133150100708008,
      "loss/total_with_kl": 3.7407402992248535,
      "metrics/advantage_var": 364.9190368652344,
      "regularization/advantage_var": 364.9190368652344,
      "regularization/kl": 0.6075901389122009,
      "rewards/chosen": 0.5498898104301788,
      "rewards/margins": 0.8730859135439575,
      "rewards/rejected": -0.3231961031137786,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 18.598962783813477,
      "kl": 18.174997329711914,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34538869.84126984,
      "logits/rejected": -34755177.55076923,
      "logps/chosen": -465.79578373015875,
      "logps/rejected": -381.8952884615385,
      "loss": 0.4713,
      "loss/base_kto": 3.144207000732422,
      "loss/total_with_kl": 3.7700016498565674,
      "metrics/advantage_var": 375.8526306152344,
      "regularization/advantage_var": 375.8526306152344,
      "regularization/kl": 0.6257945895195007,
      "rewards/chosen": 0.5810432070777529,
      "rewards/margins": 0.8297481349623683,
      "rewards/rejected": -0.24870492788461537,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 8.991421699523926,
      "kl": 13.875744819641113,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -35983386.256410256,
      "logits/rejected": -37287817.36585366,
      "logps/chosen": -476.5966546474359,
      "logps/rejected": -367.7307069359756,
      "loss": 0.4629,
      "loss/base_kto": 3.1394855976104736,
      "loss/total_with_kl": 3.703155517578125,
      "metrics/advantage_var": 338.5404357910156,
      "regularization/advantage_var": 338.5404357910156,
      "regularization/kl": 0.5636698603630066,
      "rewards/chosen": 0.5675454261975411,
      "rewards/margins": 0.8820818232476078,
      "rewards/rejected": -0.3145363970500667,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 14.043456077575684,
      "kl": 11.75983715057373,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34603116.27794562,
      "logits/rejected": -36045151.275080904,
      "logps/chosen": -487.02841767371604,
      "logps/rejected": -367.93805622977345,
      "loss": 0.4656,
      "loss/base_kto": 3.124924421310425,
      "loss/total_with_kl": 3.724797010421753,
      "metrics/advantage_var": 360.28399658203125,
      "regularization/advantage_var": 360.28399658203125,
      "regularization/kl": 0.599872887134552,
      "rewards/chosen": 0.5678141614101444,
      "rewards/margins": 0.8667035437817692,
      "rewards/rejected": -0.2988893823716247,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 11.319145202636719,
      "kl": 20.254621505737305,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34745140.49211357,
      "logits/rejected": -35675890.526315786,
      "logps/chosen": -484.886238170347,
      "logps/rejected": -388.7073335913313,
      "loss": 0.4664,
      "loss/base_kto": 3.105311155319214,
      "loss/total_with_kl": 3.7308807373046875,
      "metrics/advantage_var": 375.7174987792969,
      "regularization/advantage_var": 375.7174987792969,
      "regularization/kl": 0.6255696415901184,
      "rewards/chosen": 0.7004149524195337,
      "rewards/margins": 0.894981023637546,
      "rewards/rejected": -0.19456607121801228,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 6.997559070587158,
      "kl": 13.4202880859375,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34595846.28220859,
      "logits/rejected": -36061069.85987261,
      "logps/chosen": -465.620446702454,
      "logps/rejected": -370.55667794585986,
      "loss": 0.4606,
      "loss/base_kto": 3.147129535675049,
      "loss/total_with_kl": 3.6847634315490723,
      "metrics/advantage_var": 322.9032287597656,
      "regularization/advantage_var": 322.9032287597656,
      "regularization/kl": 0.5376338362693787,
      "rewards/chosen": 0.5557675507902368,
      "rewards/margins": 0.8490083834652715,
      "rewards/rejected": -0.2932408326750348,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 12.055425643920898,
      "kl": 14.774609565734863,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -35488962.97451274,
      "logits/rejected": -36899754.80587275,
      "logps/chosen": -524.7817185157421,
      "logps/rejected": -358.01661908646,
      "loss": 0.4679,
      "loss/base_kto": 3.0890705585479736,
      "loss/total_with_kl": 3.743443727493286,
      "metrics/advantage_var": 393.0169372558594,
      "regularization/advantage_var": 393.0169372558594,
      "regularization/kl": 0.6543732285499573,
      "rewards/chosen": 0.6401997213063391,
      "rewards/margins": 0.9174135849093927,
      "rewards/rejected": -0.2772138636030536,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 8.9901704788208,
      "kl": 16.92176628112793,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -34209161.35614308,
      "logits/rejected": -36074195.39089482,
      "logps/chosen": -491.05350894245726,
      "logps/rejected": -366.507751177394,
      "loss": 0.4678,
      "loss/base_kto": 3.1431918144226074,
      "loss/total_with_kl": 3.742206335067749,
      "metrics/advantage_var": 359.76837158203125,
      "regularization/advantage_var": 359.76837158203125,
      "regularization/kl": 0.5990143418312073,
      "rewards/chosen": 0.5929558785113724,
      "rewards/margins": 0.846540089013482,
      "rewards/rejected": -0.2535842105021095,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 7.918067932128906,
      "kl": 19.62078857421875,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -34853530.33492823,
      "logits/rejected": -35989707.85911179,
      "logps/chosen": -449.56539074960125,
      "logps/rejected": -385.4023258039816,
      "loss": 0.4543,
      "loss/base_kto": 3.0972354412078857,
      "loss/total_with_kl": 3.634321689605713,
      "metrics/advantage_var": 322.5742492675781,
      "regularization/advantage_var": 322.5742492675781,
      "regularization/kl": 0.5370860695838928,
      "rewards/chosen": 0.6514162979247657,
      "rewards/margins": 0.8836477768860502,
      "rewards/rejected": -0.23223147896128446,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 10.17796802520752,
      "kl": 15.71185302734375,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -33575923.39692308,
      "logits/rejected": -36114002.895238094,
      "logps/chosen": -475.3112980769231,
      "logps/rejected": -370.634375,
      "loss": 0.4581,
      "loss/base_kto": 3.1170105934143066,
      "loss/total_with_kl": 3.6648223400115967,
      "metrics/advantage_var": 329.01629638671875,
      "regularization/advantage_var": 329.01629638671875,
      "regularization/kl": 0.5478121042251587,
      "rewards/chosen": 0.6221265587439904,
      "rewards/margins": 0.8973633121774601,
      "rewards/rejected": -0.27523675343346976,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 9.963101387023926,
      "kl": 15.151724815368652,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -34225013.89473684,
      "logits/rejected": -36751024.76190476,
      "logps/chosen": -474.8861019736842,
      "logps/rejected": -376.24004836309524,
      "loss": 0.4609,
      "loss/base_kto": 3.1314008235931396,
      "loss/total_with_kl": 3.6872289180755615,
      "metrics/advantage_var": 333.8307189941406,
      "regularization/advantage_var": 333.8307189941406,
      "regularization/kl": 0.5558281540870667,
      "rewards/chosen": 0.5898361707988539,
      "rewards/margins": 0.8616058306586474,
      "rewards/rejected": -0.2717696598597935,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 8.92536449432373,
      "kl": 13.524930000305176,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34468356.72615384,
      "logits/rejected": -36522699.17460317,
      "logps/chosen": -478.20240384615386,
      "logps/rejected": -361.2171626984127,
      "loss": 0.4656,
      "loss/base_kto": 3.1335270404815674,
      "loss/total_with_kl": 3.724987030029297,
      "metrics/advantage_var": 355.2311096191406,
      "regularization/advantage_var": 355.2311096191406,
      "regularization/kl": 0.5914598107337952,
      "rewards/chosen": 0.5554096867487981,
      "rewards/margins": 0.8794731846251622,
      "rewards/rejected": -0.32406349787636407,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 10.868731498718262,
      "kl": 13.2905912399292,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34153105.94015748,
      "logits/rejected": -35693204.440310076,
      "logps/chosen": -508.6543799212598,
      "logps/rejected": -383.42766472868215,
      "loss": 0.4592,
      "loss/base_kto": 3.1157286167144775,
      "loss/total_with_kl": 3.6738109588623047,
      "metrics/advantage_var": 335.1844787597656,
      "regularization/advantage_var": 335.1844787597656,
      "regularization/kl": 0.5580821633338928,
      "rewards/chosen": 0.5697481891301673,
      "rewards/margins": 0.8933302336281082,
      "rewards/rejected": -0.32358204449794087,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 10.145153045654297,
      "kl": 15.356244087219238,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34969000.3456,
      "logits/rejected": -36647656.15877862,
      "logps/chosen": -460.0201,
      "logps/rejected": -378.9165791984733,
      "loss": 0.4654,
      "loss/base_kto": 3.089923620223999,
      "loss/total_with_kl": 3.7231736183166504,
      "metrics/advantage_var": 380.330322265625,
      "regularization/advantage_var": 380.330322265625,
      "regularization/kl": 0.6332499384880066,
      "rewards/chosen": 0.59560576171875,
      "rewards/margins": 0.9119439412124284,
      "rewards/rejected": -0.31633817949367843,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 18.175031661987305,
      "kl": 13.56359577178955,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35729968.83692308,
      "logits/rejected": -36075240.43174603,
      "logps/chosen": -483.73841346153847,
      "logps/rejected": -394.5961557539683,
      "loss": 0.4727,
      "loss/base_kto": 3.1271884441375732,
      "loss/total_with_kl": 3.7812340259552,
      "metrics/advantage_var": 392.8203430175781,
      "regularization/advantage_var": 392.8203430175781,
      "regularization/kl": 0.654045820236206,
      "rewards/chosen": 0.5983189978966346,
      "rewards/margins": 0.891011248950702,
      "rewards/rejected": -0.2926922510540675,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 13.319075584411621,
      "kl": 15.165995597839355,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -34791957.82295082,
      "logits/rejected": -36172347.60597015,
      "logps/chosen": -474.82899590163936,
      "logps/rejected": -401.5444962686567,
      "loss": 0.4701,
      "loss/base_kto": 3.161012649536133,
      "loss/total_with_kl": 3.761154890060425,
      "metrics/advantage_var": 360.4458923339844,
      "regularization/advantage_var": 360.4458923339844,
      "regularization/kl": 0.6001423597335815,
      "rewards/chosen": 0.5497582607581967,
      "rewards/margins": 0.8475414775477839,
      "rewards/rejected": -0.29778321678958725,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 15.17506217956543,
      "kl": 9.989168167114258,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35060965.54531001,
      "logits/rejected": -35499975.37327189,
      "logps/chosen": -507.5405405405405,
      "logps/rejected": -360.3473742319508,
      "loss": 0.4705,
      "loss/base_kto": 3.1441006660461426,
      "loss/total_with_kl": 3.76410174369812,
      "metrics/advantage_var": 372.3731384277344,
      "regularization/advantage_var": 372.3731384277344,
      "regularization/kl": 0.6200013160705566,
      "rewards/chosen": 0.5026958326088037,
      "rewards/margins": 0.8739539134384768,
      "rewards/rejected": -0.3712580808296731,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 10.091706275939941,
      "kl": 12.291145324707031,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35119788.8,
      "logits/rejected": -35423200.0,
      "logps/chosen": -499.049609375,
      "logps/rejected": -407.73125,
      "loss": 0.464,
      "loss/base_kto": 3.096036911010742,
      "loss/total_with_kl": 3.7123208045959473,
      "metrics/advantage_var": 370.140380859375,
      "regularization/advantage_var": 370.140380859375,
      "regularization/kl": 0.6162838339805603,
      "rewards/chosen": 0.5213249683380127,
      "rewards/margins": 0.9092860221862793,
      "rewards/rejected": -0.3879610538482666,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 12.230925559997559,
      "kl": 11.54342269897461,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -35085299.2,
      "logits/rejected": -36306348.8,
      "logps/chosen": -498.4134765625,
      "logps/rejected": -377.816064453125,
      "loss": 0.4591,
      "loss/base_kto": 3.092241048812866,
      "loss/total_with_kl": 3.672832489013672,
      "metrics/advantage_var": 348.70391845703125,
      "regularization/advantage_var": 348.70391845703125,
      "regularization/kl": 0.5805919766426086,
      "rewards/chosen": 0.5286848068237304,
      "rewards/margins": 0.9156812667846679,
      "rewards/rejected": -0.3869964599609375,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 10.809049606323242,
      "kl": 9.5625,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -34057415.98825257,
      "logits/rejected": -36385041.52253756,
      "logps/chosen": -503.7184287812041,
      "logps/rejected": -352.2060204507512,
      "loss": 0.492,
      "loss/base_kto": 3.1409976482391357,
      "loss/total_with_kl": 3.936307907104492,
      "metrics/advantage_var": 477.6637268066406,
      "regularization/advantage_var": 477.6637268066406,
      "regularization/kl": 0.7953101396560669,
      "rewards/chosen": 0.5560439861818557,
      "rewards/margins": 0.9007480808154898,
      "rewards/rejected": -0.34470409463363416,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 9.063849449157715,
      "kl": 15.310641288757324,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -33109751.116719242,
      "logits/rejected": -36221435.24458204,
      "logps/chosen": -474.403785488959,
      "logps/rejected": -352.32580301857587,
      "loss": 0.4633,
      "loss/base_kto": 3.134089708328247,
      "loss/total_with_kl": 3.7060189247131348,
      "metrics/advantage_var": 343.5010681152344,
      "regularization/advantage_var": 343.5010681152344,
      "regularization/kl": 0.5719292759895325,
      "rewards/chosen": 0.5670358555550079,
      "rewards/margins": 0.8584075347134106,
      "rewards/rejected": -0.2913716791584027,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 10.3336763381958,
      "kl": 17.177276611328125,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34365693.99058085,
      "logits/rejected": -37128803.53343701,
      "logps/chosen": -492.1367739403454,
      "logps/rejected": -375.41927488335926,
      "loss": 0.456,
      "loss/base_kto": 3.0802464485168457,
      "loss/total_with_kl": 3.647966146469116,
      "metrics/advantage_var": 340.9729919433594,
      "regularization/advantage_var": 340.9729919433594,
      "regularization/kl": 0.5677200555801392,
      "rewards/chosen": 0.6075130444680877,
      "rewards/margins": 0.914466249202973,
      "rewards/rejected": -0.3069532047348853,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 12.794276237487793,
      "kl": 12.41401195526123,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35897003.71692308,
      "logits/rejected": -37448583.72063492,
      "logps/chosen": -484.5631730769231,
      "logps/rejected": -380.8362599206349,
      "loss": 0.46,
      "loss/base_kto": 3.1174824237823486,
      "loss/total_with_kl": 3.679964065551758,
      "metrics/advantage_var": 337.826904296875,
      "regularization/advantage_var": 337.826904296875,
      "regularization/kl": 0.5624818205833435,
      "rewards/chosen": 0.6082245342548077,
      "rewards/margins": 0.9049272552602021,
      "rewards/rejected": -0.29670272100539435,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 17.76041603088379,
      "kl": 16.423965454101562,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34931012.05063291,
      "logits/rejected": -35125466.074074075,
      "logps/chosen": -482.17513844936707,
      "logps/rejected": -353.14067322530866,
      "loss": 0.4694,
      "loss/base_kto": 3.076963186264038,
      "loss/total_with_kl": 3.7554919719696045,
      "metrics/advantage_var": 407.5247497558594,
      "regularization/advantage_var": 407.5247497558594,
      "regularization/kl": 0.6785287261009216,
      "rewards/chosen": 0.6269692529605914,
      "rewards/margins": 0.9440071885707622,
      "rewards/rejected": -0.31703793561017074,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 9.461185455322266,
      "kl": 15.859339714050293,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35878449.23076923,
      "logits/rejected": -35323769.75609756,
      "logps/chosen": -459.3909254807692,
      "logps/rejected": -403.8202648628049,
      "loss": 0.464,
      "loss/base_kto": 3.079575300216675,
      "loss/total_with_kl": 3.712149143218994,
      "metrics/advantage_var": 379.92425537109375,
      "regularization/advantage_var": 379.92425537109375,
      "regularization/kl": 0.6325739026069641,
      "rewards/chosen": 0.6018277192727114,
      "rewards/margins": 0.9291157188677952,
      "rewards/rejected": -0.32728799959508387,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 11.36967945098877,
      "kl": 12.651641845703125,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -34188158.197183095,
      "logits/rejected": -35288410.65834633,
      "logps/chosen": -464.17233959311426,
      "logps/rejected": -374.37760822932916,
      "loss": 0.4644,
      "loss/base_kto": 3.135221481323242,
      "loss/total_with_kl": 3.715381622314453,
      "metrics/advantage_var": 348.4446716308594,
      "regularization/advantage_var": 348.4446716308594,
      "regularization/kl": 0.58016037940979,
      "rewards/chosen": 0.5999769231709703,
      "rewards/margins": 0.8528162481697514,
      "rewards/rejected": -0.2528393249987812,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 11.526399612426758,
      "kl": 14.372525215148926,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35875684.17391305,
      "logits/rejected": -36564476.77987421,
      "logps/chosen": -496.04580745341616,
      "logps/rejected": -369.88600628930817,
      "loss": 0.4594,
      "loss/base_kto": 3.0919206142425537,
      "loss/total_with_kl": 3.6748077869415283,
      "metrics/advantage_var": 350.0826416015625,
      "regularization/advantage_var": 350.0826416015625,
      "regularization/kl": 0.582887589931488,
      "rewards/chosen": 0.5898574923876649,
      "rewards/margins": 0.9130547192419053,
      "rewards/rejected": -0.32319722685424035,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 8.888420104980469,
      "kl": 15.188580513000488,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35071442.91823899,
      "logits/rejected": -36805921.39130435,
      "logps/chosen": -470.28911163522014,
      "logps/rejected": -371.9250776397516,
      "loss": 0.4576,
      "loss/base_kto": 3.078230619430542,
      "loss/total_with_kl": 3.6605050563812256,
      "metrics/advantage_var": 349.7142333984375,
      "regularization/advantage_var": 349.7142333984375,
      "regularization/kl": 0.5822741985321045,
      "rewards/chosen": 0.59386304939318,
      "rewards/margins": 0.9161499514002889,
      "rewards/rejected": -0.3222869020071089,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 13.013932228088379,
      "kl": 15.081308364868164,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -35560366.82926829,
      "logits/rejected": -35769974.15384615,
      "logps/chosen": -469.3050209603659,
      "logps/rejected": -389.21817407852564,
      "loss": 0.4646,
      "loss/base_kto": 3.158924102783203,
      "loss/total_with_kl": 3.7165520191192627,
      "metrics/advantage_var": 334.9115905761719,
      "regularization/advantage_var": 334.9115905761719,
      "regularization/kl": 0.55762779712677,
      "rewards/chosen": 0.5677999170815072,
      "rewards/margins": 0.8535911674571082,
      "rewards/rejected": -0.28579125037560094,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.753754138946533,
      "kl": 13.397297859191895,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34964381.3206951,
      "logits/rejected": -37000560.766615145,
      "logps/chosen": -477.7504443127962,
      "logps/rejected": -362.28767387944356,
      "loss": 0.4592,
      "loss/base_kto": 3.149555206298828,
      "loss/total_with_kl": 3.6733086109161377,
      "metrics/advantage_var": 314.5665588378906,
      "regularization/advantage_var": 314.5665588378906,
      "regularization/kl": 0.5237533450126648,
      "rewards/chosen": 0.5751637824903733,
      "rewards/margins": 0.8500131096131515,
      "rewards/rejected": -0.2748493271227782,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 9.024127960205078,
      "kl": 14.589932441711426,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -33744376.788732395,
      "logits/rejected": -35623889.67238689,
      "logps/chosen": -474.76951291079814,
      "logps/rejected": -351.01794071762873,
      "loss": 0.4655,
      "loss/base_kto": 3.133312940597534,
      "loss/total_with_kl": 3.7237255573272705,
      "metrics/advantage_var": 354.6022033691406,
      "regularization/advantage_var": 354.6022033691406,
      "regularization/kl": 0.5904126167297363,
      "rewards/chosen": 0.5715717918622848,
      "rewards/margins": 0.8627933271983076,
      "rewards/rejected": -0.2912215353360228,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.972016217523487e+17,
      "train_loss": 0.5023879745463633,
      "train_runtime": 11027.9329,
      "train_samples_per_second": 13.44,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.972016217523487e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}