{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 16.537830352783203,
      "kl": 20.99601173400879,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36331023.31968504,
      "logits/rejected": -37604871.14418605,
      "logps/chosen": -479.68499015748034,
      "logps/rejected": -353.2168362403101,
      "loss": 0.5789,
      "loss/base_kto": 3.9304776191711426,
      "loss/total_with_kl": 4.631022930145264,
      "metrics/advantage_var": 420.7478942871094,
      "regularization/advantage_var": 420.7478942871094,
      "regularization/kl": 0.7005452513694763,
      "rewards/chosen": 0.28216278797059546,
      "rewards/margins": 0.06821970869491717,
      "rewards/rejected": 0.2139430792756783,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 15.986422538757324,
      "kl": 5.433504581451416,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36267776.0,
      "logits/rejected": -36990547.2,
      "logps/chosen": -476.61943359375,
      "logps/rejected": -412.352294921875,
      "loss": 0.5335,
      "loss/base_kto": 3.932533025741577,
      "loss/total_with_kl": 4.268146991729736,
      "metrics/advantage_var": 201.5700225830078,
      "regularization/advantage_var": 201.5700225830078,
      "regularization/kl": 0.33561408519744873,
      "rewards/chosen": -0.014302882552146911,
      "rewards/margins": 0.057395032048225395,
      "rewards/rejected": -0.07169791460037231,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 15.14694595336914,
      "kl": 4.5898332595825195,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36111591.8490566,
      "logits/rejected": -38411314.88198758,
      "logps/chosen": -487.9725825471698,
      "logps/rejected": -374.0280958850932,
      "loss": 0.5208,
      "loss/base_kto": 3.8561952114105225,
      "loss/total_with_kl": 4.166226387023926,
      "metrics/advantage_var": 186.20510864257812,
      "regularization/advantage_var": 186.20510864257812,
      "regularization/kl": 0.3100314736366272,
      "rewards/chosen": 0.0318767859500909,
      "rewards/margins": 0.15225735341782187,
      "rewards/rejected": -0.12038056746773097,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 16.574857711791992,
      "kl": 10.953143119812012,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37778135.748427674,
      "logits/rejected": -37664049.29192547,
      "logps/chosen": -503.7589426100629,
      "logps/rejected": -382.98825698757764,
      "loss": 0.5201,
      "loss/base_kto": 3.8481743335723877,
      "loss/total_with_kl": 4.160851001739502,
      "metrics/advantage_var": 187.7936553955078,
      "regularization/advantage_var": 187.7936553955078,
      "regularization/kl": 0.31267642974853516,
      "rewards/chosen": 0.19398306600702633,
      "rewards/margins": 0.13317571944978823,
      "rewards/rejected": 0.06080734655723809,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 29.886825561523438,
      "kl": 14.196418762207031,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36849322.666666664,
      "logits/rejected": -37452472.57680251,
      "logps/chosen": -506.85835280373834,
      "logps/rejected": -378.2287911442006,
      "loss": 0.5202,
      "loss/base_kto": 3.798060178756714,
      "loss/total_with_kl": 4.1617608070373535,
      "metrics/advantage_var": 218.4387664794922,
      "regularization/advantage_var": 218.4387664794922,
      "regularization/kl": 0.36370059847831726,
      "rewards/chosen": 0.2566196049485251,
      "rewards/margins": 0.1754785187408632,
      "rewards/rejected": 0.08114108620766189,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 14.523025512695312,
      "kl": 8.677205085754395,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -36201331.14064915,
      "logits/rejected": -37661542.319115326,
      "logps/chosen": -468.98000386398763,
      "logps/rejected": -370.2740422590837,
      "loss": 0.5248,
      "loss/base_kto": 3.798210859298706,
      "loss/total_with_kl": 4.1985764503479,
      "metrics/advantage_var": 240.4597930908203,
      "regularization/advantage_var": 240.4597930908203,
      "regularization/kl": 0.40036556124687195,
      "rewards/chosen": 0.13418971773645794,
      "rewards/margins": 0.18964537603291948,
      "rewards/rejected": -0.05545565829646154,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 17.636444091796875,
      "kl": 9.173022270202637,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36726727.633027524,
      "logits/rejected": -38022880.10223642,
      "logps/chosen": -479.09250764525996,
      "logps/rejected": -384.2581619408946,
      "loss": 0.5267,
      "loss/base_kto": 3.7889816761016846,
      "loss/total_with_kl": 4.213504314422607,
      "metrics/advantage_var": 254.96865844726562,
      "regularization/advantage_var": 254.96865844726562,
      "regularization/kl": 0.4245227873325348,
      "rewards/chosen": 0.1997123974908018,
      "rewards/margins": 0.19239941792973037,
      "rewards/rejected": 0.007312979561071426,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 15.215126991271973,
      "kl": 12.488289833068848,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -36265196.307692304,
      "logits/rejected": -37743466.463492066,
      "logps/chosen": -493.04855769230767,
      "logps/rejected": -360.37276785714283,
      "loss": 0.5221,
      "loss/base_kto": 3.7549118995666504,
      "loss/total_with_kl": 4.176564693450928,
      "metrics/advantage_var": 253.24485778808594,
      "regularization/advantage_var": 253.24485778808594,
      "regularization/kl": 0.421652615070343,
      "rewards/chosen": 0.23179610032301684,
      "rewards/margins": 0.23726204985081786,
      "rewards/rejected": -0.005465949527801029,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 17.90062141418457,
      "kl": 11.389983177185059,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36015610.40873635,
      "logits/rejected": -36947951.97496088,
      "logps/chosen": -525.790025351014,
      "logps/rejected": -366.84952073552427,
      "loss": 0.5205,
      "loss/base_kto": 3.7394680976867676,
      "loss/total_with_kl": 4.1636176109313965,
      "metrics/advantage_var": 254.74453735351562,
      "regularization/advantage_var": 254.74453735351562,
      "regularization/kl": 0.42414966225624084,
      "rewards/chosen": 0.2547431743461145,
      "rewards/margins": 0.23224709267897406,
      "rewards/rejected": 0.02249608166714043,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 14.606820106506348,
      "kl": 13.57861328125,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -36329533.27874016,
      "logits/rejected": -37751955.64651163,
      "logps/chosen": -460.625,
      "logps/rejected": -373.60237403100774,
      "loss": 0.5201,
      "loss/base_kto": 3.7257869243621826,
      "loss/total_with_kl": 4.161072254180908,
      "metrics/advantage_var": 261.4327392578125,
      "regularization/advantage_var": 261.4327392578125,
      "regularization/kl": 0.4352855384349823,
      "rewards/chosen": 0.25165583092396654,
      "rewards/margins": 0.2519974947281317,
      "rewards/rejected": -0.0003416638041651526,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 15.993019104003906,
      "kl": 10.934139251708984,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37423401.18611988,
      "logits/rejected": -38687883.49226006,
      "logps/chosen": -475.56269716088326,
      "logps/rejected": -378.687935371517,
      "loss": 0.5129,
      "loss/base_kto": 3.7245919704437256,
      "loss/total_with_kl": 4.102898597717285,
      "metrics/advantage_var": 227.2112274169922,
      "regularization/advantage_var": 227.2112274169922,
      "regularization/kl": 0.37830671668052673,
      "rewards/chosen": 0.23485413560355875,
      "rewards/margins": 0.2576988723936991,
      "rewards/rejected": -0.022844736790140346,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 14.208782196044922,
      "kl": 3.9187302589416504,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -35056635.0531401,
      "logits/rejected": -37595947.119878605,
      "logps/chosen": -482.7842190016103,
      "logps/rejected": -351.66445371775416,
      "loss": 0.5095,
      "loss/base_kto": 3.697678327560425,
      "loss/total_with_kl": 4.075837135314941,
      "metrics/advantage_var": 227.122314453125,
      "regularization/advantage_var": 227.122314453125,
      "regularization/kl": 0.3781586289405823,
      "rewards/chosen": 0.04967179352150639,
      "rewards/margins": 0.2854993654077687,
      "rewards/rejected": -0.23582757188626233,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 16.923664093017578,
      "kl": 5.567387580871582,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -37492409.741324924,
      "logits/rejected": -37807310.06811146,
      "logps/chosen": -484.5050276025237,
      "logps/rejected": -390.71084075077397,
      "loss": 0.5169,
      "loss/base_kto": 3.747518539428711,
      "loss/total_with_kl": 4.134842872619629,
      "metrics/advantage_var": 232.62698364257812,
      "regularization/advantage_var": 232.62698364257812,
      "regularization/kl": 0.38732391595840454,
      "rewards/chosen": 0.03686676867752797,
      "rewards/margins": 0.24184379829488478,
      "rewards/rejected": -0.2049770296173568,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 13.19844913482666,
      "kl": 10.004745483398438,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -37259758.77675841,
      "logits/rejected": -38464577.4313099,
      "logps/chosen": -492.9337729357798,
      "logps/rejected": -376.2843949680511,
      "loss": 0.5165,
      "loss/base_kto": 3.7302138805389404,
      "loss/total_with_kl": 4.13217306137085,
      "metrics/advantage_var": 241.4169921875,
      "regularization/advantage_var": 241.4169921875,
      "regularization/kl": 0.40195927023887634,
      "rewards/chosen": 0.22754428875191132,
      "rewards/margins": 0.26821775490723465,
      "rewards/rejected": -0.04067346615532336,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 10.370858192443848,
      "kl": 8.129781723022461,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -35331912.4992,
      "logits/rejected": -36473516.751145035,
      "logps/chosen": -496.1955,
      "logps/rejected": -397.9780057251908,
      "loss": 0.5201,
      "loss/base_kto": 3.7339470386505127,
      "loss/total_with_kl": 4.160952091217041,
      "metrics/advantage_var": 256.45947265625,
      "regularization/advantage_var": 256.45947265625,
      "regularization/kl": 0.4270050525665283,
      "rewards/chosen": 0.1226624755859375,
      "rewards/margins": 0.256651463166448,
      "rewards/rejected": -0.1339889875805105,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 18.85415267944336,
      "kl": 6.334065914154053,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -35705573.51724138,
      "logits/rejected": -36782858.367601246,
      "logps/chosen": -471.8168593260188,
      "logps/rejected": -366.6216413551402,
      "loss": 0.5193,
      "loss/base_kto": 3.683685302734375,
      "loss/total_with_kl": 4.1541266441345215,
      "metrics/advantage_var": 282.54736328125,
      "regularization/advantage_var": 282.54736328125,
      "regularization/kl": 0.47044140100479126,
      "rewards/chosen": 0.08399816291832998,
      "rewards/margins": 0.31805401179756476,
      "rewards/rejected": -0.2340558488792348,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 15.948773384094238,
      "kl": 14.533068656921387,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36502211.156950675,
      "logits/rejected": -37504350.271685764,
      "logps/chosen": -455.46337817638266,
      "logps/rejected": -359.12193126022913,
      "loss": 0.5115,
      "loss/base_kto": 3.6408157348632812,
      "loss/total_with_kl": 4.092252254486084,
      "metrics/advantage_var": 271.1326904296875,
      "regularization/advantage_var": 271.1326904296875,
      "regularization/kl": 0.45143595337867737,
      "rewards/chosen": 0.28905634174432454,
      "rewards/margins": 0.3318007051326608,
      "rewards/rejected": -0.04274436338833623,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 16.080219268798828,
      "kl": 13.947799682617188,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -35981587.692307696,
      "logits/rejected": -36368252.87804878,
      "logps/chosen": -483.78074919871796,
      "logps/rejected": -374.1924066310976,
      "loss": 0.5124,
      "loss/base_kto": 3.667691946029663,
      "loss/total_with_kl": 4.099027633666992,
      "metrics/advantage_var": 259.0605163574219,
      "regularization/advantage_var": 259.0605163574219,
      "regularization/kl": 0.43133577704429626,
      "rewards/chosen": 0.25421406672551083,
      "rewards/margins": 0.29900450039983467,
      "rewards/rejected": -0.04479043367432385,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 21.614364624023438,
      "kl": 12.22073745727539,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -37742965.21898928,
      "logits/rejected": -37564559.71929824,
      "logps/chosen": -482.3780627871363,
      "logps/rejected": -372.8219946172249,
      "loss": 0.5222,
      "loss/base_kto": 3.696674346923828,
      "loss/total_with_kl": 4.177842140197754,
      "metrics/advantage_var": 288.9895935058594,
      "regularization/advantage_var": 288.9895935058594,
      "regularization/kl": 0.4811677038669586,
      "rewards/chosen": 0.20185038871823555,
      "rewards/margins": 0.28784650850795535,
      "rewards/rejected": -0.08599611978971977,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 25.629762649536133,
      "kl": 5.442298412322998,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36602081.3136289,
      "logits/rejected": -38403513.03725783,
      "logps/chosen": -492.5006670771757,
      "logps/rejected": -376.49224571535024,
      "loss": 0.5199,
      "loss/base_kto": 3.730147123336792,
      "loss/total_with_kl": 4.159346580505371,
      "metrics/advantage_var": 257.77734375,
      "regularization/advantage_var": 257.77734375,
      "regularization/kl": 0.4291992783546448,
      "rewards/chosen": 0.03159951459010834,
      "rewards/margins": 0.25040941272779926,
      "rewards/rejected": -0.21880989813769094,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 15.508956909179688,
      "kl": 6.528402805328369,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -36286199.741935484,
      "logits/rejected": -36577885.60890302,
      "logps/chosen": -490.13608870967744,
      "logps/rejected": -368.38235294117646,
      "loss": 0.5175,
      "loss/base_kto": 3.636345624923706,
      "loss/total_with_kl": 4.1403422355651855,
      "metrics/advantage_var": 302.70098876953125,
      "regularization/advantage_var": 302.70098876953125,
      "regularization/kl": 0.5039971470832825,
      "rewards/chosen": 0.15539588283650155,
      "rewards/margins": 0.3653011706352148,
      "rewards/rejected": -0.20990528779871323,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 15.6985502243042,
      "kl": 8.336493492126465,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -35088697.65765766,
      "logits/rejected": -38028051.17915309,
      "logps/chosen": -507.7996433933934,
      "logps/rejected": -362.4643729641694,
      "loss": 0.5211,
      "loss/base_kto": 3.701464891433716,
      "loss/total_with_kl": 4.16871976852417,
      "metrics/advantage_var": 280.633544921875,
      "regularization/advantage_var": 280.633544921875,
      "regularization/kl": 0.4672548472881317,
      "rewards/chosen": 0.14615589863545186,
      "rewards/margins": 0.3055379266249419,
      "rewards/rejected": -0.15938202798949003,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 16.868915557861328,
      "kl": 15.688308715820312,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35149326.01610306,
      "logits/rejected": -36754408.69195751,
      "logps/chosen": -472.32352053140096,
      "logps/rejected": -353.294812215478,
      "loss": 0.5178,
      "loss/base_kto": 3.7071945667266846,
      "loss/total_with_kl": 4.142777919769287,
      "metrics/advantage_var": 261.61181640625,
      "regularization/advantage_var": 261.61181640625,
      "regularization/kl": 0.4355836510658264,
      "rewards/chosen": 0.2514052828728865,
      "rewards/margins": 0.2728425848559456,
      "rewards/rejected": -0.021437301983059084,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 11.048973083496094,
      "kl": 5.838799476623535,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36868629.40192926,
      "logits/rejected": -37682253.811550155,
      "logps/chosen": -484.14328778135047,
      "logps/rejected": -388.22701367781156,
      "loss": 0.5122,
      "loss/base_kto": 3.668853759765625,
      "loss/total_with_kl": 4.09730863571167,
      "metrics/advantage_var": 257.3302307128906,
      "regularization/advantage_var": 257.3302307128906,
      "regularization/kl": 0.42845478653907776,
      "rewards/chosen": 0.09705469309325387,
      "rewards/margins": 0.313090947605032,
      "rewards/rejected": -0.2160362545117781,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 26.0384464263916,
      "kl": 7.713786602020264,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -36236119.9752322,
      "logits/rejected": -37724476.56782334,
      "logps/chosen": -498.0817530959752,
      "logps/rejected": -356.036597988959,
      "loss": 0.5166,
      "loss/base_kto": 3.686602830886841,
      "loss/total_with_kl": 4.132496356964111,
      "metrics/advantage_var": 267.8038024902344,
      "regularization/advantage_var": 267.8038024902344,
      "regularization/kl": 0.4458933472633362,
      "rewards/chosen": 0.1319815573677559,
      "rewards/margins": 0.2882677789411614,
      "rewards/rejected": -0.15628622157340546,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 11.767626762390137,
      "kl": 10.853642463684082,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36151056.17257319,
      "logits/rejected": -36493139.980982564,
      "logps/chosen": -485.22404661016947,
      "logps/rejected": -367.3628169572108,
      "loss": 0.514,
      "loss/base_kto": 3.650493621826172,
      "loss/total_with_kl": 4.112260341644287,
      "metrics/advantage_var": 277.3372497558594,
      "regularization/advantage_var": 277.3372497558594,
      "regularization/kl": 0.4617665410041809,
      "rewards/chosen": 0.22506149597638192,
      "rewards/margins": 0.32605570885694724,
      "rewards/rejected": -0.10099421288056533,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 17.659984588623047,
      "kl": 20.0924072265625,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37527906.69879518,
      "logits/rejected": -37452075.22077922,
      "logps/chosen": -479.65342620481925,
      "logps/rejected": -357.5139255275974,
      "loss": 0.5092,
      "loss/base_kto": 3.613422155380249,
      "loss/total_with_kl": 4.073502540588379,
      "metrics/advantage_var": 276.324462890625,
      "regularization/advantage_var": 276.324462890625,
      "regularization/kl": 0.46008023619651794,
      "rewards/chosen": 0.37712696948683405,
      "rewards/margins": 0.3512136504564687,
      "rewards/rejected": 0.025913319030365388,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 17.75587272644043,
      "kl": 19.05929183959961,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36423829.853658535,
      "logits/rejected": -38204685.12820513,
      "logps/chosen": -465.6052305640244,
      "logps/rejected": -347.2238581730769,
      "loss": 0.5168,
      "loss/base_kto": 3.6852288246154785,
      "loss/total_with_kl": 4.134213447570801,
      "metrics/advantage_var": 269.6606140136719,
      "regularization/advantage_var": 269.6606140136719,
      "regularization/kl": 0.44898486137390137,
      "rewards/chosen": 0.3508343347689001,
      "rewards/margins": 0.277042836975947,
      "rewards/rejected": 0.0737914977929531,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 16.573986053466797,
      "kl": 24.6936092376709,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -38083644.23529412,
      "logits/rejected": -39725339.306666665,
      "logps/chosen": -472.7370404411765,
      "logps/rejected": -384.35723958333335,
      "loss": 0.5203,
      "loss/base_kto": 3.684603214263916,
      "loss/total_with_kl": 4.162057399749756,
      "metrics/advantage_var": 286.75927734375,
      "regularization/advantage_var": 286.75927734375,
      "regularization/kl": 0.47745418548583984,
      "rewards/chosen": 0.43507479499368107,
      "rewards/margins": 0.24651265611835554,
      "rewards/rejected": 0.18856213887532552,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 17.07406997680664,
      "kl": 24.37955665588379,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37937413.51384615,
      "logits/rejected": -37525526.755555555,
      "logps/chosen": -448.4246153846154,
      "logps/rejected": -393.0105406746032,
      "loss": 0.5137,
      "loss/base_kto": 3.678205966949463,
      "loss/total_with_kl": 4.109408855438232,
      "metrics/advantage_var": 258.98077392578125,
      "regularization/advantage_var": 258.98077392578125,
      "regularization/kl": 0.4312029778957367,
      "rewards/chosen": 0.434171142578125,
      "rewards/margins": 0.2800571792844742,
      "rewards/rejected": 0.1541139632936508,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 27.32758140563965,
      "kl": 25.332792282104492,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -36482125.283018865,
      "logits/rejected": -37889904.89440994,
      "logps/chosen": -484.06348270440253,
      "logps/rejected": -386.7630046583851,
      "loss": 0.516,
      "loss/base_kto": 3.6858062744140625,
      "loss/total_with_kl": 4.127625942230225,
      "metrics/advantage_var": 265.3571472167969,
      "regularization/advantage_var": 265.3571472167969,
      "regularization/kl": 0.4418196380138397,
      "rewards/chosen": 0.4418405976685338,
      "rewards/margins": 0.2693183549032848,
      "rewards/rejected": 0.17252224276524894,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 11.3949556350708,
      "kl": 28.55071449279785,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -36017894.77221325,
      "logits/rejected": -37295334.82602118,
      "logps/chosen": -490.53130048465266,
      "logps/rejected": -365.21683528744325,
      "loss": 0.5115,
      "loss/base_kto": 3.6491081714630127,
      "loss/total_with_kl": 4.091750621795654,
      "metrics/advantage_var": 265.85137939453125,
      "regularization/advantage_var": 265.85137939453125,
      "regularization/kl": 0.4426425099372864,
      "rewards/chosen": 0.4128284361905543,
      "rewards/margins": 0.28994236712793614,
      "rewards/rejected": 0.12288606906261819,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 15.193487167358398,
      "kl": 19.492938995361328,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -37432749.01529052,
      "logits/rejected": -38655659.7571885,
      "logps/chosen": -493.74522171253824,
      "logps/rejected": -369.65869608626195,
      "loss": 0.5145,
      "loss/base_kto": 3.6358978748321533,
      "loss/total_with_kl": 4.1160454750061035,
      "metrics/advantage_var": 288.37677001953125,
      "regularization/advantage_var": 288.37677001953125,
      "regularization/kl": 0.48014727234840393,
      "rewards/chosen": 0.37165909513421014,
      "rewards/margins": 0.32228481639230655,
      "rewards/rejected": 0.04937427874190358,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 26.08849334716797,
      "kl": 22.695865631103516,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35710002.7904,
      "logits/rejected": -37808418.784732826,
      "logps/chosen": -471.93155,
      "logps/rejected": -374.5293893129771,
      "loss": 0.5082,
      "loss/base_kto": 3.6345341205596924,
      "loss/total_with_kl": 4.0658087730407715,
      "metrics/advantage_var": 259.023681640625,
      "regularization/advantage_var": 259.023681640625,
      "regularization/kl": 0.4312744140625,
      "rewards/chosen": 0.334289599609375,
      "rewards/margins": 0.29120993510238996,
      "rewards/rejected": 0.043079664506985034,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 11.127323150634766,
      "kl": 14.99573802947998,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -35764466.095846646,
      "logits/rejected": -38377709.99388379,
      "logps/chosen": -483.01747204472844,
      "logps/rejected": -362.012996941896,
      "loss": 0.5077,
      "loss/base_kto": 3.615683078765869,
      "loss/total_with_kl": 4.061391353607178,
      "metrics/advantage_var": 267.6928405761719,
      "regularization/advantage_var": 267.6928405761719,
      "regularization/kl": 0.44570857286453247,
      "rewards/chosen": 0.30560897485897565,
      "rewards/margins": 0.3548047515154735,
      "rewards/rejected": -0.049195776656497874,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 12.544903755187988,
      "kl": 5.25020170211792,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -37378966.51664025,
      "logits/rejected": -38261032.62865948,
      "logps/chosen": -484.17670364500793,
      "logps/rejected": -372.1931818181818,
      "loss": 0.5267,
      "loss/base_kto": 3.656712770462036,
      "loss/total_with_kl": 4.213390827178955,
      "metrics/advantage_var": 334.3412170410156,
      "regularization/advantage_var": 334.3412170410156,
      "regularization/kl": 0.5566781163215637,
      "rewards/chosen": 0.03584157759338476,
      "rewards/margins": 0.32360517611334333,
      "rewards/rejected": -0.2877635985199586,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 14.217246055603027,
      "kl": 18.524065017700195,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -38396444.981132075,
      "logits/rejected": -39524243.875776395,
      "logps/chosen": -514.2330483490566,
      "logps/rejected": -352.86781832298135,
      "loss": 0.5154,
      "loss/base_kto": 3.6119332313537598,
      "loss/total_with_kl": 4.1233439445495605,
      "metrics/advantage_var": 307.15313720703125,
      "regularization/advantage_var": 307.15313720703125,
      "regularization/kl": 0.5114099383354187,
      "rewards/chosen": 0.311815729681051,
      "rewards/margins": 0.3291521204547531,
      "rewards/rejected": -0.017336390773702113,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 16.856353759765625,
      "kl": 9.3283109664917,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -38511310.596491225,
      "logits/rejected": -37825042.03369066,
      "logps/chosen": -489.4402412280702,
      "logps/rejected": -397.62035796324653,
      "loss": 0.5151,
      "loss/base_kto": 3.7072436809539795,
      "loss/total_with_kl": 4.120835781097412,
      "metrics/advantage_var": 248.4034881591797,
      "regularization/advantage_var": 248.4034881591797,
      "regularization/kl": 0.41359183192253113,
      "rewards/chosen": 0.10794661185768042,
      "rewards/margins": 0.2651011799792707,
      "rewards/rejected": -0.15715456812159026,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 17.514570236206055,
      "kl": 9.719375610351562,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -38009808.73846154,
      "logits/rejected": -38305022.36942675,
      "logps/chosen": -479.805,
      "logps/rejected": -400.1239798964968,
      "loss": 0.5051,
      "loss/base_kto": 3.418736696243286,
      "loss/total_with_kl": 4.0411505699157715,
      "metrics/advantage_var": 373.8221740722656,
      "regularization/advantage_var": 373.8221740722656,
      "regularization/kl": 0.6224138736724854,
      "rewards/chosen": 0.2725811063326322,
      "rewards/margins": 0.5847772183154273,
      "rewards/rejected": -0.31219611198279507,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 12.080039978027344,
      "kl": 15.027986526489258,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -38061731.26934984,
      "logits/rejected": -37633844.49211357,
      "logps/chosen": -465.4192143962848,
      "logps/rejected": -385.28916108044166,
      "loss": 0.5599,
      "loss/base_kto": 3.164376735687256,
      "loss/total_with_kl": 4.479323863983154,
      "metrics/advantage_var": 789.7579956054688,
      "regularization/advantage_var": 789.7579956054688,
      "regularization/kl": 1.3149470090866089,
      "rewards/chosen": 0.5711726610874613,
      "rewards/margins": 0.9308842690114805,
      "rewards/rejected": -0.35971160792401913,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 26.646738052368164,
      "kl": 12.82618236541748,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -36088989.79672131,
      "logits/rejected": -37796316.84776119,
      "logps/chosen": -490.16296106557377,
      "logps/rejected": -369.60452425373137,
      "loss": 0.5234,
      "loss/base_kto": 3.207984209060669,
      "loss/total_with_kl": 4.187113285064697,
      "metrics/advantage_var": 588.0658569335938,
      "regularization/advantage_var": 588.0658569335938,
      "regularization/kl": 0.9791294932365417,
      "rewards/chosen": 0.44803656906378075,
      "rewards/margins": 0.8352686065083936,
      "rewards/rejected": -0.3872320374446129,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 24.08795738220215,
      "kl": 11.3661470413208,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36696518.02232855,
      "logits/rejected": -37635173.92955589,
      "logps/chosen": -468.5043361244019,
      "logps/rejected": -355.8814127105666,
      "loss": 0.5215,
      "loss/base_kto": 3.134793519973755,
      "loss/total_with_kl": 4.172141075134277,
      "metrics/advantage_var": 623.0315551757812,
      "regularization/advantage_var": 623.0315551757812,
      "regularization/kl": 1.037347435951233,
      "rewards/chosen": 0.4973068602347488,
      "rewards/margins": 0.9334474493427121,
      "rewards/rejected": -0.4361405891079633,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 9.80352783203125,
      "kl": 7.617015838623047,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -34527132.90322581,
      "logits/rejected": -35678018.71515152,
      "logps/chosen": -494.403377016129,
      "logps/rejected": -370.2385890151515,
      "loss": 0.5081,
      "loss/base_kto": 3.1501028537750244,
      "loss/total_with_kl": 4.064969539642334,
      "metrics/advantage_var": 549.4691772460938,
      "regularization/advantage_var": 549.4691772460938,
      "regularization/kl": 0.9148662686347961,
      "rewards/chosen": 0.4089845718876008,
      "rewards/margins": 0.9288768540845704,
      "rewards/rejected": -0.5198922821969697,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 29.720600128173828,
      "kl": 8.63927936553955,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35373025.06344411,
      "logits/rejected": -36494829.77346278,
      "logps/chosen": -491.7433912386707,
      "logps/rejected": -386.94993932038835,
      "loss": 0.5123,
      "loss/base_kto": 3.148014783859253,
      "loss/total_with_kl": 4.098679065704346,
      "metrics/advantage_var": 570.9694213867188,
      "regularization/advantage_var": 570.9694213867188,
      "regularization/kl": 0.9506641626358032,
      "rewards/chosen": 0.5039736007275538,
      "rewards/margins": 0.9435728446817154,
      "rewards/rejected": -0.4395992439541616,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 29.414081573486328,
      "kl": 20.815481185913086,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -35595772.83950617,
      "logits/rejected": -36601716.658227846,
      "logps/chosen": -471.42158564814815,
      "logps/rejected": -394.53926028481015,
      "loss": 0.5574,
      "loss/base_kto": 3.1448068618774414,
      "loss/total_with_kl": 4.458906650543213,
      "metrics/advantage_var": 789.2491455078125,
      "regularization/advantage_var": 789.2491455078125,
      "regularization/kl": 1.314099669456482,
      "rewards/chosen": 0.6382946909209828,
      "rewards/margins": 0.9447758712327411,
      "rewards/rejected": -0.3064811803117583,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 26.01615333557129,
      "kl": 15.914934158325195,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34975544.97805642,
      "logits/rejected": -36403914.566978194,
      "logps/chosen": -472.46581112852664,
      "logps/rejected": -408.1698062694704,
      "loss": 0.5317,
      "loss/base_kto": 3.1391713619232178,
      "loss/total_with_kl": 4.253345012664795,
      "metrics/advantage_var": 669.1731567382812,
      "regularization/advantage_var": 669.1731567382812,
      "regularization/kl": 1.1141732931137085,
      "rewards/chosen": 0.5612045814251078,
      "rewards/margins": 0.9698403715280575,
      "rewards/rejected": -0.40863579010294976,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 17.740093231201172,
      "kl": 21.399534225463867,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35322617.6,
      "logits/rejected": -37008326.4,
      "logps/chosen": -469.24599609375,
      "logps/rejected": -384.849755859375,
      "loss": 0.5626,
      "loss/base_kto": 3.1882617473602295,
      "loss/total_with_kl": 4.500464916229248,
      "metrics/advantage_var": 788.1102294921875,
      "regularization/advantage_var": 788.1102294921875,
      "regularization/kl": 1.3122034072875977,
      "rewards/chosen": 0.6511719703674317,
      "rewards/margins": 0.9023769855499268,
      "rewards/rejected": -0.2512050151824951,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 22.442768096923828,
      "kl": 12.02442455291748,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36062712.0743034,
      "logits/rejected": -36736658.977917984,
      "logps/chosen": -479.61977554179566,
      "logps/rejected": -369.4289974369085,
      "loss": 0.5109,
      "loss/base_kto": 3.197706937789917,
      "loss/total_with_kl": 4.087076663970947,
      "metrics/advantage_var": 534.15576171875,
      "regularization/advantage_var": 534.15576171875,
      "regularization/kl": 0.8893694281578064,
      "rewards/chosen": 0.47495215448432665,
      "rewards/margins": 0.8866932006076508,
      "rewards/rejected": -0.41174104612332413,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 12.254045486450195,
      "kl": 25.559003829956055,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -37180858.98432602,
      "logits/rejected": -36697757.906542055,
      "logps/chosen": -469.63665752351096,
      "logps/rejected": -371.635343652648,
      "loss": 0.5212,
      "loss/base_kto": 3.090243101119995,
      "loss/total_with_kl": 4.169286727905273,
      "metrics/advantage_var": 648.07421875,
      "regularization/advantage_var": 648.07421875,
      "regularization/kl": 1.0790436267852783,
      "rewards/chosen": 0.8071287149172218,
      "rewards/margins": 0.9585992438733647,
      "rewards/rejected": -0.1514705289561429,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 19.815689086914062,
      "kl": 14.280487060546875,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -34838058.27522936,
      "logits/rejected": -34883070.36421725,
      "logps/chosen": -481.26739296636083,
      "logps/rejected": -363.7084165335463,
      "loss": 0.5205,
      "loss/base_kto": 3.2393035888671875,
      "loss/total_with_kl": 4.163845062255859,
      "metrics/advantage_var": 555.2802124023438,
      "regularization/advantage_var": 555.2802124023438,
      "regularization/kl": 0.9245415925979614,
      "rewards/chosen": 0.544169901343296,
      "rewards/margins": 0.8161835076981283,
      "rewards/rejected": -0.2720136063548323,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 19.823097229003906,
      "kl": 21.81817054748535,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -36044031.17152104,
      "logits/rejected": -38181488.918429,
      "logps/chosen": -485.4643001618123,
      "logps/rejected": -362.6658327039275,
      "loss": 0.5133,
      "loss/base_kto": 3.191328763961792,
      "loss/total_with_kl": 4.106320858001709,
      "metrics/advantage_var": 549.5450439453125,
      "regularization/advantage_var": 549.5450439453125,
      "regularization/kl": 0.9149925112724304,
      "rewards/chosen": 0.6180900894708232,
      "rewards/margins": 0.8244822220945015,
      "rewards/rejected": -0.20639213262367825,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.847859382629395,
      "kl": 13.577710151672363,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35605410.90909091,
      "logits/rejected": -37161368.323987536,
      "logps/chosen": -480.18221003134795,
      "logps/rejected": -396.22765771028037,
      "loss": 0.5425,
      "loss/base_kto": 3.2130470275878906,
      "loss/total_with_kl": 4.340249538421631,
      "metrics/advantage_var": 676.9984130859375,
      "regularization/advantage_var": 676.9984130859375,
      "regularization/kl": 1.1272023916244507,
      "rewards/chosen": 0.47794239199647337,
      "rewards/margins": 0.831569843480179,
      "rewards/rejected": -0.3536274514837057,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 10.016629219055176,
      "kl": 14.0739164352417,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -36337693.3036566,
      "logits/rejected": -37938152.40552995,
      "logps/chosen": -474.0241454689984,
      "logps/rejected": -374.08172523041475,
      "loss": 0.524,
      "loss/base_kto": 3.190122365951538,
      "loss/total_with_kl": 4.191695690155029,
      "metrics/advantage_var": 601.5457763671875,
      "regularization/advantage_var": 601.5457763671875,
      "regularization/kl": 1.0015738010406494,
      "rewards/chosen": 0.5609835820281449,
      "rewards/margins": 0.8730009261381679,
      "rewards/rejected": -0.312017344110023,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 26.99571418762207,
      "kl": 14.935075759887695,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35748702.81481481,
      "logits/rejected": -36598557.164556965,
      "logps/chosen": -491.79050925925924,
      "logps/rejected": -366.63815268987344,
      "loss": 0.5187,
      "loss/base_kto": 3.191035032272339,
      "loss/total_with_kl": 4.149347305297852,
      "metrics/advantage_var": 575.5628051757812,
      "regularization/advantage_var": 575.5628051757812,
      "regularization/kl": 0.9583120346069336,
      "rewards/chosen": 0.5541232073748553,
      "rewards/margins": 0.8421052788771546,
      "rewards/rejected": -0.28798207150229926,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 25.332813262939453,
      "kl": 19.600326538085938,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -35614053.78313253,
      "logits/rejected": -37066319.79220779,
      "logps/chosen": -474.968938253012,
      "logps/rejected": -362.3044592126623,
      "loss": 0.5097,
      "loss/base_kto": 3.144695997238159,
      "loss/total_with_kl": 4.077559947967529,
      "metrics/advantage_var": 560.2787475585938,
      "regularization/advantage_var": 560.2787475585938,
      "regularization/kl": 0.9328641891479492,
      "rewards/chosen": 0.6686081484139684,
      "rewards/margins": 0.910741945403112,
      "rewards/rejected": -0.24213379698914367,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 15.155717849731445,
      "kl": 20.27742576599121,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34903524.07272727,
      "logits/rejected": -36508767.79354839,
      "logps/chosen": -472.25364583333334,
      "logps/rejected": -385.275,
      "loss": 0.4985,
      "loss/base_kto": 3.2158966064453125,
      "loss/total_with_kl": 3.988041400909424,
      "metrics/advantage_var": 463.75030517578125,
      "regularization/advantage_var": 463.75030517578125,
      "regularization/kl": 0.7721441984176636,
      "rewards/chosen": 0.6540827433268229,
      "rewards/margins": 0.8281449676841818,
      "rewards/rejected": -0.17406222435735888,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 21.98465919494629,
      "kl": 21.62624168395996,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -35109864.95176849,
      "logits/rejected": -36197684.1337386,
      "logps/chosen": -491.34405144694534,
      "logps/rejected": -357.59213525835867,
      "loss": 0.4973,
      "loss/base_kto": 3.1464684009552,
      "loss/total_with_kl": 3.97809100151062,
      "metrics/advantage_var": 499.4730529785156,
      "regularization/advantage_var": 499.4730529785156,
      "regularization/kl": 0.8316227197647095,
      "rewards/chosen": 0.6417734323973824,
      "rewards/margins": 0.8699830004094692,
      "rewards/rejected": -0.2282095680120868,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 16.958070755004883,
      "kl": 9.069159507751465,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34802558.979266346,
      "logits/rejected": -37141838.01531394,
      "logps/chosen": -492.8044258373206,
      "logps/rejected": -385.7053024502297,
      "loss": 0.5338,
      "loss/base_kto": 3.22055983543396,
      "loss/total_with_kl": 4.270147323608398,
      "metrics/advantage_var": 630.3829345703125,
      "regularization/advantage_var": 630.3829345703125,
      "regularization/kl": 1.0495874881744385,
      "rewards/chosen": 0.3879310814767743,
      "rewards/margins": 0.8503653915720079,
      "rewards/rejected": -0.46243431009523356,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 16.89603614807129,
      "kl": 12.140111923217773,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34872773.04242424,
      "logits/rejected": -37482796.59354839,
      "logps/chosen": -496.0032196969697,
      "logps/rejected": -378.95584677419356,
      "loss": 0.533,
      "loss/base_kto": 3.2100415229797363,
      "loss/total_with_kl": 4.264197826385498,
      "metrics/advantage_var": 633.1267700195312,
      "regularization/advantage_var": 633.1267700195312,
      "regularization/kl": 1.0541560649871826,
      "rewards/chosen": 0.536898202607126,
      "rewards/margins": 0.8187301039113094,
      "rewards/rejected": -0.28183190130418345,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 16.812923431396484,
      "kl": 12.90421199798584,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34195034.546623796,
      "logits/rejected": -37324971.185410336,
      "logps/chosen": -483.4717142282958,
      "logps/rejected": -359.94849449088144,
      "loss": 0.5093,
      "loss/base_kto": 3.219403028488159,
      "loss/total_with_kl": 4.074177265167236,
      "metrics/advantage_var": 513.3780517578125,
      "regularization/advantage_var": 513.3780517578125,
      "regularization/kl": 0.8547744750976562,
      "rewards/chosen": 0.5112144249428506,
      "rewards/margins": 0.8330791472904,
      "rewards/rejected": -0.32186472234754937,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 14.058871269226074,
      "kl": 14.516571044921875,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34791809.80847724,
      "logits/rejected": -35851013.972006224,
      "logps/chosen": -458.04017857142856,
      "logps/rejected": -373.04199066874025,
      "loss": 0.5007,
      "loss/base_kto": 3.2256600856781006,
      "loss/total_with_kl": 4.005448818206787,
      "metrics/advantage_var": 468.3414001464844,
      "regularization/advantage_var": 468.3414001464844,
      "regularization/kl": 0.7797884941101074,
      "rewards/chosen": 0.474783377894427,
      "rewards/margins": 0.8102318372645669,
      "rewards/rejected": -0.33544845937013995,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 11.235255241394043,
      "kl": 17.65205955505371,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34895403.256528415,
      "logits/rejected": -35339379.58664547,
      "logps/chosen": -483.04881912442397,
      "logps/rejected": -374.9543918918919,
      "loss": 0.5281,
      "loss/base_kto": 3.202352285385132,
      "loss/total_with_kl": 4.224853038787842,
      "metrics/advantage_var": 614.1143798828125,
      "regularization/advantage_var": 614.1143798828125,
      "regularization/kl": 1.0225003957748413,
      "rewards/chosen": 0.6097218503234207,
      "rewards/margins": 0.8648844638934476,
      "rewards/rejected": -0.25516261357002684,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 16.98574447631836,
      "kl": 9.064486503601074,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -36055082.1933439,
      "logits/rejected": -36566803.328197226,
      "logps/chosen": -484.90664619651346,
      "logps/rejected": -381.66838405238826,
      "loss": 0.5263,
      "loss/base_kto": 3.227729558944702,
      "loss/total_with_kl": 4.210367202758789,
      "metrics/advantage_var": 590.1730346679688,
      "regularization/advantage_var": 590.1730346679688,
      "regularization/kl": 0.9826381802558899,
      "rewards/chosen": 0.41754561483198793,
      "rewards/margins": 0.8201956691523365,
      "rewards/rejected": -0.4026500543203486,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 20.735435485839844,
      "kl": 13.433310508728027,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34812483.2,
      "logits/rejected": -35821484.8,
      "logps/chosen": -489.5673828125,
      "logps/rejected": -380.184228515625,
      "loss": 0.5075,
      "loss/base_kto": 3.173840045928955,
      "loss/total_with_kl": 4.0600409507751465,
      "metrics/advantage_var": 532.2528076171875,
      "regularization/advantage_var": 532.2528076171875,
      "regularization/kl": 0.8862008452415466,
      "rewards/chosen": 0.5355085372924805,
      "rewards/margins": 0.8867841720581056,
      "rewards/rejected": -0.351275634765625,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 10.443603515625,
      "kl": 15.014068603515625,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -35359488.76876877,
      "logits/rejected": -36065006.488599345,
      "logps/chosen": -468.1949605855856,
      "logps/rejected": -372.7525447882736,
      "loss": 0.5377,
      "loss/base_kto": 3.2357544898986816,
      "loss/total_with_kl": 4.301290512084961,
      "metrics/advantage_var": 639.9616088867188,
      "regularization/advantage_var": 639.9616088867188,
      "regularization/kl": 1.0655361413955688,
      "rewards/chosen": 0.5401951328770176,
      "rewards/margins": 0.8020933600343618,
      "rewards/rejected": -0.26189822715734423,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 10.77260684967041,
      "kl": 20.780960083007812,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -35667223.27272727,
      "logits/rejected": -38056867.489096574,
      "logps/chosen": -479.54545454545456,
      "logps/rejected": -379.64751265576325,
      "loss": 0.5154,
      "loss/base_kto": 3.1758131980895996,
      "loss/total_with_kl": 4.122966766357422,
      "metrics/advantage_var": 568.861083984375,
      "regularization/advantage_var": 568.861083984375,
      "regularization/kl": 0.9471536874771118,
      "rewards/chosen": 0.6971722067710374,
      "rewards/margins": 0.8432453312614604,
      "rewards/rejected": -0.146073124490423,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 21.820354461669922,
      "kl": 18.302148818969727,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34120516.2403698,
      "logits/rejected": -36425189.22345483,
      "logps/chosen": -480.7200500770416,
      "logps/rejected": -358.38076961172743,
      "loss": 0.4989,
      "loss/base_kto": 3.1738641262054443,
      "loss/total_with_kl": 3.991513967514038,
      "metrics/advantage_var": 491.0809020996094,
      "regularization/advantage_var": 491.0809020996094,
      "regularization/kl": 0.8176496624946594,
      "rewards/chosen": 0.5981124619306144,
      "rewards/margins": 0.8380587577640509,
      "rewards/rejected": -0.23994629583343652,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 19.11968231201172,
      "kl": 15.336816787719727,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -35227818.131661445,
      "logits/rejected": -35232662.728971966,
      "logps/chosen": -469.18172021943576,
      "logps/rejected": -366.48568925233644,
      "loss": 0.5098,
      "loss/base_kto": 3.1704258918762207,
      "loss/total_with_kl": 4.078118801116943,
      "metrics/advantage_var": 545.1610107421875,
      "regularization/advantage_var": 545.1610107421875,
      "regularization/kl": 0.9076930284500122,
      "rewards/chosen": 0.601460567462407,
      "rewards/margins": 0.8971618528513892,
      "rewards/rejected": -0.29570128538898216,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 15.077584266662598,
      "kl": 14.563433647155762,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35686443.150729336,
      "logits/rejected": -36516939.680241324,
      "logps/chosen": -496.18096636953,
      "logps/rejected": -382.3719362745098,
      "loss": 0.5261,
      "loss/base_kto": 3.1742496490478516,
      "loss/total_with_kl": 4.2089457511901855,
      "metrics/advantage_var": 621.4389038085938,
      "regularization/advantage_var": 621.4389038085938,
      "regularization/kl": 1.0346958637237549,
      "rewards/chosen": 0.5384568967154578,
      "rewards/margins": 0.863727513525884,
      "rewards/rejected": -0.32527061681042607,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 17.804264068603516,
      "kl": 12.497262954711914,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -36677968.4338118,
      "logits/rejected": -36208094.284839205,
      "logps/chosen": -500.66487240829343,
      "logps/rejected": -386.2694056278714,
      "loss": 0.51,
      "loss/base_kto": 3.1934421062469482,
      "loss/total_with_kl": 4.079799652099609,
      "metrics/advantage_var": 532.346923828125,
      "regularization/advantage_var": 532.346923828125,
      "regularization/kl": 0.8863576054573059,
      "rewards/chosen": 0.502348295999676,
      "rewards/margins": 0.8429343288378986,
      "rewards/rejected": -0.34058603283822264,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 16.499343872070312,
      "kl": 12.454108238220215,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -34069816.756589144,
      "logits/rejected": -35322900.963779524,
      "logps/chosen": -476.69578488372093,
      "logps/rejected": -396.577312992126,
      "loss": 0.5123,
      "loss/base_kto": 3.1449525356292725,
      "loss/total_with_kl": 4.098038673400879,
      "metrics/advantage_var": 572.4239501953125,
      "regularization/advantage_var": 572.4239501953125,
      "regularization/kl": 0.953085720539093,
      "rewards/chosen": 0.44505615234375,
      "rewards/margins": 0.9144545667753445,
      "rewards/rejected": -0.46939841443159447,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 68.79667663574219,
      "kl": 19.708200454711914,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34935476.28169014,
      "logits/rejected": -34988153.41029641,
      "logps/chosen": -475.5610817683881,
      "logps/rejected": -365.95154056162244,
      "loss": 0.497,
      "loss/base_kto": 3.1570169925689697,
      "loss/total_with_kl": 3.9757080078125,
      "metrics/advantage_var": 491.70635986328125,
      "regularization/advantage_var": 491.70635986328125,
      "regularization/kl": 0.818691074848175,
      "rewards/chosen": 0.6771548753239925,
      "rewards/margins": 0.8825103246493875,
      "rewards/rejected": -0.2053554493253949,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 11.845552444458008,
      "kl": 17.667652130126953,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -35823884.8,
      "logits/rejected": -35276588.8,
      "logps/chosen": -486.55625,
      "logps/rejected": -366.40927734375,
      "loss": 0.5097,
      "loss/base_kto": 3.158928394317627,
      "loss/total_with_kl": 4.077810287475586,
      "metrics/advantage_var": 551.8809814453125,
      "regularization/advantage_var": 551.8809814453125,
      "regularization/kl": 0.9188818335533142,
      "rewards/chosen": 0.6351908683776856,
      "rewards/margins": 0.8914699792861939,
      "rewards/rejected": -0.2562791109085083,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 18.535634994506836,
      "kl": 16.133092880249023,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -35617760.19875777,
      "logits/rejected": -35217517.48427673,
      "logps/chosen": -473.39091614906835,
      "logps/rejected": -381.6313138757862,
      "loss": 0.5112,
      "loss/base_kto": 3.206963300704956,
      "loss/total_with_kl": 4.089570045471191,
      "metrics/advantage_var": 530.0939331054688,
      "regularization/advantage_var": 530.0939331054688,
      "regularization/kl": 0.8826065063476562,
      "rewards/chosen": 0.613495015209506,
      "rewards/margins": 0.8020739387528003,
      "rewards/rejected": -0.18857892354329428,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 11.974020957946777,
      "kl": 16.73115348815918,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -33763296.49230769,
      "logits/rejected": -34868994.43809524,
      "logps/chosen": -494.2183173076923,
      "logps/rejected": -364.3859623015873,
      "loss": 0.5122,
      "loss/base_kto": 3.1801657676696777,
      "loss/total_with_kl": 4.097968578338623,
      "metrics/advantage_var": 551.2330322265625,
      "regularization/advantage_var": 551.2330322265625,
      "regularization/kl": 0.9178029894828796,
      "rewards/chosen": 0.6548098520132212,
      "rewards/margins": 0.8398223787524325,
      "rewards/rejected": -0.1850125267392113,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 9.843354225158691,
      "kl": 14.801050186157227,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -36169348.49840256,
      "logits/rejected": -36987512.56269113,
      "logps/chosen": -480.01472643769966,
      "logps/rejected": -367.8846999235474,
      "loss": 0.5033,
      "loss/base_kto": 3.2000465393066406,
      "loss/total_with_kl": 4.026675224304199,
      "metrics/advantage_var": 496.4736022949219,
      "regularization/advantage_var": 496.4736022949219,
      "regularization/kl": 0.8266285061836243,
      "rewards/chosen": 0.4919905982459315,
      "rewards/margins": 0.8278221966502821,
      "rewards/rejected": -0.3358315984043506,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 13.826387405395508,
      "kl": 13.46362590789795,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -34888960.39813375,
      "logits/rejected": -35481497.1177394,
      "logps/chosen": -495.83466174183513,
      "logps/rejected": -372.1906887755102,
      "loss": 0.5309,
      "loss/base_kto": 3.240717649459839,
      "loss/total_with_kl": 4.24738883972168,
      "metrics/advantage_var": 604.6072998046875,
      "regularization/advantage_var": 604.6072998046875,
      "regularization/kl": 1.0066711902618408,
      "rewards/chosen": 0.5279172716392885,
      "rewards/margins": 0.7773599545492914,
      "rewards/rejected": -0.24944268291000293,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 10.795230865478516,
      "kl": 17.875226974487305,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34820278.46390169,
      "logits/rejected": -36204207.5661882,
      "logps/chosen": -492.9363959293395,
      "logps/rejected": -363.6246012759171,
      "loss": 0.4693,
      "loss/base_kto": 3.107238292694092,
      "loss/total_with_kl": 3.754481554031372,
      "metrics/advantage_var": 388.7345886230469,
      "regularization/advantage_var": 388.7345886230469,
      "regularization/kl": 0.6472430229187012,
      "rewards/chosen": 0.6626991002424155,
      "rewards/margins": 0.9209322025571958,
      "rewards/rejected": -0.2582331023147802,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 19.27705955505371,
      "kl": 18.832422256469727,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34594490.97072419,
      "logits/rejected": -34871091.524564184,
      "logps/chosen": -475.6154661016949,
      "logps/rejected": -404.0041848256735,
      "loss": 0.4666,
      "loss/base_kto": 3.088207721710205,
      "loss/total_with_kl": 3.73294997215271,
      "metrics/advantage_var": 387.2326354980469,
      "regularization/advantage_var": 387.2326354980469,
      "regularization/kl": 0.6447422504425049,
      "rewards/chosen": 0.660996071179218,
      "rewards/margins": 0.9134467149211702,
      "rewards/rejected": -0.25245064374195225,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 16.828039169311523,
      "kl": 18.012939453125,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33559192.89433385,
      "logits/rejected": -35118578.11802233,
      "logps/chosen": -491.3185777182236,
      "logps/rejected": -388.03668261562996,
      "loss": 0.4702,
      "loss/base_kto": 3.0965676307678223,
      "loss/total_with_kl": 3.761559247970581,
      "metrics/advantage_var": 399.39447021484375,
      "regularization/advantage_var": 399.39447021484375,
      "regularization/kl": 0.6649917960166931,
      "rewards/chosen": 0.6639900616445731,
      "rewards/margins": 0.9009370072000455,
      "rewards/rejected": -0.2369469455554725,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 13.841697692871094,
      "kl": 18.363540649414062,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -36010543.16917923,
      "logits/rejected": -36651898.56515373,
      "logps/chosen": -480.6165201005025,
      "logps/rejected": -403.83848828696927,
      "loss": 0.4719,
      "loss/base_kto": 3.1108250617980957,
      "loss/total_with_kl": 3.7754433155059814,
      "metrics/advantage_var": 399.1703796386719,
      "regularization/advantage_var": 399.1703796386719,
      "regularization/kl": 0.6646186709403992,
      "rewards/chosen": 0.6045121236063128,
      "rewards/margins": 0.9177538169270829,
      "rewards/rejected": -0.31324169332077006,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 13.886054039001465,
      "kl": 10.77209758758545,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -33384008.681388013,
      "logits/rejected": -35382937.758513935,
      "logps/chosen": -479.59261632492115,
      "logps/rejected": -364.9218024380805,
      "loss": 0.4606,
      "loss/base_kto": 3.1307339668273926,
      "loss/total_with_kl": 3.6846272945404053,
      "metrics/advantage_var": 332.6686706542969,
      "regularization/advantage_var": 332.6686706542969,
      "regularization/kl": 0.5538933277130127,
      "rewards/chosen": 0.4953169476722693,
      "rewards/margins": 0.8828692149299124,
      "rewards/rejected": -0.3875522672576432,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 12.666690826416016,
      "kl": 13.711188316345215,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -35280860.52283464,
      "logits/rejected": -34790204.72558139,
      "logps/chosen": -479.02677165354334,
      "logps/rejected": -378.6731346899225,
      "loss": 0.4647,
      "loss/base_kto": 3.136176824569702,
      "loss/total_with_kl": 3.7178847789764404,
      "metrics/advantage_var": 349.37420654296875,
      "regularization/advantage_var": 349.37420654296875,
      "regularization/kl": 0.5817080736160278,
      "rewards/chosen": 0.5767086960199311,
      "rewards/margins": 0.8669471636873851,
      "rewards/rejected": -0.290238467667454,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 8.175899505615234,
      "kl": 13.4633207321167,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -35221138.61469265,
      "logits/rejected": -36860077.729200654,
      "logps/chosen": -483.1257496251874,
      "logps/rejected": -372.60741231647637,
      "loss": 0.4736,
      "loss/base_kto": 3.147956132888794,
      "loss/total_with_kl": 3.7891335487365723,
      "metrics/advantage_var": 385.0915222167969,
      "regularization/advantage_var": 385.0915222167969,
      "regularization/kl": 0.6411773562431335,
      "rewards/chosen": 0.5833238318584848,
      "rewards/margins": 0.8691109942036367,
      "rewards/rejected": -0.2857871623451519,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 16.640430450439453,
      "kl": 21.772733688354492,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -35476442.675118856,
      "logits/rejected": -35618418.39137134,
      "logps/chosen": -477.05744849445324,
      "logps/rejected": -355.0386893297381,
      "loss": 0.4617,
      "loss/base_kto": 3.1115479469299316,
      "loss/total_with_kl": 3.6933600902557373,
      "metrics/advantage_var": 349.43682861328125,
      "regularization/advantage_var": 349.43682861328125,
      "regularization/kl": 0.58181232213974,
      "rewards/chosen": 0.6841963633493463,
      "rewards/margins": 0.8865140111804446,
      "rewards/rejected": -0.20231764783109832,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 13.235726356506348,
      "kl": 15.409103393554688,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -33728573.08634223,
      "logits/rejected": -34643827.85692068,
      "logps/chosen": -453.91768053375193,
      "logps/rejected": -356.26489599533437,
      "loss": 0.4593,
      "loss/base_kto": 3.140038251876831,
      "loss/total_with_kl": 3.6741511821746826,
      "metrics/advantage_var": 320.7887268066406,
      "regularization/advantage_var": 320.7887268066406,
      "regularization/kl": 0.5341132283210754,
      "rewards/chosen": 0.5848530001318436,
      "rewards/margins": 0.8610822415341448,
      "rewards/rejected": -0.2762292414023012,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 20.878890991210938,
      "kl": 15.422149658203125,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -33578972.57861635,
      "logits/rejected": -36763638.45962733,
      "logps/chosen": -471.4190251572327,
      "logps/rejected": -374.738887810559,
      "loss": 0.4662,
      "loss/base_kto": 3.098297119140625,
      "loss/total_with_kl": 3.729501485824585,
      "metrics/advantage_var": 379.1016540527344,
      "regularization/advantage_var": 379.1016540527344,
      "regularization/kl": 0.6312043070793152,
      "rewards/chosen": 0.5744507987544222,
      "rewards/margins": 0.9202158118864583,
      "rewards/rejected": -0.3457650131320361,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 9.892145156860352,
      "kl": 13.838292121887207,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34713857.22098569,
      "logits/rejected": -36125738.47004608,
      "logps/chosen": -491.6842209856916,
      "logps/rejected": -372.671634984639,
      "loss": 0.4732,
      "loss/base_kto": 3.0913591384887695,
      "loss/total_with_kl": 3.7855026721954346,
      "metrics/advantage_var": 416.90313720703125,
      "regularization/advantage_var": 416.90313720703125,
      "regularization/kl": 0.6941437125205994,
      "rewards/chosen": 0.6123622293881658,
      "rewards/margins": 0.9411510218708847,
      "rewards/rejected": -0.3287887924827189,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 16.5397891998291,
      "kl": 16.939855575561523,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -36120921.56037152,
      "logits/rejected": -34855603.2807571,
      "logps/chosen": -469.1806308049536,
      "logps/rejected": -391.3821963722398,
      "loss": 0.4673,
      "loss/base_kto": 3.0890920162200928,
      "loss/total_with_kl": 3.738292694091797,
      "metrics/advantage_var": 389.9102478027344,
      "regularization/advantage_var": 389.9102478027344,
      "regularization/kl": 0.6492005586624146,
      "rewards/chosen": 0.5950904113958495,
      "rewards/margins": 0.9190824456340692,
      "rewards/rejected": -0.32399203423821965,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 10.007122039794922,
      "kl": 18.840436935424805,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34272680.585365854,
      "logits/rejected": -36623041.64102564,
      "logps/chosen": -455.6271913109756,
      "logps/rejected": -372.45587940705127,
      "loss": 0.4669,
      "loss/base_kto": 3.1442363262176514,
      "loss/total_with_kl": 3.7351715564727783,
      "metrics/advantage_var": 354.916015625,
      "regularization/advantage_var": 354.916015625,
      "regularization/kl": 0.590935230255127,
      "rewards/chosen": 0.6542033683962938,
      "rewards/margins": 0.8423987681452672,
      "rewards/rejected": -0.18819539974897337,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 12.622456550598145,
      "kl": 22.887765884399414,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -36191325.09090909,
      "logits/rejected": -37970868.02580645,
      "logps/chosen": -500.6345170454546,
      "logps/rejected": -361.6321068548387,
      "loss": 0.4576,
      "loss/base_kto": 3.096804618835449,
      "loss/total_with_kl": 3.6607301235198975,
      "metrics/advantage_var": 338.69384765625,
      "regularization/advantage_var": 338.69384765625,
      "regularization/kl": 0.5639252066612244,
      "rewards/chosen": 0.7195481271454782,
      "rewards/margins": 0.874699199001577,
      "rewards/rejected": -0.15515107185609878,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 15.0021333694458,
      "kl": 23.994291305541992,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -35000614.36085627,
      "logits/rejected": -36952773.92971246,
      "logps/chosen": -498.0998184250765,
      "logps/rejected": -393.1023861821086,
      "loss": 0.4579,
      "loss/base_kto": 3.073777675628662,
      "loss/total_with_kl": 3.663127899169922,
      "metrics/advantage_var": 353.964111328125,
      "regularization/advantage_var": 353.964111328125,
      "regularization/kl": 0.5893502235412598,
      "rewards/chosen": 0.763191246476013,
      "rewards/margins": 0.9050856022255638,
      "rewards/rejected": -0.14189435574955073,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 13.450627326965332,
      "kl": 18.962284088134766,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35935625.36585366,
      "logits/rejected": -37088347.897435896,
      "logps/chosen": -487.7211318597561,
      "logps/rejected": -347.68359375,
      "loss": 0.4654,
      "loss/base_kto": 3.119527578353882,
      "loss/total_with_kl": 3.7229549884796143,
      "metrics/advantage_var": 362.41876220703125,
      "regularization/advantage_var": 362.41876220703125,
      "regularization/kl": 0.6034272909164429,
      "rewards/chosen": 0.6604757076356469,
      "rewards/margins": 0.8571033245179711,
      "rewards/rejected": -0.19662761688232422,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 12.219781875610352,
      "kl": 21.565269470214844,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -34777532.37735849,
      "logits/rejected": -36161440.596273296,
      "logps/chosen": -469.9175019654088,
      "logps/rejected": -363.8256259704969,
      "loss": 0.4677,
      "loss/base_kto": 3.0906436443328857,
      "loss/total_with_kl": 3.7416934967041016,
      "metrics/advantage_var": 391.020751953125,
      "regularization/advantage_var": 391.020751953125,
      "regularization/kl": 0.6510495543479919,
      "rewards/chosen": 0.7124719199894359,
      "rewards/margins": 0.9116966408199386,
      "rewards/rejected": -0.19922472083050272,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 13.32406234741211,
      "kl": 18.75167465209961,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35298579.2688172,
      "logits/rejected": -36003703.24960254,
      "logps/chosen": -479.2941628264209,
      "logps/rejected": -384.1168521462639,
      "loss": 0.467,
      "loss/base_kto": 3.1590993404388428,
      "loss/total_with_kl": 3.7356860637664795,
      "metrics/advantage_var": 346.2985534667969,
      "regularization/advantage_var": 346.2985534667969,
      "regularization/kl": 0.5765871405601501,
      "rewards/chosen": 0.6431738506264401,
      "rewards/margins": 0.8293609155204931,
      "rewards/rejected": -0.18618706489405307,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 6.914154529571533,
      "kl": 18.390832901000977,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -33529994.37837838,
      "logits/rejected": -36759185.094462544,
      "logps/chosen": -497.8991647897898,
      "logps/rejected": -378.38110749185665,
      "loss": 0.4656,
      "loss/base_kto": 3.1070802211761475,
      "loss/total_with_kl": 3.7246978282928467,
      "metrics/advantage_var": 370.9416198730469,
      "regularization/advantage_var": 370.9416198730469,
      "regularization/kl": 0.6176177859306335,
      "rewards/chosen": 0.6875801430092202,
      "rewards/margins": 0.9103769279475727,
      "rewards/rejected": -0.2227967849383525,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 11.438712120056152,
      "kl": 22.58893394470215,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34451230.591194965,
      "logits/rejected": -36731490.583850935,
      "logps/chosen": -459.84974449685535,
      "logps/rejected": -372.14358501552795,
      "loss": 0.4625,
      "loss/base_kto": 3.102107286453247,
      "loss/total_with_kl": 3.7000434398651123,
      "metrics/advantage_var": 359.1208801269531,
      "regularization/advantage_var": 359.1208801269531,
      "regularization/kl": 0.5979362726211548,
      "rewards/chosen": 0.7382436308470912,
      "rewards/margins": 0.9092386915486391,
      "rewards/rejected": -0.17099506070154794,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 11.214678764343262,
      "kl": 18.963571548461914,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -33975406.87898089,
      "logits/rejected": -34658417.0797546,
      "logps/chosen": -459.12291003184714,
      "logps/rejected": -355.2796922929448,
      "loss": 0.4605,
      "loss/base_kto": 3.1519150733947754,
      "loss/total_with_kl": 3.6843254566192627,
      "metrics/advantage_var": 319.7659912109375,
      "regularization/advantage_var": 319.7659912109375,
      "regularization/kl": 0.5324103832244873,
      "rewards/chosen": 0.6899675016949891,
      "rewards/margins": 0.853054457227695,
      "rewards/rejected": -0.16308695553270586,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 9.632356643676758,
      "kl": 17.70616912841797,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -36399908.45696401,
      "logits/rejected": -37554153.6349454,
      "logps/chosen": -495.7028071205008,
      "logps/rejected": -352.6200760530421,
      "loss": 0.4641,
      "loss/base_kto": 3.113957166671753,
      "loss/total_with_kl": 3.7127532958984375,
      "metrics/advantage_var": 359.63726806640625,
      "regularization/advantage_var": 359.63726806640625,
      "regularization/kl": 0.5987960696220398,
      "rewards/chosen": 0.6335635953852455,
      "rewards/margins": 0.8784684452901792,
      "rewards/rejected": -0.2449048499049337,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.035210609436035,
      "kl": 19.576251983642578,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -36141401.94594595,
      "logits/rejected": -36757673.674418606,
      "logps/chosen": -471.6283255912162,
      "logps/rejected": -411.13349382267444,
      "loss": 0.4655,
      "loss/base_kto": 3.1428756713867188,
      "loss/total_with_kl": 3.724013566970825,
      "metrics/advantage_var": 349.03204345703125,
      "regularization/advantage_var": 349.03204345703125,
      "regularization/kl": 0.5811383128166199,
      "rewards/chosen": 0.5696450826284047,
      "rewards/margins": 0.8397197052040315,
      "rewards/rejected": -0.2700746225756268,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 9.367024421691895,
      "kl": 15.989706993103027,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35538447.900621116,
      "logits/rejected": -36447689.25786164,
      "logps/chosen": -485.09297360248445,
      "logps/rejected": -375.897258254717,
      "loss": 0.4632,
      "loss/base_kto": 3.1421048641204834,
      "loss/total_with_kl": 3.7057876586914062,
      "metrics/advantage_var": 338.54840087890625,
      "regularization/advantage_var": 338.54840087890625,
      "regularization/kl": 0.563683032989502,
      "rewards/chosen": 0.567249179626844,
      "rewards/margins": 0.8488750592624449,
      "rewards/rejected": -0.28162587963560093,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 12.35507583618164,
      "kl": 16.977344512939453,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35135086.9470305,
      "logits/rejected": -36150566.57534247,
      "logps/chosen": -487.79429173354737,
      "logps/rejected": -385.1545376712329,
      "loss": 0.4572,
      "loss/base_kto": 3.122990608215332,
      "loss/total_with_kl": 3.657369375228882,
      "metrics/advantage_var": 320.9480895996094,
      "regularization/advantage_var": 320.9480895996094,
      "regularization/kl": 0.5343785285949707,
      "rewards/chosen": 0.5660365121513844,
      "rewards/margins": 0.8591012508503117,
      "rewards/rejected": -0.2930647386989274,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 24.943979263305664,
      "kl": 14.890515327453613,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -36561734.82534776,
      "logits/rejected": -36859793.99684044,
      "logps/chosen": -512.2392774343122,
      "logps/rejected": -367.7308204976303,
      "loss": 0.4735,
      "loss/base_kto": 3.0660271644592285,
      "loss/total_with_kl": 3.7877564430236816,
      "metrics/advantage_var": 433.4712829589844,
      "regularization/advantage_var": 433.4712829589844,
      "regularization/kl": 0.7217296957969666,
      "rewards/chosen": 0.6244497402373937,
      "rewards/margins": 0.9588832009249678,
      "rewards/rejected": -0.33443346068757407,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 12.352337837219238,
      "kl": 17.1584529876709,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -37212582.4,
      "logits/rejected": -36345235.2,
      "logps/chosen": -485.367578125,
      "logps/rejected": -377.096240234375,
      "loss": 0.4536,
      "loss/base_kto": 3.046826124191284,
      "loss/total_with_kl": 3.628899335861206,
      "metrics/advantage_var": 349.5934753417969,
      "regularization/advantage_var": 349.5934753417969,
      "regularization/kl": 0.5820731520652771,
      "rewards/chosen": 0.6046004295349121,
      "rewards/margins": 0.9382753372192383,
      "rewards/rejected": -0.33367490768432617,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.924802303314209,
      "kl": 16.00665855407715,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -35062070.303030305,
      "logits/rejected": -36446565.53751914,
      "logps/chosen": -488.86034688995215,
      "logps/rejected": -363.7005168453293,
      "loss": 0.4561,
      "loss/base_kto": 3.123929738998413,
      "loss/total_with_kl": 3.648754835128784,
      "metrics/advantage_var": 315.210205078125,
      "regularization/advantage_var": 315.210205078125,
      "regularization/kl": 0.5248250365257263,
      "rewards/chosen": 0.6202993195212819,
      "rewards/margins": 0.8590380995210427,
      "rewards/rejected": -0.23873877999976073,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 13.827885627746582,
      "kl": 15.445418357849121,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -33636513.514195584,
      "logits/rejected": -36219774.018575855,
      "logps/chosen": -457.5471214511041,
      "logps/rejected": -382.59904702012386,
      "loss": 0.4622,
      "loss/base_kto": 3.138892889022827,
      "loss/total_with_kl": 3.6973092555999756,
      "metrics/advantage_var": 335.38525390625,
      "regularization/advantage_var": 335.38525390625,
      "regularization/kl": 0.558416485786438,
      "rewards/chosen": 0.5851179538089019,
      "rewards/margins": 0.8563093884034017,
      "rewards/rejected": -0.2711914345944998,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 11.497026443481445,
      "kl": 15.475537300109863,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -36151808.0,
      "logits/rejected": -36169260.8,
      "logps/chosen": -468.385791015625,
      "logps/rejected": -400.343603515625,
      "loss": 0.4573,
      "loss/base_kto": 3.059650421142578,
      "loss/total_with_kl": 3.6584396362304688,
      "metrics/advantage_var": 359.6334228515625,
      "regularization/advantage_var": 359.6334228515625,
      "regularization/kl": 0.598789632320404,
      "rewards/chosen": 0.6046765804290771,
      "rewards/margins": 0.9387316703796387,
      "rewards/rejected": -0.33405508995056155,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 8.553000450134277,
      "kl": 16.97987937927246,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34956386.461538464,
      "logits/rejected": -36204500.29268292,
      "logps/chosen": -490.9650440705128,
      "logps/rejected": -376.5561880716463,
      "loss": 0.4666,
      "loss/base_kto": 3.1323304176330566,
      "loss/total_with_kl": 3.73240327835083,
      "metrics/advantage_var": 360.404052734375,
      "regularization/advantage_var": 360.404052734375,
      "regularization/kl": 0.6000728011131287,
      "rewards/chosen": 0.5717535752516526,
      "rewards/margins": 0.864862864281402,
      "rewards/rejected": -0.2931092890297494,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 19.277727127075195,
      "kl": 15.771048545837402,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -34332610.81761006,
      "logits/rejected": -36033930.335403726,
      "logps/chosen": -470.49744496855345,
      "logps/rejected": -371.26353843167703,
      "loss": 0.4706,
      "loss/base_kto": 3.1594808101654053,
      "loss/total_with_kl": 3.7644569873809814,
      "metrics/advantage_var": 363.3489685058594,
      "regularization/advantage_var": 363.3489685058594,
      "regularization/kl": 0.6049759984016418,
      "rewards/chosen": 0.5511305707055818,
      "rewards/margins": 0.851343809913353,
      "rewards/rejected": -0.30021323920777127,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 11.23862361907959,
      "kl": 15.174311637878418,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35027817.55450237,
      "logits/rejected": -35086766.49149923,
      "logps/chosen": -480.51639020537124,
      "logps/rejected": -380.3885722565688,
      "loss": 0.4619,
      "loss/base_kto": 3.0904994010925293,
      "loss/total_with_kl": 3.695472002029419,
      "metrics/advantage_var": 363.34686279296875,
      "regularization/advantage_var": 363.34686279296875,
      "regularization/kl": 0.6049725413322449,
      "rewards/chosen": 0.5956331225933057,
      "rewards/margins": 0.9028084183921248,
      "rewards/rejected": -0.3071752957988191,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 7.307109832763672,
      "kl": 13.383758544921875,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35780903.68389058,
      "logits/rejected": -34818821.76205788,
      "logps/chosen": -477.3777545592705,
      "logps/rejected": -376.16941318327974,
      "loss": 0.4591,
      "loss/base_kto": 3.1255557537078857,
      "loss/total_with_kl": 3.6724822521209717,
      "metrics/advantage_var": 328.4842224121094,
      "regularization/advantage_var": 328.4842224121094,
      "regularization/kl": 0.5469262003898621,
      "rewards/chosen": 0.6030227985787899,
      "rewards/margins": 0.8759559244580102,
      "rewards/rejected": -0.27293312587922025,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 11.22283935546875,
      "kl": 15.68100643157959,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -36057814.605965465,
      "logits/rejected": -37328243.06065319,
      "logps/chosen": -493.3335949764521,
      "logps/rejected": -364.83558514774495,
      "loss": 0.4692,
      "loss/base_kto": 3.1190693378448486,
      "loss/total_with_kl": 3.7533156871795654,
      "metrics/advantage_var": 380.9286804199219,
      "regularization/advantage_var": 380.9286804199219,
      "regularization/kl": 0.634246289730072,
      "rewards/chosen": 0.573127111797243,
      "rewards/margins": 0.8706419842518016,
      "rewards/rejected": -0.2975148724545587,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.224340438842773,
      "kl": 15.672713279724121,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -36120357.85055644,
      "logits/rejected": -36371668.35023042,
      "logps/chosen": -479.6764705882353,
      "logps/rejected": -388.6004704301075,
      "loss": 0.4564,
      "loss/base_kto": 3.077691078186035,
      "loss/total_with_kl": 3.650958299636841,
      "metrics/advantage_var": 344.30462646484375,
      "regularization/advantage_var": 344.30462646484375,
      "regularization/kl": 0.5732671618461609,
      "rewards/chosen": 0.6376804175930296,
      "rewards/margins": 0.9425419287297423,
      "rewards/rejected": -0.3048615111367127,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 9.535575866699219,
      "kl": 20.456087112426758,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35746735.686274506,
      "logits/rejected": -36416695.95209581,
      "logps/chosen": -460.29268790849676,
      "logps/rejected": -370.57218375748505,
      "loss": 0.4566,
      "loss/base_kto": 3.088155508041382,
      "loss/total_with_kl": 3.652820587158203,
      "metrics/advantage_var": 339.1382751464844,
      "regularization/advantage_var": 339.1382751464844,
      "regularization/kl": 0.5646652579307556,
      "rewards/chosen": 0.6121223798764297,
      "rewards/margins": 0.8912944434640981,
      "rewards/rejected": -0.27917206358766844,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 11.117263793945312,
      "kl": 13.020883560180664,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -35574272.0,
      "logits/rejected": -36800747.78947368,
      "logps/chosen": -501.8949032738095,
      "logps/rejected": -358.2474814967105,
      "loss": 0.4607,
      "loss/base_kto": 3.154110908508301,
      "loss/total_with_kl": 3.6857082843780518,
      "metrics/advantage_var": 319.2776794433594,
      "regularization/advantage_var": 319.2776794433594,
      "regularization/kl": 0.5315973162651062,
      "rewards/chosen": 0.5785947527204242,
      "rewards/margins": 0.8441579897600906,
      "rewards/rejected": -0.26556323703966644,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.969850461966828e+17,
      "train_loss": 0.5006774948050938,
      "train_runtime": 11026.3717,
      "train_samples_per_second": 13.442,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.969850461966828e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}