{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 22.61551284790039,
      "kl": 17.726173400878906,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37893408.429210134,
      "logits/rejected": -37338174.2134647,
      "logps/chosen": -463.83597242921013,
      "logps/rejected": -394.84208230706076,
      "loss": 0.6117,
      "loss/base_kto": 3.895165205001831,
      "metrics/advantage_var": 223.74169921875,
      "regularization/lipschitz_norm": 1031.2908935546875,
      "regularization/w1": 0.9982895851135254,
      "rewards/chosen": 0.29134313182574983,
      "rewards/margins": 0.08558140555910498,
      "rewards/rejected": 0.20576172626664485,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 29.268369674682617,
      "kl": 7.626328468322754,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36383921.94098361,
      "logits/rejected": -37250466.77014925,
      "logps/chosen": -483.99006147540985,
      "logps/rejected": -372.2102145522388,
      "loss": 0.5907,
      "loss/base_kto": 3.900822401046753,
      "metrics/advantage_var": 168.17161560058594,
      "regularization/lipschitz_norm": 852.3118286132812,
      "regularization/w1": 0.825037956237793,
      "rewards/chosen": 0.05755308807873335,
      "rewards/margins": 0.09396182844975422,
      "rewards/rejected": -0.03640874037102087,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 18.46786117553711,
      "kl": 1.1394057273864746,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36016378.31111111,
      "logits/rejected": -37364632.024615385,
      "logps/chosen": -483.00376984126984,
      "logps/rejected": -347.98759615384614,
      "loss": 0.6011,
      "loss/base_kto": 3.8704826831817627,
      "metrics/advantage_var": 204.76536560058594,
      "regularization/lipschitz_norm": 969.6908569335938,
      "regularization/w1": 0.9386608004570007,
      "rewards/chosen": -0.13712601434616817,
      "rewards/margins": 0.12312108413640394,
      "rewards/rejected": -0.2602470984825721,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 26.17717170715332,
      "kl": 6.6596598625183105,
      "learning_rate": 7.9e-07,
      "logits/chosen": -35716534.85714286,
      "logits/rejected": -38076664.91076923,
      "logps/chosen": -491.4034226190476,
      "logps/rejected": -381.05134615384617,
      "loss": 0.6097,
      "loss/base_kto": 3.9487831592559814,
      "metrics/advantage_var": 205.52835083007812,
      "regularization/lipschitz_norm": 959.7703247070312,
      "regularization/w1": 0.9290577173233032,
      "rewards/chosen": -0.05661453973679315,
      "rewards/margins": 0.0005846625806647798,
      "rewards/rejected": -0.05719920231745793,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 24.305004119873047,
      "kl": 0.8844783902168274,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35351495.380726695,
      "logits/rejected": -35581217.63214838,
      "logps/chosen": -499.3340244865719,
      "logps/rejected": -365.4772024729521,
      "loss": 0.5984,
      "loss/base_kto": 3.8583943843841553,
      "metrics/advantage_var": 189.59117126464844,
      "regularization/lipschitz_norm": 959.7283325195312,
      "regularization/w1": 0.9290170669555664,
      "rewards/chosen": -0.14001726388554625,
      "rewards/margins": 0.14358763658658474,
      "rewards/rejected": -0.283604900472131,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 25.851768493652344,
      "kl": 10.99168586730957,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35168529.878157504,
      "logits/rejected": -37160055.77594728,
      "logps/chosen": -483.59193907875186,
      "logps/rejected": -377.68441103789127,
      "loss": 0.5913,
      "loss/base_kto": 3.818002462387085,
      "metrics/advantage_var": 194.14869689941406,
      "regularization/lipschitz_norm": 942.8248291015625,
      "regularization/w1": 0.9126545190811157,
      "rewards/chosen": 0.2000261507983725,
      "rewards/margins": 0.16513429011297684,
      "rewards/rejected": 0.03489186068539565,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 18.723735809326172,
      "kl": 14.190146446228027,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -34976542.591194965,
      "logits/rejected": -36276980.86956522,
      "logps/chosen": -488.0369988207547,
      "logps/rejected": -405.663189052795,
      "loss": 0.5926,
      "loss/base_kto": 3.81254506111145,
      "metrics/advantage_var": 205.521728515625,
      "regularization/lipschitz_norm": 959.1095581054688,
      "regularization/w1": 0.928417980670929,
      "rewards/chosen": 0.2530964065647725,
      "rewards/margins": 0.1688930181848048,
      "rewards/rejected": 0.08420338837996773,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 18.48126983642578,
      "kl": 9.433762550354004,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -35836909.15337423,
      "logits/rejected": -37283781.299363054,
      "logps/chosen": -460.209643404908,
      "logps/rejected": -385.0283638535032,
      "loss": 0.5933,
      "loss/base_kto": 3.8530025482177734,
      "metrics/advantage_var": 190.32125854492188,
      "regularization/lipschitz_norm": 923.2146606445312,
      "regularization/w1": 0.8936716318130493,
      "rewards/chosen": 0.14656586442256997,
      "rewards/margins": 0.1347334965802055,
      "rewards/rejected": 0.011832367842364463,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 23.23832893371582,
      "kl": 10.999950408935547,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35727274.392282955,
      "logits/rejected": -37129882.0668693,
      "logps/chosen": -498.3453074758842,
      "logps/rejected": -370.99472834346506,
      "loss": 0.593,
      "loss/base_kto": 3.8195252418518066,
      "metrics/advantage_var": 206.8389434814453,
      "regularization/lipschitz_norm": 955.2222900390625,
      "regularization/w1": 0.9246550798416138,
      "rewards/chosen": 0.1618742866148136,
      "rewards/margins": 0.16023750031336598,
      "rewards/rejected": 0.0016367863014476278,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 20.174917221069336,
      "kl": 10.249463081359863,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -36790023.85276074,
      "logits/rejected": -36648183.847133756,
      "logps/chosen": -476.25532016871165,
      "logps/rejected": -383.9594944267516,
      "loss": 0.5926,
      "loss/base_kto": 3.8088600635528564,
      "metrics/advantage_var": 214.56057739257812,
      "regularization/lipschitz_norm": 962.4050903320312,
      "regularization/w1": 0.9316080212593079,
      "rewards/chosen": 0.14388895327328172,
      "rewards/margins": 0.1813243694766773,
      "rewards/rejected": -0.03743541620339558,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 22.677093505859375,
      "kl": 11.488913536071777,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -36652774.1952,
      "logits/rejected": -35998462.04580153,
      "logps/chosen": -489.1762,
      "logps/rejected": -355.66991889312976,
      "loss": 0.5942,
      "loss/base_kto": 3.8311614990234375,
      "metrics/advantage_var": 200.62957763671875,
      "regularization/lipschitz_norm": 952.9503784179688,
      "regularization/w1": 0.9224559664726257,
      "rewards/chosen": 0.1296946044921875,
      "rewards/margins": 0.1469469397857899,
      "rewards/rejected": -0.017252335293602396,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 25.6236572265625,
      "kl": 16.71211814880371,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -35016283.00463679,
      "logits/rejected": -36606003.766192734,
      "logps/chosen": -465.0539992272025,
      "logps/rejected": -363.51128060821486,
      "loss": 0.5856,
      "loss/base_kto": 3.722736358642578,
      "metrics/advantage_var": 248.9115447998047,
      "regularization/lipschitz_norm": 993.5855712890625,
      "regularization/w1": 0.9617907404899597,
      "rewards/chosen": 0.320264577497102,
      "rewards/margins": 0.2624493150054329,
      "rewards/rejected": 0.057815262491669135,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 17.011987686157227,
      "kl": 5.227573871612549,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36314480.38034865,
      "logits/rejected": -38333952.78890601,
      "logps/chosen": -496.1202456418383,
      "logps/rejected": -386.7652157164869,
      "loss": 0.6054,
      "loss/base_kto": 3.800288438796997,
      "metrics/advantage_var": 247.61289978027344,
      "regularization/lipschitz_norm": 1077.5584716796875,
      "regularization/w1": 1.0430763959884644,
      "rewards/chosen": 0.020326623448101353,
      "rewards/margins": 0.19650560119105326,
      "rewards/rejected": -0.1761789777429519,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 17.925739288330078,
      "kl": 5.149405002593994,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -35811164.94267516,
      "logits/rejected": -36887809.57055215,
      "logps/chosen": -484.67257165605093,
      "logps/rejected": -371.10280866564415,
      "loss": 0.5957,
      "loss/base_kto": 3.77862548828125,
      "metrics/advantage_var": 242.25364685058594,
      "regularization/lipschitz_norm": 1019.4447631835938,
      "regularization/w1": 0.9868223071098328,
      "rewards/chosen": 0.015709421437257415,
      "rewards/margins": 0.20944999702873676,
      "rewards/rejected": -0.19374057559147934,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 22.183860778808594,
      "kl": 8.57587718963623,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -37178321.52738654,
      "logits/rejected": -37794692.99219969,
      "logps/chosen": -485.09448356807513,
      "logps/rejected": -376.52296216848674,
      "loss": 0.5897,
      "loss/base_kto": 3.7043817043304443,
      "metrics/advantage_var": 265.66552734375,
      "regularization/lipschitz_norm": 1046.708984375,
      "regularization/w1": 1.0132144689559937,
      "rewards/chosen": 0.1665053986980695,
      "rewards/margins": 0.27850994510307026,
      "rewards/rejected": -0.11200454640500074,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 17.32640838623047,
      "kl": 6.009871006011963,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -35340411.003003,
      "logits/rejected": -36968414.64495114,
      "logps/chosen": -485.8840559309309,
      "logps/rejected": -349.19360749185665,
      "loss": 0.5929,
      "loss/base_kto": 3.793537139892578,
      "metrics/advantage_var": 233.20884704589844,
      "regularization/lipschitz_norm": 981.0484619140625,
      "regularization/w1": 0.9496549963951111,
      "rewards/chosen": 0.05680670752539649,
      "rewards/margins": 0.21321438996285833,
      "rewards/rejected": -0.15640768243746184,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.678268432617188,
      "kl": 9.558260917663574,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -35297317.01204819,
      "logits/rejected": -36870852.15584416,
      "logps/chosen": -477.7679781626506,
      "logps/rejected": -387.1367948457792,
      "loss": 0.5889,
      "loss/base_kto": 3.772874116897583,
      "metrics/advantage_var": 212.17727661132812,
      "regularization/lipschitz_norm": 969.6173095703125,
      "regularization/w1": 0.9385895133018494,
      "rewards/chosen": 0.19018784488540097,
      "rewards/margins": 0.20245667299256603,
      "rewards/rejected": -0.012268828107165052,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 22.296146392822266,
      "kl": 15.797822952270508,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -34978442.003189795,
      "logits/rejected": -36334581.0229709,
      "logps/chosen": -499.66417464114835,
      "logps/rejected": -354.1043501148545,
      "loss": 0.591,
      "loss/base_kto": 3.765805959701538,
      "metrics/advantage_var": 213.3668975830078,
      "regularization/lipschitz_norm": 994.2394409179688,
      "regularization/w1": 0.9624237418174744,
      "rewards/chosen": 0.2144070614658094,
      "rewards/margins": 0.19706860519072236,
      "rewards/rejected": 0.01733845627508704,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 24.641950607299805,
      "kl": 7.255638122558594,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -36362105.6512,
      "logits/rejected": -38003601.00152672,
      "logps/chosen": -495.8204,
      "logps/rejected": -375.6777671755725,
      "loss": 0.5975,
      "loss/base_kto": 3.740105390548706,
      "metrics/advantage_var": 270.93975830078125,
      "regularization/lipschitz_norm": 1074.212158203125,
      "regularization/w1": 1.0398372411727905,
      "rewards/chosen": 0.06908189086914063,
      "rewards/margins": 0.2576003393275137,
      "rewards/rejected": -0.1885184484583731,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 17.612085342407227,
      "kl": 7.0018157958984375,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -35700540.65230769,
      "logits/rejected": -37447728.76190476,
      "logps/chosen": -490.6022115384615,
      "logps/rejected": -384.22442956349204,
      "loss": 0.6001,
      "loss/base_kto": 3.771484136581421,
      "metrics/advantage_var": 257.8808288574219,
      "regularization/lipschitz_norm": 1063.3167724609375,
      "regularization/w1": 1.0292905569076538,
      "rewards/chosen": 0.1267694326547476,
      "rewards/margins": 0.2220495137830648,
      "rewards/rejected": -0.0952800811283172,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 19.3057918548584,
      "kl": 14.914984703063965,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -35668982.518518515,
      "logits/rejected": -39240788.25316456,
      "logps/chosen": -492.21387924382714,
      "logps/rejected": -355.3556912579114,
      "loss": 0.5822,
      "loss/base_kto": 3.6853816509246826,
      "metrics/advantage_var": 262.89837646484375,
      "regularization/lipschitz_norm": 1004.1189575195312,
      "regularization/w1": 0.9719871878623962,
      "rewards/chosen": 0.2619024912516276,
      "rewards/margins": 0.28822025065683615,
      "rewards/rejected": -0.02631775940520854,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 23.008037567138672,
      "kl": 11.956717491149902,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -34780769.1318328,
      "logits/rejected": -36797866.40729483,
      "logps/chosen": -477.988344051447,
      "logps/rejected": -374.3081781914894,
      "loss": 0.6018,
      "loss/base_kto": 3.7611629962921143,
      "metrics/advantage_var": 257.25885009765625,
      "regularization/lipschitz_norm": 1088.156494140625,
      "regularization/w1": 1.0533353090286255,
      "rewards/chosen": 0.11665805436407255,
      "rewards/margins": 0.23021567860122585,
      "rewards/rejected": -0.1135576242371533,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 21.199129104614258,
      "kl": 9.169313430786133,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35709703.75757576,
      "logits/rejected": -36700483.716129035,
      "logps/chosen": -460.24460227272726,
      "logps/rejected": -381.5554435483871,
      "loss": 0.598,
      "loss/base_kto": 3.760751485824585,
      "metrics/advantage_var": 256.7850036621094,
      "regularization/lipschitz_norm": 1056.8564453125,
      "regularization/w1": 1.023037075996399,
      "rewards/chosen": 0.18825216582327178,
      "rewards/margins": 0.23152568244747634,
      "rewards/rejected": -0.043273516624204575,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 27.59272575378418,
      "kl": 7.0042724609375,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -37679747.94165316,
      "logits/rejected": -37919384.132730015,
      "logps/chosen": -501.1313816855754,
      "logps/rejected": -365.96021870286575,
      "loss": 0.5998,
      "loss/base_kto": 3.720513105392456,
      "metrics/advantage_var": 290.1617126464844,
      "regularization/lipschitz_norm": 1113.8450927734375,
      "regularization/w1": 1.0782021284103394,
      "rewards/chosen": 0.09823385822908225,
      "rewards/margins": 0.27393960829315533,
      "rewards/rejected": -0.1757057500640731,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 16.752378463745117,
      "kl": 10.725379943847656,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35477635.724349156,
      "logits/rejected": -36208378.69218501,
      "logps/chosen": -494.9874617151608,
      "logps/rejected": -381.44734350079744,
      "loss": 0.5858,
      "loss/base_kto": 3.7046101093292236,
      "metrics/advantage_var": 234.78515625,
      "regularization/lipschitz_norm": 1014.1542358398438,
      "regularization/w1": 0.9817013144493103,
      "rewards/chosen": 0.1933942271958449,
      "rewards/margins": 0.2569185859665287,
      "rewards/rejected": -0.06352435877068381,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 23.219057083129883,
      "kl": 9.880836486816406,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36577298.73170732,
      "logits/rejected": -37500960.820512824,
      "logps/chosen": -478.9034870426829,
      "logps/rejected": -366.79422075320514,
      "loss": 0.5925,
      "loss/base_kto": 3.789264678955078,
      "metrics/advantage_var": 215.0338592529297,
      "regularization/lipschitz_norm": 981.8986206054688,
      "regularization/w1": 0.9504777789115906,
      "rewards/chosen": 0.09943557367092226,
      "rewards/margins": 0.1600638838989277,
      "rewards/rejected": -0.06062831022800543,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 23.475032806396484,
      "kl": 17.236083984375,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37205993.4591195,
      "logits/rejected": -38576764.02484472,
      "logps/chosen": -482.5070754716981,
      "logps/rejected": -377.8761888586956,
      "loss": 0.5985,
      "loss/base_kto": 3.7453103065490723,
      "metrics/advantage_var": 252.90396118164062,
      "regularization/lipschitz_norm": 1076.9658203125,
      "regularization/w1": 1.0425028800964355,
      "rewards/chosen": 0.296068659368551,
      "rewards/margins": 0.20513520910803323,
      "rewards/rejected": 0.09093345026051776,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 22.165006637573242,
      "kl": 11.516542434692383,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35175617.20754717,
      "logits/rejected": -37775474.48447205,
      "logps/chosen": -473.7337853773585,
      "logps/rejected": -359.1034064440994,
      "loss": 0.5915,
      "loss/base_kto": 3.7129135131835938,
      "metrics/advantage_var": 256.4697265625,
      "regularization/lipschitz_norm": 1052.5374755859375,
      "regularization/w1": 1.0188562870025635,
      "rewards/chosen": 0.25117048827357263,
      "rewards/margins": 0.255254651844569,
      "rewards/rejected": -0.004084163570996397,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 18.9310359954834,
      "kl": 12.891018867492676,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -35657514.53776435,
      "logits/rejected": -36446983.45631068,
      "logps/chosen": -482.76680513595164,
      "logps/rejected": -364.8552791262136,
      "loss": 0.5986,
      "loss/base_kto": 3.7332704067230225,
      "metrics/advantage_var": 258.4113464355469,
      "regularization/lipschitz_norm": 1090.2811279296875,
      "regularization/w1": 1.0553919076919556,
      "rewards/chosen": 0.243781421119713,
      "rewards/margins": 0.25185478036533426,
      "rewards/rejected": -0.00807335924562127,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 27.660295486450195,
      "kl": 11.09241771697998,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36485962.71770335,
      "logits/rejected": -36978882.45022971,
      "logps/chosen": -482.4744816586922,
      "logps/rejected": -372.57580398162327,
      "loss": 0.5955,
      "loss/base_kto": 3.7170052528381348,
      "metrics/advantage_var": 254.47903442382812,
      "regularization/lipschitz_norm": 1081.9442138671875,
      "regularization/w1": 1.0473219156265259,
      "rewards/chosen": 0.15547682718036657,
      "rewards/margins": 0.25378128362538,
      "rewards/rejected": -0.0983044564450134,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 23.2921085357666,
      "kl": 14.014410972595215,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -37297054.71175523,
      "logits/rejected": -37293576.54628225,
      "logps/chosen": -504.9441425120773,
      "logps/rejected": -389.2247723823976,
      "loss": 0.5945,
      "loss/base_kto": 3.727022647857666,
      "metrics/advantage_var": 236.0036163330078,
      "regularization/lipschitz_norm": 1063.3231201171875,
      "regularization/w1": 1.0292967557907104,
      "rewards/chosen": 0.19683140065167068,
      "rewards/margins": 0.24935243624692924,
      "rewards/rejected": -0.05252103559525856,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 21.310455322265625,
      "kl": 11.659140586853027,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35771240.540192924,
      "logits/rejected": -36612967.4893617,
      "logps/chosen": -479.94433279742765,
      "logps/rejected": -383.04796732522794,
      "loss": 0.5955,
      "loss/base_kto": 3.7672271728515625,
      "metrics/advantage_var": 243.11131286621094,
      "regularization/lipschitz_norm": 1029.5413818359375,
      "regularization/w1": 0.9965960383415222,
      "rewards/chosen": 0.10298527168690967,
      "rewards/margins": 0.20089206265504228,
      "rewards/rejected": -0.0979067909681326,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 38.099945068359375,
      "kl": 16.773916244506836,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36485869.26829268,
      "logits/rejected": -36950294.974358976,
      "logps/chosen": -503.9336890243902,
      "logps/rejected": -381.6227213541667,
      "loss": 0.6022,
      "loss/base_kto": 3.734379529953003,
      "metrics/advantage_var": 271.9481506347656,
      "regularization/lipschitz_norm": 1119.2958984375,
      "regularization/w1": 1.083478331565857,
      "rewards/chosen": 0.28929428937958507,
      "rewards/margins": 0.24367292177833713,
      "rewards/rejected": 0.045621367601247936,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 18.359342575073242,
      "kl": 18.409404754638672,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35800295.541401275,
      "logits/rejected": -35299252.61349693,
      "logps/chosen": -469.1358976910828,
      "logps/rejected": -364.7176715874233,
      "loss": 0.594,
      "loss/base_kto": 3.6930909156799316,
      "metrics/advantage_var": 275.6893615722656,
      "regularization/lipschitz_norm": 1094.0638427734375,
      "regularization/w1": 1.0590537786483765,
      "rewards/chosen": 0.3039243661673965,
      "rewards/margins": 0.27000463975467603,
      "rewards/rejected": 0.03391972641272047,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 37.54812240600586,
      "kl": 10.203047752380371,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -37182692.9305136,
      "logits/rejected": -37239247.94822007,
      "logps/chosen": -477.6900962990937,
      "logps/rejected": -371.96875,
      "loss": 0.603,
      "loss/base_kto": 3.7724015712738037,
      "metrics/advantage_var": 259.531494140625,
      "regularization/lipschitz_norm": 1086.3216552734375,
      "regularization/w1": 1.051559329032898,
      "rewards/chosen": 0.09012801985726256,
      "rewards/margins": 0.2202559606061501,
      "rewards/rejected": -0.13012794074888753,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 19.995777130126953,
      "kl": 19.587913513183594,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -35426880.90140845,
      "logits/rejected": -35140095.20124805,
      "logps/chosen": -492.51085680751174,
      "logps/rejected": -373.3675897035881,
      "loss": 0.5881,
      "loss/base_kto": 3.656334400177002,
      "metrics/advantage_var": 249.142822265625,
      "regularization/lipschitz_norm": 1083.2322998046875,
      "regularization/w1": 1.048568844795227,
      "rewards/chosen": 0.317853375406519,
      "rewards/margins": 0.290600606828979,
      "rewards/rejected": 0.027252768577539978,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 46.883968353271484,
      "kl": 21.90428924560547,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -35776368.50955414,
      "logits/rejected": -36817436.26993865,
      "logps/chosen": -474.05996218152865,
      "logps/rejected": -358.19500095858893,
      "loss": 0.5894,
      "loss/base_kto": 3.702869415283203,
      "metrics/advantage_var": 256.7891845703125,
      "regularization/lipschitz_norm": 1045.6673583984375,
      "regularization/w1": 1.0122060775756836,
      "rewards/chosen": 0.3643581609057773,
      "rewards/margins": 0.24737368237068091,
      "rewards/rejected": 0.11698447853509635,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 22.3043155670166,
      "kl": 13.030131340026855,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -34736035.34477379,
      "logits/rejected": -36060722.47887324,
      "logps/chosen": -508.2879290171607,
      "logps/rejected": -371.79491881846633,
      "loss": 0.5975,
      "loss/base_kto": 3.744248628616333,
      "metrics/advantage_var": 254.7359619140625,
      "regularization/lipschitz_norm": 1069.6435546875,
      "regularization/w1": 1.0354150533676147,
      "rewards/chosen": 0.151040706545254,
      "rewards/margins": 0.23194629251331933,
      "rewards/rejected": -0.08090558596806534,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 26.595840454101562,
      "kl": 11.584898948669434,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -35833795.294851795,
      "logits/rejected": -36832845.96546311,
      "logps/chosen": -489.06679017160684,
      "logps/rejected": -362.74347527472526,
      "loss": 0.6011,
      "loss/base_kto": 3.5896096229553223,
      "metrics/advantage_var": 393.7645568847656,
      "regularization/lipschitz_norm": 1259.10400390625,
      "regularization/w1": 1.2188128232955933,
      "rewards/chosen": 0.23580975465581122,
      "rewards/margins": 0.4212782996083475,
      "rewards/rejected": -0.1854685449525363,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 21.800310134887695,
      "kl": 26.010726928710938,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -34446830.67076923,
      "logits/rejected": -35252067.961904764,
      "logps/chosen": -483.19057692307695,
      "logps/rejected": -381.0619543650794,
      "loss": 0.6209,
      "loss/base_kto": 3.236591100692749,
      "metrics/advantage_var": 728.69482421875,
      "regularization/lipschitz_norm": 1787.618408203125,
      "regularization/w1": 1.730414628982544,
      "rewards/chosen": 0.7645455228365384,
      "rewards/margins": 0.8732299536401098,
      "rewards/rejected": -0.10868443080357143,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 16.8453426361084,
      "kl": 17.86311149597168,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -34530329.28395062,
      "logits/rejected": -35855583.594936706,
      "logps/chosen": -495.580680941358,
      "logps/rejected": -399.43225870253167,
      "loss": 0.62,
      "loss/base_kto": 3.245777130126953,
      "metrics/advantage_var": 709.994384765625,
      "regularization/lipschitz_norm": 1771.098876953125,
      "regularization/w1": 1.7144235372543335,
      "rewards/chosen": 0.5647619741934317,
      "rewards/margins": 0.8513128581988213,
      "rewards/rejected": -0.28655088400538964,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 23.292949676513672,
      "kl": 19.082748413085938,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -34898210.215780996,
      "logits/rejected": -35900436.20030349,
      "logps/chosen": -490.05454911433173,
      "logps/rejected": -393.5083459787557,
      "loss": 0.5966,
      "loss/base_kto": 3.2628426551818848,
      "metrics/advantage_var": 595.5111694335938,
      "regularization/lipschitz_norm": 1559.912353515625,
      "regularization/w1": 1.5099951028823853,
      "rewards/chosen": 0.5989260957628825,
      "rewards/margins": 0.7875744643552098,
      "rewards/rejected": -0.1886483685923274,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 21.66841697692871,
      "kl": 6.183696269989014,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -34671666.7904,
      "logits/rejected": -35740301.483969465,
      "logps/chosen": -487.8895,
      "logps/rejected": -390.0688454198473,
      "loss": 0.6208,
      "loss/base_kto": 3.3012702465057373,
      "metrics/advantage_var": 663.9572143554688,
      "regularization/lipschitz_norm": 1720.2760009765625,
      "regularization/w1": 1.6652271747589111,
      "rewards/chosen": 0.264857421875,
      "rewards/margins": 0.8203466722328245,
      "rewards/rejected": -0.5554892503578245,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 15.83867359161377,
      "kl": 7.2326979637146,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35209350.821316615,
      "logits/rejected": -36016210.940809965,
      "logps/chosen": -472.8581504702194,
      "logps/rejected": -364.3094820872274,
      "loss": 0.607,
      "loss/base_kto": 3.3075034618377686,
      "metrics/advantage_var": 630.6771850585938,
      "regularization/lipschitz_norm": 1599.310302734375,
      "regularization/w1": 1.548132300376892,
      "rewards/chosen": 0.2560465941234816,
      "rewards/margins": 0.7796168598488757,
      "rewards/rejected": -0.5235702657253942,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 20.087535858154297,
      "kl": 10.071452140808105,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -34956357.316923074,
      "logits/rejected": -36206019.86031746,
      "logps/chosen": -474.38697115384616,
      "logps/rejected": -383.57896825396824,
      "loss": 0.5899,
      "loss/base_kto": 3.325768232345581,
      "metrics/advantage_var": 507.9310607910156,
      "regularization/lipschitz_norm": 1439.6236572265625,
      "regularization/w1": 1.3935555219650269,
      "rewards/chosen": 0.3780939659705529,
      "rewards/margins": 0.7193564684894641,
      "rewards/rejected": -0.3412625025189112,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 13.59377670288086,
      "kl": 16.066341400146484,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -35124335.68049155,
      "logits/rejected": -36426629.901430845,
      "logps/chosen": -488.8565668202765,
      "logps/rejected": -355.8337639109698,
      "loss": 0.5978,
      "loss/base_kto": 3.2414710521698,
      "metrics/advantage_var": 649.274169921875,
      "regularization/lipschitz_norm": 1591.873046875,
      "regularization/w1": 1.5409330129623413,
      "rewards/chosen": 0.5456774546070948,
      "rewards/margins": 0.8671934339797349,
      "rewards/rejected": -0.3215159793726401,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 20.43739891052246,
      "kl": 11.587209701538086,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -33973825.933227345,
      "logits/rejected": -34953722.49462365,
      "logps/chosen": -473.96114864864865,
      "logps/rejected": -347.6049347158218,
      "loss": 0.5924,
      "loss/base_kto": 3.3000848293304443,
      "metrics/advantage_var": 534.9948120117188,
      "regularization/lipschitz_norm": 1486.8486328125,
      "regularization/w1": 1.4392694234848022,
      "rewards/chosen": 0.4382156347811506,
      "rewards/margins": 0.7501723187589252,
      "rewards/rejected": -0.3119566839777746,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 14.55308723449707,
      "kl": 7.113320827484131,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -34393926.4,
      "logits/rejected": -35935897.6,
      "logps/chosen": -437.600537109375,
      "logps/rejected": -377.1755615234375,
      "loss": 0.5929,
      "loss/base_kto": 3.3974227905273438,
      "metrics/advantage_var": 510.9998474121094,
      "regularization/lipschitz_norm": 1390.6151123046875,
      "regularization/w1": 1.3461154699325562,
      "rewards/chosen": 0.20054919719696046,
      "rewards/margins": 0.6592526197433473,
      "rewards/rejected": -0.45870342254638674,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 20.532312393188477,
      "kl": 11.9752836227417,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -33429356.070626,
      "logits/rejected": -36452704.243531205,
      "logps/chosen": -512.6451143659712,
      "logps/rejected": -390.00085616438355,
      "loss": 0.625,
      "loss/base_kto": 3.2616021633148193,
      "metrics/advantage_var": 725.3668212890625,
      "regularization/lipschitz_norm": 1795.5465087890625,
      "regularization/w1": 1.7380889654159546,
      "rewards/chosen": 0.4411593662219101,
      "rewards/margins": 0.8262716699804122,
      "rewards/rejected": -0.3851123037585022,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 20.995637893676758,
      "kl": 12.625959396362305,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -36112063.18987342,
      "logits/rejected": -34620156.83950617,
      "logps/chosen": -468.85957278481015,
      "logps/rejected": -375.61188271604937,
      "loss": 0.6247,
      "loss/base_kto": 3.288928985595703,
      "metrics/advantage_var": 1292.3441162109375,
      "regularization/lipschitz_norm": 1765.2825927734375,
      "regularization/w1": 1.7087936401367188,
      "rewards/chosen": 0.4622271573996242,
      "rewards/margins": 0.8298365582076995,
      "rewards/rejected": -0.3676094008080753,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 24.140911102294922,
      "kl": 18.27957534790039,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -34437724.175491676,
      "logits/rejected": -36863358.13893376,
      "logps/chosen": -471.59351361573374,
      "logps/rejected": -376.7125908723748,
      "loss": 0.6149,
      "loss/base_kto": 3.3231351375579834,
      "metrics/advantage_var": 665.4700317382812,
      "regularization/lipschitz_norm": 1648.6927490234375,
      "regularization/w1": 1.595934510231018,
      "rewards/chosen": 0.5793022121135117,
      "rewards/margins": 0.7244230701041279,
      "rewards/rejected": -0.14512085799061616,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 19.79603385925293,
      "kl": 26.32721519470215,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -34581654.18666667,
      "logits/rejected": -36698975.20661157,
      "logps/chosen": -482.1162962962963,
      "logps/rejected": -359.8138688016529,
      "loss": 0.6026,
      "loss/base_kto": 3.294232130050659,
      "metrics/advantage_var": 626.8693237304688,
      "regularization/lipschitz_norm": 1576.6903076171875,
      "regularization/w1": 1.5262361764907837,
      "rewards/chosen": 0.7386127387152778,
      "rewards/margins": 0.7364574899314411,
      "rewards/rejected": 0.00215524878383668,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 14.683740615844727,
      "kl": 21.47702980041504,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35653150.300455235,
      "logits/rejected": -37041433.97101449,
      "logps/chosen": -484.8400986342944,
      "logps/rejected": -381.7333685587762,
      "loss": 0.6125,
      "loss/base_kto": 3.2820115089416504,
      "metrics/advantage_var": 797.7058715820312,
      "regularization/lipschitz_norm": 1671.8375244140625,
      "regularization/w1": 1.6183385848999023,
      "rewards/chosen": 0.5844798804414834,
      "rewards/margins": 0.7289509675670558,
      "rewards/rejected": -0.1444710871255724,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 17.051183700561523,
      "kl": 9.099251747131348,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35140609.57296467,
      "logits/rejected": -37269692.84578697,
      "logps/chosen": -499.5424347158218,
      "logps/rejected": -369.0848569157393,
      "loss": 0.6076,
      "loss/base_kto": 3.3209598064422607,
      "metrics/advantage_var": 622.4037475585938,
      "regularization/lipschitz_norm": 1590.6639404296875,
      "regularization/w1": 1.5397626161575317,
      "rewards/chosen": 0.3732815462689612,
      "rewards/margins": 0.7491195182094794,
      "rewards/rejected": -0.37583797194051816,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 19.764598846435547,
      "kl": 11.906905174255371,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34825108.54320987,
      "logits/rejected": -35921884.35443038,
      "logps/chosen": -479.0851658950617,
      "logps/rejected": -358.64240506329116,
      "loss": 0.594,
      "loss/base_kto": 3.2469890117645264,
      "metrics/advantage_var": 569.92919921875,
      "regularization/lipschitz_norm": 1555.1317138671875,
      "regularization/w1": 1.5053675174713135,
      "rewards/chosen": 0.48613465862509647,
      "rewards/margins": 0.8398204963828348,
      "rewards/rejected": -0.3536858377577383,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 24.98237419128418,
      "kl": 15.6842679977417,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35456782.86970173,
      "logits/rejected": -36521948.96423017,
      "logps/chosen": -471.10105965463106,
      "logps/rejected": -370.63224144634523,
      "loss": 0.6022,
      "loss/base_kto": 3.241093397140503,
      "metrics/advantage_var": 653.30615234375,
      "regularization/lipschitz_norm": 1628.729736328125,
      "regularization/w1": 1.5766104459762573,
      "rewards/chosen": 0.49397677866218603,
      "rewards/margins": 0.8182870576659405,
      "rewards/rejected": -0.32431027900375436,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 18.70745849609375,
      "kl": 15.941064834594727,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34274962.62910798,
      "logits/rejected": -36485570.496099845,
      "logps/chosen": -497.79367175273865,
      "logps/rejected": -393.05465093603743,
      "loss": 0.6131,
      "loss/base_kto": 3.320526123046875,
      "metrics/advantage_var": 663.6347045898438,
      "regularization/lipschitz_norm": 1636.6031494140625,
      "regularization/w1": 1.584231972694397,
      "rewards/chosen": 0.5142061952880478,
      "rewards/margins": 0.7003573754661143,
      "rewards/rejected": -0.1861511801780665,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 19.632219314575195,
      "kl": 13.42111873626709,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -32557049.756097563,
      "logits/rejected": -36331864.615384616,
      "logps/chosen": -475.3487995426829,
      "logps/rejected": -369.18704927884613,
      "loss": 0.587,
      "loss/base_kto": 3.300513982772827,
      "metrics/advantage_var": 532.9873657226562,
      "regularization/lipschitz_norm": 1441.3602294921875,
      "regularization/w1": 1.395236611366272,
      "rewards/chosen": 0.4872467227098418,
      "rewards/margins": 0.7677278166789423,
      "rewards/rejected": -0.28048109396910054,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 20.784940719604492,
      "kl": 21.042142868041992,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -35007650.69158878,
      "logits/rejected": -35474810.78369906,
      "logps/chosen": -462.86156542056074,
      "logps/rejected": -369.96017829153607,
      "loss": 0.5948,
      "loss/base_kto": 3.2664496898651123,
      "metrics/advantage_var": 625.43603515625,
      "regularization/lipschitz_norm": 1540.9881591796875,
      "regularization/w1": 1.4916763305664062,
      "rewards/chosen": 0.6386737764067367,
      "rewards/margins": 0.8052515610257243,
      "rewards/rejected": -0.16657778461898756,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 17.32353973388672,
      "kl": 11.49404239654541,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -35463141.44570503,
      "logits/rejected": -36403119.686274506,
      "logps/chosen": -472.8184764991896,
      "logps/rejected": -352.13042043740575,
      "loss": 0.5946,
      "loss/base_kto": 3.3386070728302,
      "metrics/advantage_var": 538.4461059570312,
      "regularization/lipschitz_norm": 1464.8402099609375,
      "regularization/w1": 1.417965292930603,
      "rewards/chosen": 0.386352267025742,
      "rewards/margins": 0.7313668823219807,
      "rewards/rejected": -0.34501461529623867,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 19.162174224853516,
      "kl": 17.63032341003418,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -35381686.85714286,
      "logits/rejected": -37131623.18769231,
      "logps/chosen": -499.12628968253966,
      "logps/rejected": -385.215,
      "loss": 0.5932,
      "loss/base_kto": 3.298352003097534,
      "metrics/advantage_var": 577.2579956054688,
      "regularization/lipschitz_norm": 1494.992431640625,
      "regularization/w1": 1.4471524953842163,
      "rewards/chosen": 0.4551373678540427,
      "rewards/margins": 0.7502896740439465,
      "rewards/rejected": -0.29515230618990385,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 17.638160705566406,
      "kl": 15.187868118286133,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -35187872.62745098,
      "logits/rejected": -36121566.275449105,
      "logps/chosen": -486.2753267973856,
      "logps/rejected": -395.97913547904193,
      "loss": 0.6129,
      "loss/base_kto": 3.2702064514160156,
      "metrics/advantage_var": 643.1571044921875,
      "regularization/lipschitz_norm": 1686.6243896484375,
      "regularization/w1": 1.6326522827148438,
      "rewards/chosen": 0.42208243974673204,
      "rewards/margins": 0.7818797061757881,
      "rewards/rejected": -0.35979726642905596,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 21.01573944091797,
      "kl": 9.487359046936035,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -34373709.04075235,
      "logits/rejected": -35589758.00623053,
      "logps/chosen": -497.33140673981194,
      "logps/rejected": -375.721281152648,
      "loss": 0.6018,
      "loss/base_kto": 3.2764053344726562,
      "metrics/advantage_var": 580.426513671875,
      "regularization/lipschitz_norm": 1588.6661376953125,
      "regularization/w1": 1.537828803062439,
      "rewards/chosen": 0.41636753381232855,
      "rewards/margins": 0.7823655137578114,
      "rewards/rejected": -0.3659979799454829,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 21.156766891479492,
      "kl": 9.334273338317871,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34830602.85086342,
      "logits/rejected": -36073401.928460345,
      "logps/chosen": -499.9333791208791,
      "logps/rejected": -372.42493681959564,
      "loss": 0.6191,
      "loss/base_kto": 3.3611772060394287,
      "metrics/advantage_var": 629.4342041015625,
      "regularization/lipschitz_norm": 1644.3922119140625,
      "regularization/w1": 1.5917714834213257,
      "rewards/chosen": 0.3352594061212225,
      "rewards/margins": 0.6898744945054445,
      "rewards/rejected": -0.3546150883842219,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 23.42906951904297,
      "kl": 13.179911613464355,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -36929257.87654321,
      "logits/rejected": -36407548.75949367,
      "logps/chosen": -481.268856095679,
      "logps/rejected": -380.07048556170884,
      "loss": 0.6013,
      "loss/base_kto": 3.306577444076538,
      "metrics/advantage_var": 570.9453735351562,
      "regularization/lipschitz_norm": 1553.357421875,
      "regularization/w1": 1.5036499500274658,
      "rewards/chosen": 0.46391852108048803,
      "rewards/margins": 0.7183936438461378,
      "rewards/rejected": -0.2544751227656497,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 17.261743545532227,
      "kl": 11.59704875946045,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34884243.2,
      "logits/rejected": -35944140.8,
      "logps/chosen": -509.5953125,
      "logps/rejected": -363.245556640625,
      "loss": 0.6085,
      "loss/base_kto": 3.3158957958221436,
      "metrics/advantage_var": 582.8440551757812,
      "regularization/lipschitz_norm": 1603.8192138671875,
      "regularization/w1": 1.5524970293045044,
      "rewards/chosen": 0.38999156951904296,
      "rewards/margins": 0.732364273071289,
      "rewards/rejected": -0.3423727035522461,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 21.07180404663086,
      "kl": 11.0244140625,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -33544349.02446982,
      "logits/rejected": -34929822.12893553,
      "logps/chosen": -464.3866231647635,
      "logps/rejected": -372.451813155922,
      "loss": 0.6008,
      "loss/base_kto": 3.3070545196533203,
      "metrics/advantage_var": 546.11376953125,
      "regularization/lipschitz_norm": 1548.9422607421875,
      "regularization/w1": 1.4993761777877808,
      "rewards/chosen": 0.33109727769263614,
      "rewards/margins": 0.71784746034057,
      "rewards/rejected": -0.38675018264793387,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 19.116294860839844,
      "kl": 9.207300186157227,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -34389936.49689441,
      "logits/rejected": -35984287.39622641,
      "logps/chosen": -463.06327639751555,
      "logps/rejected": -408.23786360062894,
      "loss": 0.6013,
      "loss/base_kto": 3.3536152839660645,
      "metrics/advantage_var": 572.0338134765625,
      "regularization/lipschitz_norm": 1504.552978515625,
      "regularization/w1": 1.4564071893692017,
      "rewards/chosen": 0.3255089943453392,
      "rewards/margins": 0.6919748927751619,
      "rewards/rejected": -0.36646589842982263,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 21.031543731689453,
      "kl": 10.517437934875488,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35247339.80032206,
      "logits/rejected": -36800151.502276175,
      "logps/chosen": -486.52143719806764,
      "logps/rejected": -394.4023852427921,
      "loss": 0.6167,
      "loss/base_kto": 3.3329575061798096,
      "metrics/advantage_var": 612.3126831054688,
      "regularization/lipschitz_norm": 1653.721923828125,
      "regularization/w1": 1.6008027791976929,
      "rewards/chosen": 0.3014136174642713,
      "rewards/margins": 0.7257554473112222,
      "rewards/rejected": -0.42434182984695085,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 16.131656646728516,
      "kl": 6.698751926422119,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34359353.69014084,
      "logits/rejected": -34682828.8798752,
      "logps/chosen": -467.41070031298904,
      "logps/rejected": -363.97235764430576,
      "loss": 0.6061,
      "loss/base_kto": 3.352490186691284,
      "metrics/advantage_var": 574.9630737304688,
      "regularization/lipschitz_norm": 1545.9027099609375,
      "regularization/w1": 1.4964336156845093,
      "rewards/chosen": 0.28778023637702954,
      "rewards/margins": 0.7440857298077043,
      "rewards/rejected": -0.4563054934306747,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 21.114540100097656,
      "kl": 8.279240608215332,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -34537630.55483871,
      "logits/rejected": -36046376.339393936,
      "logps/chosen": -488.0250504032258,
      "logps/rejected": -374.4557765151515,
      "loss": 0.6054,
      "loss/base_kto": 3.2804977893829346,
      "metrics/advantage_var": 591.7890014648438,
      "regularization/lipschitz_norm": 1614.3330078125,
      "regularization/w1": 1.5626744031906128,
      "rewards/chosen": 0.31956624677104334,
      "rewards/margins": 0.7858509813236001,
      "rewards/rejected": -0.4662847345525568,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 19.38349723815918,
      "kl": 11.979022026062012,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -36146015.61445783,
      "logits/rejected": -35302725.81818182,
      "logps/chosen": -488.31988893072287,
      "logps/rejected": -385.71996753246754,
      "loss": 0.5986,
      "loss/base_kto": 3.2539162635803223,
      "metrics/advantage_var": 611.529052734375,
      "regularization/lipschitz_norm": 1586.0054931640625,
      "regularization/w1": 1.5352532863616943,
      "rewards/chosen": 0.4698529530720538,
      "rewards/margins": 0.8464411799849114,
      "rewards/rejected": -0.37658822691285765,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 20.861276626586914,
      "kl": 12.396482467651367,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34633788.89514867,
      "logits/rejected": -36115243.53198128,
      "logps/chosen": -485.30545774647885,
      "logps/rejected": -367.8154494929797,
      "loss": 0.5911,
      "loss/base_kto": 3.289091110229492,
      "metrics/advantage_var": 562.3770141601562,
      "regularization/lipschitz_norm": 1487.4207763671875,
      "regularization/w1": 1.4398232698440552,
      "rewards/chosen": 0.4437481278731416,
      "rewards/margins": 0.7884234151287725,
      "rewards/rejected": -0.34467528725563085,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 19.111433029174805,
      "kl": 11.73603343963623,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -36290983.294832826,
      "logits/rejected": -36096991.073954985,
      "logps/chosen": -479.78590425531917,
      "logps/rejected": -377.9596563504823,
      "loss": 0.6152,
      "loss/base_kto": 3.2357006072998047,
      "metrics/advantage_var": 659.8549194335938,
      "regularization/lipschitz_norm": 1741.893798828125,
      "regularization/w1": 1.6861534118652344,
      "rewards/chosen": 0.4998804341695954,
      "rewards/margins": 0.8565492805698165,
      "rewards/rejected": -0.35666884640022106,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 18.78829002380371,
      "kl": 9.772809982299805,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35879267.26530612,
      "logits/rejected": -36516653.87283237,
      "logps/chosen": -465.80963010204084,
      "logps/rejected": -378.1799132947977,
      "loss": 0.6062,
      "loss/base_kto": 3.3241653442382812,
      "metrics/advantage_var": 555.0695190429688,
      "regularization/lipschitz_norm": 1576.09375,
      "regularization/w1": 1.5256588459014893,
      "rewards/chosen": 0.33075140609222203,
      "rewards/margins": 0.7227623539646706,
      "rewards/rejected": -0.3920109478724485,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 18.655101776123047,
      "kl": 6.049205303192139,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -35240254.43902439,
      "logits/rejected": -36435193.43589743,
      "logps/chosen": -498.1463414634146,
      "logps/rejected": -359.73452524038464,
      "loss": 0.605,
      "loss/base_kto": 3.335129499435425,
      "metrics/advantage_var": 560.3248291015625,
      "regularization/lipschitz_norm": 1554.9830322265625,
      "regularization/w1": 1.5052235126495361,
      "rewards/chosen": 0.23430573068014005,
      "rewards/margins": 0.7539112205576941,
      "rewards/rejected": -0.5196054898775541,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 20.932680130004883,
      "kl": 11.282088279724121,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -34745622.72189349,
      "logits/rejected": -36486503.41721854,
      "logps/chosen": -485.5318047337278,
      "logps/rejected": -370.7444639900662,
      "loss": 0.6061,
      "loss/base_kto": 3.3267104625701904,
      "metrics/advantage_var": 577.7645874023438,
      "regularization/lipschitz_norm": 1572.5972900390625,
      "regularization/w1": 1.522274136543274,
      "rewards/chosen": 0.44450044349805845,
      "rewards/margins": 0.7331363037138077,
      "rewards/rejected": -0.28863586021574916,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 15.270308494567871,
      "kl": 17.181119918823242,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34989366.76616915,
      "logits/rejected": -36915457.8962963,
      "logps/chosen": -481.052964344942,
      "logps/rejected": -360.38916666666665,
      "loss": 0.5617,
      "loss/base_kto": 3.1734278202056885,
      "metrics/advantage_var": 480.20709228515625,
      "regularization/lipschitz_norm": 1363.9676513671875,
      "regularization/w1": 1.3203204870224,
      "rewards/chosen": 0.5815272291699057,
      "rewards/margins": 0.8491762905848363,
      "rewards/rejected": -0.26764906141493056,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 20.278337478637695,
      "kl": 8.590179443359375,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34879850.76972625,
      "logits/rejected": -35662734.56752656,
      "logps/chosen": -465.47735507246375,
      "logps/rejected": -375.18138277693475,
      "loss": 0.5686,
      "loss/base_kto": 3.1985442638397217,
      "metrics/advantage_var": 473.2780456542969,
      "regularization/lipschitz_norm": 1394.8795166015625,
      "regularization/w1": 1.3502434492111206,
      "rewards/chosen": 0.420198204052813,
      "rewards/margins": 0.849558670461434,
      "rewards/rejected": -0.429360466408621,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 23.854448318481445,
      "kl": 7.3163347244262695,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33588370.75159235,
      "logits/rejected": -36218898.84662577,
      "logps/chosen": -458.65898686305735,
      "logps/rejected": -380.2924175613497,
      "loss": 0.5549,
      "loss/base_kto": 3.1283676624298096,
      "metrics/advantage_var": 464.0574645996094,
      "regularization/lipschitz_norm": 1354.0435791015625,
      "regularization/w1": 1.3107141256332397,
      "rewards/chosen": 0.4381056378601463,
      "rewards/margins": 0.9256654654189796,
      "rewards/rejected": -0.48755982755883337,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 12.578923225402832,
      "kl": 8.453057289123535,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34610426.45201238,
      "logits/rejected": -36772676.643533126,
      "logps/chosen": -484.20646284829724,
      "logps/rejected": -378.3252661671924,
      "loss": 0.5595,
      "loss/base_kto": 3.1872541904449463,
      "metrics/advantage_var": 466.6294860839844,
      "regularization/lipschitz_norm": 1331.5301513671875,
      "regularization/w1": 1.2889211177825928,
      "rewards/chosen": 0.4097718041009578,
      "rewards/margins": 0.8629459980518155,
      "rewards/rejected": -0.4531741939508577,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 15.493986129760742,
      "kl": 11.95250415802002,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -33701711.682119206,
      "logits/rejected": -35296637.72781065,
      "logps/chosen": -451.88907284768214,
      "logps/rejected": -374.33089866863907,
      "loss": 0.5547,
      "loss/base_kto": 3.1352806091308594,
      "metrics/advantage_var": 467.86309814453125,
      "regularization/lipschitz_norm": 1345.5999755859375,
      "regularization/w1": 1.3025407791137695,
      "rewards/chosen": 0.4550409885431757,
      "rewards/margins": 0.9068607752018296,
      "rewards/rejected": -0.45181978665865385,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 16.363561630249023,
      "kl": 9.333621978759766,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -34649361.47181009,
      "logits/rejected": -36363936.52805281,
      "logps/chosen": -468.34643916913944,
      "logps/rejected": -372.58281765676566,
      "loss": 0.5619,
      "loss/base_kto": 3.2002129554748535,
      "metrics/advantage_var": 480.2209167480469,
      "regularization/lipschitz_norm": 1338.0262451171875,
      "regularization/w1": 1.2952094078063965,
      "rewards/chosen": 0.42227883720963927,
      "rewards/margins": 0.8857319223252798,
      "rewards/rejected": -0.4634530851156405,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 18.24790382385254,
      "kl": 17.03498649597168,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34141521.245508984,
      "logits/rejected": -35395389.90849673,
      "logps/chosen": -469.58495508982037,
      "logps/rejected": -356.304993872549,
      "loss": 0.5517,
      "loss/base_kto": 3.1420974731445312,
      "metrics/advantage_var": 436.03887939453125,
      "regularization/lipschitz_norm": 1313.9178466796875,
      "regularization/w1": 1.2718724012374878,
      "rewards/chosen": 0.5971917706335376,
      "rewards/margins": 0.8868393404769046,
      "rewards/rejected": -0.28964756984336704,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 17.500394821166992,
      "kl": 20.996641159057617,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34774957.70015456,
      "logits/rejected": -34581460.32227488,
      "logps/chosen": -474.52347372488407,
      "logps/rejected": -366.91409952606637,
      "loss": 0.5553,
      "loss/base_kto": 3.163972854614258,
      "metrics/advantage_var": 431.9277038574219,
      "regularization/lipschitz_norm": 1320.319091796875,
      "regularization/w1": 1.2780689001083374,
      "rewards/chosen": 0.6257666185429144,
      "rewards/margins": 0.8554037524862521,
      "rewards/rejected": -0.2296371339433378,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 14.845892906188965,
      "kl": 17.076929092407227,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34378566.70476191,
      "logits/rejected": -36923070.62153846,
      "logps/chosen": -470.5708829365079,
      "logps/rejected": -351.86776442307695,
      "loss": 0.563,
      "loss/base_kto": 3.1664352416992188,
      "metrics/advantage_var": 471.05078125,
      "regularization/lipschitz_norm": 1381.47412109375,
      "regularization/w1": 1.3372669219970703,
      "rewards/chosen": 0.6034041147383432,
      "rewards/margins": 0.8736330435244009,
      "rewards/rejected": -0.2702289287860577,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 19.282777786254883,
      "kl": 14.259164810180664,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -35830581.68996961,
      "logits/rejected": -36015515.5755627,
      "logps/chosen": -510.91660334346506,
      "logps/rejected": -366.6775020096463,
      "loss": 0.5712,
      "loss/base_kto": 3.2258782386779785,
      "metrics/advantage_var": 456.6734313964844,
      "regularization/lipschitz_norm": 1387.8778076171875,
      "regularization/w1": 1.3434656858444214,
      "rewards/chosen": 0.5217259183843085,
      "rewards/margins": 0.8170433198409881,
      "rewards/rejected": -0.29531740145667956,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 17.025760650634766,
      "kl": 18.5394287109375,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -35158551.86766917,
      "logits/rejected": -36200288.15609756,
      "logps/chosen": -478.14699248120303,
      "logps/rejected": -369.25307418699185,
      "loss": 0.5508,
      "loss/base_kto": 3.1588451862335205,
      "metrics/advantage_var": 416.6886901855469,
      "regularization/lipschitz_norm": 1289.041015625,
      "regularization/w1": 1.2477917671203613,
      "rewards/chosen": 0.6125300127760809,
      "rewards/margins": 0.8588316544729178,
      "rewards/rejected": -0.24630164169683688,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 16.39432716369629,
      "kl": 23.536481857299805,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34092204.548031494,
      "logits/rejected": -36183892.53953488,
      "logps/chosen": -484.46929133858265,
      "logps/rejected": -369.04161821705424,
      "loss": 0.5592,
      "loss/base_kto": 3.124847412109375,
      "metrics/advantage_var": 508.1956481933594,
      "regularization/lipschitz_norm": 1393.3729248046875,
      "regularization/w1": 1.3487849235534668,
      "rewards/chosen": 0.7038074403297244,
      "rewards/margins": 0.9230517777812748,
      "rewards/rejected": -0.2192443374515504,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 17.85162353515625,
      "kl": 22.998136520385742,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -35762218.666666664,
      "logits/rejected": -36882403.90243903,
      "logps/chosen": -470.07882612179486,
      "logps/rejected": -387.3581364329268,
      "loss": 0.56,
      "loss/base_kto": 3.2242088317871094,
      "metrics/advantage_var": 440.51593017578125,
      "regularization/lipschitz_norm": 1297.2030029296875,
      "regularization/w1": 1.255692481994629,
      "rewards/chosen": 0.6222702417618189,
      "rewards/margins": 0.7873438109898283,
      "rewards/rejected": -0.16507356922800948,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 25.382761001586914,
      "kl": 19.590219497680664,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -33445353.8816,
      "logits/rejected": -35821396.03053435,
      "logps/chosen": -490.8473,
      "logps/rejected": -381.35605916030534,
      "loss": 0.5564,
      "loss/base_kto": 3.1736061573028564,
      "metrics/advantage_var": 415.2522888183594,
      "regularization/lipschitz_norm": 1320.162841796875,
      "regularization/w1": 1.2779176235198975,
      "rewards/chosen": 0.583317578125,
      "rewards/margins": 0.838374270932729,
      "rewards/rejected": -0.255056692807729,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 15.28670883178711,
      "kl": 16.164997100830078,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -34706287.50920245,
      "logits/rejected": -36549312.40764331,
      "logps/chosen": -481.28513228527606,
      "logps/rejected": -390.67384056528664,
      "loss": 0.5569,
      "loss/base_kto": 3.1864078044891357,
      "metrics/advantage_var": 432.43316650390625,
      "regularization/lipschitz_norm": 1310.327392578125,
      "regularization/w1": 1.2683968544006348,
      "rewards/chosen": 0.5721272661642063,
      "rewards/margins": 0.8397347910603299,
      "rewards/rejected": -0.2676075248961236,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 16.13686752319336,
      "kl": 20.435731887817383,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35811775.61676647,
      "logits/rejected": -36197038.013071895,
      "logps/chosen": -462.16991017964074,
      "logps/rejected": -395.7742800245098,
      "loss": 0.5632,
      "loss/base_kto": 3.184861183166504,
      "metrics/advantage_var": 480.7662048339844,
      "regularization/lipschitz_norm": 1364.7376708984375,
      "regularization/w1": 1.3210660219192505,
      "rewards/chosen": 0.6134483200347353,
      "rewards/margins": 0.8577557944079481,
      "rewards/rejected": -0.24430747437321282,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 24.212142944335938,
      "kl": 20.278656005859375,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -35962473.5751938,
      "logits/rejected": -35862179.67874016,
      "logps/chosen": -492.68023255813955,
      "logps/rejected": -361.7607775590551,
      "loss": 0.5593,
      "loss/base_kto": 3.1684212684631348,
      "metrics/advantage_var": 451.9801940917969,
      "regularization/lipschitz_norm": 1348.925537109375,
      "regularization/w1": 1.3057597875595093,
      "rewards/chosen": 0.6777543415394864,
      "rewards/margins": 0.866278002034073,
      "rewards/rejected": -0.1885236604945866,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 17.289684295654297,
      "kl": 18.455638885498047,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35082504.85413534,
      "logits/rejected": -35238290.939837396,
      "logps/chosen": -487.91343984962407,
      "logps/rejected": -394.54552845528457,
      "loss": 0.5683,
      "loss/base_kto": 3.18548846244812,
      "metrics/advantage_var": 485.99786376953125,
      "regularization/lipschitz_norm": 1406.13427734375,
      "regularization/w1": 1.361138105392456,
      "rewards/chosen": 0.623200059474859,
      "rewards/margins": 0.8529667563117492,
      "rewards/rejected": -0.22976669683689024,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 19.184036254882812,
      "kl": 19.55789566040039,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -33072462.953271028,
      "logits/rejected": -36717385.028213166,
      "logps/chosen": -503.6235397196262,
      "logps/rejected": -359.8566075626959,
      "loss": 0.557,
      "loss/base_kto": 3.1976475715637207,
      "metrics/advantage_var": 413.19903564453125,
      "regularization/lipschitz_norm": 1299.723876953125,
      "regularization/w1": 1.258132815361023,
      "rewards/chosen": 0.5707444045402551,
      "rewards/margins": 0.8260520236410829,
      "rewards/rejected": -0.2553076191008278,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 13.393630027770996,
      "kl": 14.76126766204834,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34477804.42813455,
      "logits/rejected": -36080558.21086262,
      "logps/chosen": -459.87485665137615,
      "logps/rejected": -376.9496305910543,
      "loss": 0.5588,
      "loss/base_kto": 3.187060594558716,
      "metrics/advantage_var": 448.4735412597656,
      "regularization/lipschitz_norm": 1325.549072265625,
      "regularization/w1": 1.2831313610076904,
      "rewards/chosen": 0.608802818741638,
      "rewards/margins": 0.8601546597023508,
      "rewards/rejected": -0.25135184096071284,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 20.297565460205078,
      "kl": 17.624088287353516,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34961089.70800628,
      "logits/rejected": -36001801.55520996,
      "logps/chosen": -484.5278649921507,
      "logps/rejected": -396.3648668351477,
      "loss": 0.5571,
      "loss/base_kto": 3.154284715652466,
      "metrics/advantage_var": 477.8008728027344,
      "regularization/lipschitz_norm": 1345.2122802734375,
      "regularization/w1": 1.3021655082702637,
      "rewards/chosen": 0.5777575434470663,
      "rewards/margins": 0.9070347125396985,
      "rewards/rejected": -0.3292771690926322,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 14.70542049407959,
      "kl": 15.789746284484863,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35181890.012578614,
      "logits/rejected": -36266766.310559005,
      "logps/chosen": -475.4132272012579,
      "logps/rejected": -403.7750630822981,
      "loss": 0.5656,
      "loss/base_kto": 3.2076053619384766,
      "metrics/advantage_var": 451.7100524902344,
      "regularization/lipschitz_norm": 1360.6304931640625,
      "regularization/w1": 1.317090392112732,
      "rewards/chosen": 0.5488386813949488,
      "rewards/margins": 0.815845928656328,
      "rewards/rejected": -0.2670072472613791,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 18.882680892944336,
      "kl": 13.536798477172852,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -33034973.515055466,
      "logits/rejected": -35850167.42064715,
      "logps/chosen": -510.31784865293184,
      "logps/rejected": -373.64283994607086,
      "loss": 0.5525,
      "loss/base_kto": 3.1619505882263184,
      "metrics/advantage_var": 423.0820007324219,
      "regularization/lipschitz_norm": 1299.3533935546875,
      "regularization/w1": 1.257773995399475,
      "rewards/chosen": 0.551079751572405,
      "rewards/margins": 0.871794834384604,
      "rewards/rejected": -0.32071508281219907,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 17.37849998474121,
      "kl": 13.970664024353027,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34963374.920821115,
      "logits/rejected": -35817728.85618729,
      "logps/chosen": -480.4471224340176,
      "logps/rejected": -381.63827341137124,
      "loss": 0.5578,
      "loss/base_kto": 3.226604461669922,
      "metrics/advantage_var": 408.15692138671875,
      "regularization/lipschitz_norm": 1276.3011474609375,
      "regularization/w1": 1.235459566116333,
      "rewards/chosen": 0.5288357105422928,
      "rewards/margins": 0.8134712224700729,
      "rewards/rejected": -0.2846355119277801,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 19.75193214416504,
      "kl": 14.514517784118652,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35115529.66037736,
      "logits/rejected": -37026784.19875777,
      "logps/chosen": -497.342570754717,
      "logps/rejected": -388.9575892857143,
      "loss": 0.5692,
      "loss/base_kto": 3.1615588665008545,
      "metrics/advantage_var": 509.42437744140625,
      "regularization/lipschitz_norm": 1438.1688232421875,
      "regularization/w1": 1.392147421836853,
      "rewards/chosen": 0.6212985920456221,
      "rewards/margins": 0.8863830660287233,
      "rewards/rejected": -0.2650844739831012,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 19.69506072998047,
      "kl": 16.02850341796875,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34802905.31189711,
      "logits/rejected": -37890903.92705167,
      "logps/chosen": -509.81797628617363,
      "logps/rejected": -335.5232712765957,
      "loss": 0.5594,
      "loss/base_kto": 3.169640302658081,
      "metrics/advantage_var": 452.1182556152344,
      "regularization/lipschitz_norm": 1348.7611083984375,
      "regularization/w1": 1.3056007623672485,
      "rewards/chosen": 0.592926025390625,
      "rewards/margins": 0.8676752246984232,
      "rewards/rejected": -0.27474919930779823,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 16.532896041870117,
      "kl": 16.380760192871094,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -34540307.2,
      "logits/rejected": -35429395.2,
      "logps/chosen": -462.204638671875,
      "logps/rejected": -384.1291259765625,
      "loss": 0.5557,
      "loss/base_kto": 3.118773937225342,
      "metrics/advantage_var": 484.7294921875,
      "regularization/lipschitz_norm": 1370.990234375,
      "regularization/w1": 1.3271185159683228,
      "rewards/chosen": 0.6463334560394287,
      "rewards/margins": 0.9337275981903076,
      "rewards/rejected": -0.2873941421508789,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 18.291141510009766,
      "kl": 15.371451377868652,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -34378296.529968455,
      "logits/rejected": -36235961.461300306,
      "logps/chosen": -487.8956526025237,
      "logps/rejected": -371.73868034055727,
      "loss": 0.5542,
      "loss/base_kto": 3.1566274166107178,
      "metrics/advantage_var": 443.59869384765625,
      "regularization/lipschitz_norm": 1319.3392333984375,
      "regularization/w1": 1.2771204710006714,
      "rewards/chosen": 0.6160349560084779,
      "rewards/margins": 0.8646803251218922,
      "rewards/rejected": -0.24864536911341428,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 14.354240417480469,
      "kl": 14.69382381439209,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34469841.45454545,
      "logits/rejected": -35720833.54216868,
      "logps/chosen": -470.97326501623377,
      "logps/rejected": -365.2261859939759,
      "loss": 0.5465,
      "loss/base_kto": 3.1806864738464355,
      "metrics/advantage_var": 398.2743835449219,
      "regularization/lipschitz_norm": 1230.29150390625,
      "regularization/w1": 1.1909221410751343,
      "rewards/chosen": 0.5174789924126166,
      "rewards/margins": 0.8392747662060325,
      "rewards/rejected": -0.32179577379341584,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 17.591693878173828,
      "kl": 14.455291748046875,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35108970.146341465,
      "logits/rejected": -36900279.79487179,
      "logps/chosen": -477.1206173780488,
      "logps/rejected": -384.06430288461536,
      "loss": 0.5648,
      "loss/base_kto": 3.2258079051971436,
      "metrics/advantage_var": 416.75469970703125,
      "regularization/lipschitz_norm": 1335.1112060546875,
      "regularization/w1": 1.2923877239227295,
      "rewards/chosen": 0.5574638552782012,
      "rewards/margins": 0.8143197465792829,
      "rewards/rejected": -0.2568558913010817,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 18.717485427856445,
      "kl": 15.758841514587402,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34577203.8585209,
      "logits/rejected": -35298055.00303952,
      "logps/chosen": -501.2543207395498,
      "logps/rejected": -386.2292220744681,
      "loss": 0.5602,
      "loss/base_kto": 3.1767189502716064,
      "metrics/advantage_var": 447.697509765625,
      "regularization/lipschitz_norm": 1348.321533203125,
      "regularization/w1": 1.3051751852035522,
      "rewards/chosen": 0.556097981247488,
      "rewards/margins": 0.8433800031845249,
      "rewards/rejected": -0.28728202193703695,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 15.613844871520996,
      "kl": 14.27246379852295,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -34433336.40677966,
      "logits/rejected": -36342087.80982567,
      "logps/chosen": -444.59639830508473,
      "logps/rejected": -366.9770701267829,
      "loss": 0.5546,
      "loss/base_kto": 3.1780214309692383,
      "metrics/advantage_var": 461.02716064453125,
      "regularization/lipschitz_norm": 1300.209228515625,
      "regularization/w1": 1.258602499961853,
      "rewards/chosen": 0.5440258149557011,
      "rewards/margins": 0.8622040915859879,
      "rewards/rejected": -0.31817827663028675,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 24.582273483276367,
      "kl": 13.369653701782227,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35645008.024279214,
      "logits/rejected": -36504762.33172303,
      "logps/chosen": -504.64007966616083,
      "logps/rejected": -373.8888385668277,
      "loss": 0.5646,
      "loss/base_kto": 3.1501927375793457,
      "metrics/advantage_var": 532.1537475585938,
      "regularization/lipschitz_norm": 1411.534912109375,
      "regularization/w1": 1.3663657903671265,
      "rewards/chosen": 0.5976630111022857,
      "rewards/margins": 0.9119348980537753,
      "rewards/rejected": -0.31427188695148955,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 19.878265380859375,
      "kl": 15.078536987304688,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -33949032.931419455,
      "logits/rejected": -35318493.89280245,
      "logps/chosen": -476.2298644338118,
      "logps/rejected": -368.3105139739663,
      "loss": 0.5646,
      "loss/base_kto": 3.1518266201019287,
      "metrics/advantage_var": 479.2774353027344,
      "regularization/lipschitz_norm": 1409.8193359375,
      "regularization/w1": 1.3647050857543945,
      "rewards/chosen": 0.5746874649558912,
      "rewards/margins": 0.8874602070476791,
      "rewards/rejected": -0.3127727420917879,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 17.30641746520996,
      "kl": 15.124449729919434,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -36250858.80254777,
      "logits/rejected": -36766374.478527606,
      "logps/chosen": -482.51393312101914,
      "logps/rejected": -395.6924127684049,
      "loss": 0.5616,
      "loss/base_kto": 3.1842668056488037,
      "metrics/advantage_var": 449.93511962890625,
      "regularization/lipschitz_norm": 1352.1444091796875,
      "regularization/w1": 1.3088759183883667,
      "rewards/chosen": 0.553188639841262,
      "rewards/margins": 0.8722517445760923,
      "rewards/rejected": -0.31906310473483035,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 22.55718421936035,
      "kl": 14.17529582977295,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -33969862.62751159,
      "logits/rejected": -36684643.083728276,
      "logps/chosen": -480.4167310664606,
      "logps/rejected": -387.9507306477093,
      "loss": 0.5575,
      "loss/base_kto": 3.2055633068084717,
      "metrics/advantage_var": 405.29302978515625,
      "regularization/lipschitz_norm": 1295.6785888671875,
      "regularization/w1": 1.2542169094085693,
      "rewards/chosen": 0.5050897848845634,
      "rewards/margins": 0.8096303858947579,
      "rewards/rejected": -0.30454060101019453,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 21.597618103027344,
      "kl": 13.912126541137695,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34215365.3498452,
      "logits/rejected": -36364695.01577287,
      "logps/chosen": -479.19732004643964,
      "logps/rejected": -364.0878844637224,
      "loss": 0.5597,
      "loss/base_kto": 3.1588613986968994,
      "metrics/advantage_var": 448.7057800292969,
      "regularization/lipschitz_norm": 1362.3128662109375,
      "regularization/w1": 1.3187189102172852,
      "rewards/chosen": 0.5791488033330109,
      "rewards/margins": 0.8799048306028255,
      "rewards/rejected": -0.30075602726981465,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 23.34357452392578,
      "kl": 17.03449821472168,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -33558875.96116505,
      "logits/rejected": -35319313.01510574,
      "logps/chosen": -500.37570792880257,
      "logps/rejected": -376.7595354984894,
      "loss": 0.5642,
      "loss/base_kto": 3.2193830013275146,
      "metrics/advantage_var": 441.334716796875,
      "regularization/lipschitz_norm": 1336.9698486328125,
      "regularization/w1": 1.2941868305206299,
      "rewards/chosen": 0.5284995662355886,
      "rewards/margins": 0.824237463757008,
      "rewards/rejected": -0.2957378975214195,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.960196480656998e+17,
      "train_loss": 0.586677935045206,
      "train_runtime": 11045.1759,
      "train_samples_per_second": 13.419,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.960196480656998e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}