{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 18.29014778137207,
      "kl": 1.4570707082748413,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37126905.17333333,
      "logits/rejected": -39094076.23529412,
      "logps/chosen": -485.8353125,
      "logps/rejected": -357.3402573529412,
      "loss": 0.5972,
      "loss/base_kto": 3.8672566413879395,
      "metrics/advantage_var": 195.62730407714844,
      "regularization/lipschitz_norm": 940.6920776367188,
      "regularization/w1": 0.910589873790741,
      "rewards/chosen": -0.2204015096028646,
      "rewards/margins": 0.10056833005418964,
      "rewards/rejected": -0.3209698396570542,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 22.535978317260742,
      "kl": 2.5399329662323,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35308072.99215071,
      "logits/rejected": -36455594.40124417,
      "logps/chosen": -465.87078100470956,
      "logps/rejected": -380.4113287325039,
      "loss": 0.5878,
      "loss/base_kto": 3.904927968978882,
      "metrics/advantage_var": 149.05447387695312,
      "regularization/lipschitz_norm": 823.9749145507812,
      "regularization/w1": 0.7976077198982239,
      "rewards/chosen": -0.06399994255796519,
      "rewards/margins": 0.08554940821022348,
      "rewards/rejected": -0.14954935076818868,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 24.34197235107422,
      "kl": 3.1339657306671143,
      "learning_rate": 5.9e-07,
      "logits/chosen": -34763361.8343949,
      "logits/rejected": -37482037.39877301,
      "logps/chosen": -482.2357683121019,
      "logps/rejected": -378.5544238880368,
      "loss": 0.5942,
      "loss/base_kto": 3.8586366176605225,
      "metrics/advantage_var": 188.65928649902344,
      "regularization/lipschitz_norm": 924.5841064453125,
      "regularization/w1": 0.8949974179267883,
      "rewards/chosen": -0.00727593291337323,
      "rewards/margins": 0.1317377202391862,
      "rewards/rejected": -0.13901365315255942,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 27.18002700805664,
      "kl": 2.128688335418701,
      "learning_rate": 7.9e-07,
      "logits/chosen": -36121831.0144,
      "logits/rejected": -37452341.93587786,
      "logps/chosen": -480.60845,
      "logps/rejected": -372.37197041984734,
      "loss": 0.5966,
      "loss/base_kto": 3.922574758529663,
      "metrics/advantage_var": 185.8826904296875,
      "regularization/lipschitz_norm": 878.3036499023438,
      "regularization/w1": 0.8501979112625122,
      "rewards/chosen": -0.0865505859375,
      "rewards/margins": 0.0659774343243082,
      "rewards/rejected": -0.1525280202618082,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 23.27158546447754,
      "kl": 4.320512294769287,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35060289.55591572,
      "logits/rejected": -36571040.2413273,
      "logps/chosen": -448.2167747163695,
      "logps/rejected": -361.9203431372549,
      "loss": 0.5886,
      "loss/base_kto": 3.891024112701416,
      "metrics/advantage_var": 155.9692840576172,
      "regularization/lipschitz_norm": 845.15283203125,
      "regularization/w1": 0.8181080222129822,
      "rewards/chosen": -0.028017547953650475,
      "rewards/margins": 0.09802821994410357,
      "rewards/rejected": -0.12604576789775404,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 23.44509506225586,
      "kl": 9.313093185424805,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35588562.823529415,
      "logits/rejected": -36494690.986666664,
      "logps/chosen": -482.75128676470587,
      "logps/rejected": -368.330546875,
      "loss": 0.5954,
      "loss/base_kto": 3.8357720375061035,
      "metrics/advantage_var": 211.01597595214844,
      "regularization/lipschitz_norm": 958.1713256835938,
      "regularization/w1": 0.9275097250938416,
      "rewards/chosen": 0.1270505792954389,
      "rewards/margins": 0.16840975593118107,
      "rewards/rejected": -0.041359176635742186,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 27.102447509765625,
      "kl": 26.627674102783203,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35467575.78507463,
      "logits/rejected": -36844597.718032785,
      "logps/chosen": -472.9744402985075,
      "logps/rejected": -354.23683401639346,
      "loss": 0.5967,
      "loss/base_kto": 3.7943520545959473,
      "metrics/advantage_var": 217.12857055664062,
      "regularization/lipschitz_norm": 1011.2306518554688,
      "regularization/w1": 0.9788713455200195,
      "rewards/chosen": 0.45366547997318096,
      "rewards/margins": 0.18523941656065535,
      "rewards/rejected": 0.2684260634125256,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 22.07541847229004,
      "kl": 29.8513240814209,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -37256623.49685535,
      "logits/rejected": -36983947.92546584,
      "logps/chosen": -471.38109276729557,
      "logps/rejected": -361.8736413043478,
      "loss": 0.5901,
      "loss/base_kto": 3.7140281200408936,
      "metrics/advantage_var": 231.1140594482422,
      "regularization/lipschitz_norm": 1039.8240966796875,
      "regularization/w1": 1.006549596786499,
      "rewards/chosen": 0.4847293110013758,
      "rewards/margins": 0.25128047684868027,
      "rewards/rejected": 0.23344883415269554,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 21.419479370117188,
      "kl": 14.638226509094238,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36142804.99843014,
      "logits/rejected": -37287455.05443235,
      "logps/chosen": -471.28051412872844,
      "logps/rejected": -347.9593944401244,
      "loss": 0.5901,
      "loss/base_kto": 3.77160906791687,
      "metrics/advantage_var": 220.82177734375,
      "regularization/lipschitz_norm": 980.3154296875,
      "regularization/w1": 0.9489454627037048,
      "rewards/chosen": 0.20925064925309067,
      "rewards/margins": 0.19540523370761673,
      "rewards/rejected": 0.013845415545473944,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 22.473234176635742,
      "kl": 25.710744857788086,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -38515946.86238532,
      "logits/rejected": -38492212.34504792,
      "logps/chosen": -507.7592698776758,
      "logps/rejected": -360.59669528753994,
      "loss": 0.595,
      "loss/base_kto": 3.7333428859710693,
      "metrics/advantage_var": 292.5412292480469,
      "regularization/lipschitz_norm": 1060.6162109375,
      "regularization/w1": 1.0266764163970947,
      "rewards/chosen": 0.44321611970207375,
      "rewards/margins": 0.22823484560001206,
      "rewards/rejected": 0.2149812741020617,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 21.400148391723633,
      "kl": 29.867273330688477,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37405573.24430956,
      "logits/rejected": -39347989.84863124,
      "logps/chosen": -495.8902219271624,
      "logps/rejected": -373.7242351046699,
      "loss": 0.5995,
      "loss/base_kto": 3.8104615211486816,
      "metrics/advantage_var": 244.73912048339844,
      "regularization/lipschitz_norm": 1018.1398315429688,
      "regularization/w1": 0.9855592846870422,
      "rewards/chosen": 0.4900019042227807,
      "rewards/margins": 0.16669079417315058,
      "rewards/rejected": 0.32331111004963015,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 22.627262115478516,
      "kl": 18.140928268432617,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36578178.189735614,
      "logits/rejected": -37561258.80062795,
      "logps/chosen": -474.09603421461895,
      "logps/rejected": -365.09205749607537,
      "loss": 0.5939,
      "loss/base_kto": 3.809729814529419,
      "metrics/advantage_var": 220.01670837402344,
      "regularization/lipschitz_norm": 972.9920043945312,
      "regularization/w1": 0.9418562054634094,
      "rewards/chosen": 0.2615461809268079,
      "rewards/margins": 0.15695515863297455,
      "rewards/rejected": 0.1045910222938334,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 20.957517623901367,
      "kl": 6.641819953918457,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -37125660.22047244,
      "logits/rejected": -38276694.524031006,
      "logps/chosen": -488.0752952755906,
      "logps/rejected": -380.35503875968993,
      "loss": 0.6034,
      "loss/base_kto": 3.8500633239746094,
      "metrics/advantage_var": 227.3952178955078,
      "regularization/lipschitz_norm": 1009.8142700195312,
      "regularization/w1": 0.9775001406669617,
      "rewards/chosen": 0.048197618619663506,
      "rewards/margins": 0.1383724311139858,
      "rewards/rejected": -0.09017481249432231,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 18.912769317626953,
      "kl": 9.864899635314941,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -35173403.8528,
      "logits/rejected": -37522114.63816794,
      "logps/chosen": -479.1098,
      "logps/rejected": -384.2582776717557,
      "loss": 0.5971,
      "loss/base_kto": 3.7604424953460693,
      "metrics/advantage_var": 254.92312622070312,
      "regularization/lipschitz_norm": 1049.9703369140625,
      "regularization/w1": 1.0163713693618774,
      "rewards/chosen": 0.09310198974609375,
      "rewards/margins": 0.18677775166052898,
      "rewards/rejected": -0.09367576191443523,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 22.78107261657715,
      "kl": 7.072554111480713,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36721923.16049383,
      "logits/rejected": -37791792.60759494,
      "logps/chosen": -482.63917824074076,
      "logps/rejected": -391.1912579113924,
      "loss": 0.6038,
      "loss/base_kto": 3.808419942855835,
      "metrics/advantage_var": 250.69656372070312,
      "regularization/lipschitz_norm": 1055.9146728515625,
      "regularization/w1": 1.022125244140625,
      "rewards/chosen": 0.09606993639910663,
      "rewards/margins": 0.1902502245932077,
      "rewards/rejected": -0.09418028819410107,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 23.265239715576172,
      "kl": 15.797464370727539,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -38567671.14586466,
      "logits/rejected": -38698278.71219512,
      "logps/chosen": -468.85075187969926,
      "logps/rejected": -387.3363821138211,
      "loss": 0.5941,
      "loss/base_kto": 3.741375684738159,
      "metrics/advantage_var": 256.41436767578125,
      "regularization/lipschitz_norm": 1044.8873291015625,
      "regularization/w1": 1.0114508867263794,
      "rewards/chosen": 0.3573325250381814,
      "rewards/margins": 0.24188839751341007,
      "rewards/rejected": 0.11544412752477134,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 22.0147647857666,
      "kl": 19.678110122680664,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -38655453.653666146,
      "logits/rejected": -38387356.24413145,
      "logps/chosen": -507.5981864274571,
      "logps/rejected": -364.62279929577466,
      "loss": 0.5866,
      "loss/base_kto": 3.7172722816467285,
      "metrics/advantage_var": 231.2719268798828,
      "regularization/lipschitz_norm": 1007.4017944335938,
      "regularization/w1": 0.9751648306846619,
      "rewards/chosen": 0.3712362447134604,
      "rewards/margins": 0.257120274477985,
      "rewards/rejected": 0.11411597023547536,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 18.77095603942871,
      "kl": 12.41693115234375,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -35779107.36336336,
      "logits/rejected": -36548648.02605863,
      "logps/chosen": -489.91272522522524,
      "logps/rejected": -388.24109324104234,
      "loss": 0.5905,
      "loss/base_kto": 3.728929281234741,
      "metrics/advantage_var": 234.8322296142578,
      "regularization/lipschitz_norm": 1027.6851806640625,
      "regularization/w1": 0.9947992563247681,
      "rewards/chosen": 0.25314079032645925,
      "rewards/margins": 0.26615497175554875,
      "rewards/rejected": -0.013014181429089475,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 24.464242935180664,
      "kl": 22.77442741394043,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35359588.17391305,
      "logits/rejected": -38313011.52201258,
      "logps/chosen": -494.44565217391306,
      "logps/rejected": -355.24002555031444,
      "loss": 0.5887,
      "loss/base_kto": 3.7706382274627686,
      "metrics/advantage_var": 218.9542236328125,
      "regularization/lipschitz_norm": 969.802734375,
      "regularization/w1": 0.9387690424919128,
      "rewards/chosen": 0.32858901882763975,
      "rewards/margins": 0.18018208438937816,
      "rewards/rejected": 0.14840693443826158,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 21.17736053466797,
      "kl": 19.017377853393555,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -37415138.49221184,
      "logits/rejected": -39053292.73981191,
      "logps/chosen": -477.3367406542056,
      "logps/rejected": -378.5361726097179,
      "loss": 0.5871,
      "loss/base_kto": 3.7642414569854736,
      "metrics/advantage_var": 217.3443603515625,
      "regularization/lipschitz_norm": 963.2283325195312,
      "regularization/w1": 0.9324049353599548,
      "rewards/chosen": 0.3229990451135368,
      "rewards/margins": 0.21354020548178348,
      "rewards/rejected": 0.10945883963175328,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 22.093955993652344,
      "kl": 24.88479232788086,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -37692469.72839506,
      "logits/rejected": -37927268.4556962,
      "logps/chosen": -482.25491898148147,
      "logps/rejected": -380.12490110759495,
      "loss": 0.5888,
      "loss/base_kto": 3.74531626701355,
      "metrics/advantage_var": 242.7454071044922,
      "regularization/lipschitz_norm": 996.7890625,
      "regularization/w1": 0.9648918509483337,
      "rewards/chosen": 0.428317411446277,
      "rewards/margins": 0.19805716603859305,
      "rewards/rejected": 0.23026024540768394,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 20.8978328704834,
      "kl": 16.53737449645996,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -37972866.01892745,
      "logits/rejected": -39153207.47987616,
      "logps/chosen": -484.7436415615142,
      "logps/rejected": -412.47779605263156,
      "loss": 0.6012,
      "loss/base_kto": 3.7755188941955566,
      "metrics/advantage_var": 272.7333984375,
      "regularization/lipschitz_norm": 1068.3348388671875,
      "regularization/w1": 1.034148097038269,
      "rewards/chosen": 0.2477119999353066,
      "rewards/margins": 0.19940300740181474,
      "rewards/rejected": 0.048308992533491864,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 16.93120574951172,
      "kl": 14.28690242767334,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -34974666.446909666,
      "logits/rejected": -37304790.18798151,
      "logps/chosen": -501.01005348652933,
      "logps/rejected": -368.0087394067797,
      "loss": 0.5904,
      "loss/base_kto": 3.686724901199341,
      "metrics/advantage_var": 277.653564453125,
      "regularization/lipschitz_norm": 1070.8505859375,
      "regularization/w1": 1.0365833044052124,
      "rewards/chosen": 0.2573503594013099,
      "rewards/margins": 0.3104527439479983,
      "rewards/rejected": -0.05310238454668841,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 16.04364585876465,
      "kl": 22.210905075073242,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -38007527.32530121,
      "logits/rejected": -38478475.63636363,
      "logps/chosen": -483.1725809487952,
      "logps/rejected": -374.72752637987014,
      "loss": 0.6012,
      "loss/base_kto": 3.7326862812042236,
      "metrics/advantage_var": 306.71075439453125,
      "regularization/lipschitz_norm": 1112.4329833984375,
      "regularization/w1": 1.07683527469635,
      "rewards/chosen": 0.4405103936252824,
      "rewards/margins": 0.22328726336199098,
      "rewards/rejected": 0.2172231302632914,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 17.009994506835938,
      "kl": 22.304758071899414,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -37415085.55932204,
      "logits/rejected": -39361560.34231379,
      "logps/chosen": -506.56346302003084,
      "logps/rejected": -370.74187797147385,
      "loss": 0.6004,
      "loss/base_kto": 3.730022430419922,
      "metrics/advantage_var": 254.05996704101562,
      "regularization/lipschitz_norm": 1108.2991943359375,
      "regularization/w1": 1.072833776473999,
      "rewards/chosen": 0.3808436694975443,
      "rewards/margins": 0.23417684194052912,
      "rewards/rejected": 0.14666682755701516,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 23.762605667114258,
      "kl": 20.390317916870117,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -37472712.22113821,
      "logits/rejected": -40082581.36541353,
      "logps/chosen": -503.1685975609756,
      "logps/rejected": -344.64191729323306,
      "loss": 0.5827,
      "loss/base_kto": 3.7008392810821533,
      "metrics/advantage_var": 255.104248046875,
      "regularization/lipschitz_norm": 992.3699340820312,
      "regularization/w1": 0.9606141448020935,
      "rewards/chosen": 0.30685841630144817,
      "rewards/margins": 0.25030279308202263,
      "rewards/rejected": 0.05655562321942552,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 22.993144989013672,
      "kl": 13.719802856445312,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37090411.87175573,
      "logits/rejected": -37467907.6864,
      "logps/chosen": -468.89274809160304,
      "logps/rejected": -393.59035,
      "loss": 0.6004,
      "loss/base_kto": 3.7587051391601562,
      "metrics/advantage_var": 267.5472106933594,
      "regularization/lipschitz_norm": 1078.6600341796875,
      "regularization/w1": 1.044142723083496,
      "rewards/chosen": 0.2237522270843273,
      "rewards/margins": 0.22937926108090934,
      "rewards/rejected": -0.005627033996582031,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 21.0300350189209,
      "kl": 20.42609214782715,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36306361.69305331,
      "logits/rejected": -38323241.82753404,
      "logps/chosen": -481.93755048465266,
      "logps/rejected": -360.97759077155825,
      "loss": 0.5868,
      "loss/base_kto": 3.732846975326538,
      "metrics/advantage_var": 230.20068359375,
      "regularization/lipschitz_norm": 992.978515625,
      "regularization/w1": 0.9612032175064087,
      "rewards/chosen": 0.3177708886166574,
      "rewards/margins": 0.21695634629353194,
      "rewards/rejected": 0.10081454232312548,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 21.744836807250977,
      "kl": 12.445245742797852,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -36472679.55987055,
      "logits/rejected": -37536062.64652568,
      "logps/chosen": -480.65544093851133,
      "logps/rejected": -357.3022092145015,
      "loss": 0.5845,
      "loss/base_kto": 3.6727676391601562,
      "metrics/advantage_var": 261.7809753417969,
      "regularization/lipschitz_norm": 1036.7802734375,
      "regularization/w1": 1.0036033391952515,
      "rewards/chosen": 0.21714681483395276,
      "rewards/margins": 0.28658312151366977,
      "rewards/rejected": -0.06943630667971701,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 19.334205627441406,
      "kl": 19.236360549926758,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37018689.76146789,
      "logits/rejected": -38043680.715654954,
      "logps/chosen": -485.98805428134557,
      "logps/rejected": -381.8394568690096,
      "loss": 0.593,
      "loss/base_kto": 3.6872031688690186,
      "metrics/advantage_var": 271.9364318847656,
      "regularization/lipschitz_norm": 1091.694580078125,
      "regularization/w1": 1.056760311126709,
      "rewards/chosen": 0.309730553116638,
      "rewards/margins": 0.29016388005720495,
      "rewards/rejected": 0.01956667305943303,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 19.244548797607422,
      "kl": 13.632672309875488,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -36514600.59546926,
      "logits/rejected": -37751418.19939577,
      "logps/chosen": -495.4087783171521,
      "logps/rejected": -384.47288047583083,
      "loss": 0.5902,
      "loss/base_kto": 3.7466392517089844,
      "metrics/advantage_var": 240.673828125,
      "regularization/lipschitz_norm": 1007.3256225585938,
      "regularization/w1": 0.9750910997390747,
      "rewards/chosen": 0.2429255019499646,
      "rewards/margins": 0.2508964667505198,
      "rewards/rejected": -0.00797096480055518,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 28.49403953552246,
      "kl": 6.630456447601318,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -36511871.805175036,
      "logits/rejected": -38109210.298555374,
      "logps/chosen": -464.95928462709287,
      "logps/rejected": -373.99934791332265,
      "loss": 0.6031,
      "loss/base_kto": 3.8054044246673584,
      "metrics/advantage_var": 254.1283721923828,
      "regularization/lipschitz_norm": 1052.9443359375,
      "regularization/w1": 1.0192501544952393,
      "rewards/chosen": 0.03811651314048825,
      "rewards/margins": 0.19720065563866992,
      "rewards/rejected": -0.15908414249818167,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 21.918201446533203,
      "kl": 11.34065055847168,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36806082.05819295,
      "logits/rejected": -37579290.94736842,
      "logps/chosen": -496.1367725880551,
      "logps/rejected": -365.33378189792666,
      "loss": 0.5917,
      "loss/base_kto": 3.709606885910034,
      "metrics/advantage_var": 249.86502075195312,
      "regularization/lipschitz_norm": 1057.7276611328125,
      "regularization/w1": 1.0238803625106812,
      "rewards/chosen": 0.2249710760656944,
      "rewards/margins": 0.2674478974297123,
      "rewards/rejected": -0.04247682136401795,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 25.505876541137695,
      "kl": 11.668891906738281,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -37083213.283018865,
      "logits/rejected": -37264530.28571428,
      "logps/chosen": -485.0509040880503,
      "logps/rejected": -365.58503979037266,
      "loss": 0.5885,
      "loss/base_kto": 3.721874237060547,
      "metrics/advantage_var": 244.218994140625,
      "regularization/lipschitz_norm": 1018.9505004882812,
      "regularization/w1": 0.9863441586494446,
      "rewards/chosen": 0.19097203428640305,
      "rewards/margins": 0.2519630727332441,
      "rewards/rejected": -0.060991038446841034,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 20.59295654296875,
      "kl": 15.112597465515137,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -37952208.285251215,
      "logits/rejected": -38671818.71493213,
      "logps/chosen": -489.17696515397085,
      "logps/rejected": -388.3527762066365,
      "loss": 0.5913,
      "loss/base_kto": 3.7084343433380127,
      "metrics/advantage_var": 286.5562744140625,
      "regularization/lipschitz_norm": 1055.5869140625,
      "regularization/w1": 1.02180814743042,
      "rewards/chosen": 0.2032725923065235,
      "rewards/margins": 0.26045843708463906,
      "rewards/rejected": -0.05718584477811557,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 18.425968170166016,
      "kl": 10.225451469421387,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -35257863.87692308,
      "logits/rejected": -38423239.92380952,
      "logps/chosen": -462.68048076923077,
      "logps/rejected": -381.8985863095238,
      "loss": 0.5899,
      "loss/base_kto": 3.7217705249786377,
      "metrics/advantage_var": 243.5096435546875,
      "regularization/lipschitz_norm": 1030.2635498046875,
      "regularization/w1": 0.997295081615448,
      "rewards/chosen": 0.14774301382211538,
      "rewards/margins": 0.24367004338638248,
      "rewards/rejected": -0.09592702956426712,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 14.970965385437012,
      "kl": 10.264970779418945,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36357974.3803681,
      "logits/rejected": -37892461.24840765,
      "logps/chosen": -497.34633819018404,
      "logps/rejected": -362.88460390127386,
      "loss": 0.6052,
      "loss/base_kto": 3.768388509750366,
      "metrics/advantage_var": 271.6504821777344,
      "regularization/lipschitz_norm": 1108.8814697265625,
      "regularization/w1": 1.0733972787857056,
      "rewards/chosen": 0.13551885218708062,
      "rewards/margins": 0.2049240495250148,
      "rewards/rejected": -0.06940519733793417,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 19.641616821289062,
      "kl": 17.239547729492188,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35284543.89627229,
      "logits/rejected": -36789311.32428356,
      "logps/chosen": -499.28307333873585,
      "logps/rejected": -373.6219834087481,
      "loss": 0.6078,
      "loss/base_kto": 3.773378849029541,
      "metrics/advantage_var": 287.3152770996094,
      "regularization/lipschitz_norm": 1124.9940185546875,
      "regularization/w1": 1.0889942646026611,
      "rewards/chosen": 0.2207545152166481,
      "rewards/margins": 0.20434218874337173,
      "rewards/rejected": 0.01641232647327636,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 23.141918182373047,
      "kl": 10.620841026306152,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -35900389.574193545,
      "logits/rejected": -36876991.416413374,
      "logps/chosen": -517.7074092741935,
      "logps/rejected": -376.32734137537994,
      "loss": 0.5987,
      "loss/base_kto": 3.5631072521209717,
      "metrics/advantage_var": 381.3092346191406,
      "regularization/lipschitz_norm": 1266.8702392578125,
      "regularization/w1": 1.2263303995132446,
      "rewards/chosen": 0.16041895958685107,
      "rewards/margins": 0.44302277129728995,
      "rewards/rejected": -0.28260381171043886,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 26.886734008789062,
      "kl": 7.9778265953063965,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36258746.658267714,
      "logits/rejected": -36793936.17364341,
      "logps/chosen": -481.85147637795274,
      "logps/rejected": -378.1058624031008,
      "loss": 0.6187,
      "loss/base_kto": 3.3150124549865723,
      "metrics/advantage_var": 659.8348999023438,
      "regularization/lipschitz_norm": 1688.483642578125,
      "regularization/w1": 1.6344521045684814,
      "rewards/chosen": 0.32000405619463584,
      "rewards/margins": 0.7908002006653239,
      "rewards/rejected": -0.47079614447068796,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 18.536203384399414,
      "kl": 10.486624717712402,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -35131913.30909091,
      "logits/rejected": -36720418.68387097,
      "logps/chosen": -481.3445075757576,
      "logps/rejected": -368.8100050403226,
      "loss": 0.607,
      "loss/base_kto": 3.3354427814483643,
      "metrics/advantage_var": 581.4777221679688,
      "regularization/lipschitz_norm": 1570.6834716796875,
      "regularization/w1": 1.5204215049743652,
      "rewards/chosen": 0.4108243537671638,
      "rewards/margins": 0.711322922394306,
      "rewards/rejected": -0.30049856862714214,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 17.286394119262695,
      "kl": 12.479207038879395,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35551176.47208931,
      "logits/rejected": -36891753.06584992,
      "logps/chosen": -479.0072767145136,
      "logps/rejected": -420.85356049004594,
      "loss": 0.6028,
      "loss/base_kto": 3.236600637435913,
      "metrics/advantage_var": 646.4462280273438,
      "regularization/lipschitz_norm": 1638.5428466796875,
      "regularization/w1": 1.5861095190048218,
      "rewards/chosen": 0.4569494804126794,
      "rewards/margins": 0.8522189940905365,
      "rewards/rejected": -0.395269513677857,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 18.596973419189453,
      "kl": 10.5941801071167,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -36412026.452648476,
      "logits/rejected": -37210289.680365294,
      "logps/chosen": -460.2596308186196,
      "logps/rejected": -381.8862014840183,
      "loss": 0.6332,
      "loss/base_kto": 3.3203790187835693,
      "metrics/advantage_var": 744.6414184570312,
      "regularization/lipschitz_norm": 1803.1883544921875,
      "regularization/w1": 1.7454862594604492,
      "rewards/chosen": 0.42800364486669845,
      "rewards/margins": 0.8059915573045671,
      "rewards/rejected": -0.3779879124378686,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 25.376361846923828,
      "kl": 10.37247085571289,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -36870867.91540785,
      "logits/rejected": -37704650.97734628,
      "logps/chosen": -477.55173716012087,
      "logps/rejected": -379.4901395631068,
      "loss": 0.6147,
      "loss/base_kto": 3.3837063312530518,
      "metrics/advantage_var": 606.6046752929688,
      "regularization/lipschitz_norm": 1584.3231201171875,
      "regularization/w1": 1.5336246490478516,
      "rewards/chosen": 0.25861393216873585,
      "rewards/margins": 0.7301805367336125,
      "rewards/rejected": -0.47156660456487665,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 17.33742904663086,
      "kl": 24.371213912963867,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -36366941.8798151,
      "logits/rejected": -37636676.96988907,
      "logps/chosen": -478.2693567026194,
      "logps/rejected": -365.2568343898574,
      "loss": 0.593,
      "loss/base_kto": 3.240446090698242,
      "metrics/advantage_var": 607.9482421875,
      "regularization/lipschitz_norm": 1553.604736328125,
      "regularization/w1": 1.5038893222808838,
      "rewards/chosen": 0.7234842215186826,
      "rewards/margins": 0.8010088329056181,
      "rewards/rejected": -0.07752461138693542,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 21.46354866027832,
      "kl": 20.619953155517578,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -37012783.82417583,
      "logits/rejected": -38748924.41679627,
      "logps/chosen": -483.8767660910518,
      "logps/rejected": -370.6149883359254,
      "loss": 0.613,
      "loss/base_kto": 3.2702362537384033,
      "metrics/advantage_var": 731.5919799804688,
      "regularization/lipschitz_norm": 1687.517578125,
      "regularization/w1": 1.6335169076919556,
      "rewards/chosen": 0.6287614711599784,
      "rewards/margins": 0.8008188647101095,
      "rewards/rejected": -0.17205739355013122,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 27.949670791625977,
      "kl": 23.042285919189453,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35671269.35015773,
      "logits/rejected": -36058314.89783282,
      "logps/chosen": -489.92044558359623,
      "logps/rejected": -373.0068933823529,
      "loss": 0.6269,
      "loss/base_kto": 3.339374542236328,
      "metrics/advantage_var": 702.3568725585938,
      "regularization/lipschitz_norm": 1730.974609375,
      "regularization/w1": 1.6755837202072144,
      "rewards/chosen": 0.5970071016425966,
      "rewards/margins": 0.699064118682554,
      "rewards/rejected": -0.10205701703995743,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 16.850645065307617,
      "kl": 19.39977264404297,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36006448.87636933,
      "logits/rejected": -36771542.86427457,
      "logps/chosen": -447.0553599374022,
      "logps/rejected": -370.2153129875195,
      "loss": 0.5865,
      "loss/base_kto": 3.266871690750122,
      "metrics/advantage_var": 522.5993041992188,
      "regularization/lipschitz_norm": 1472.3348388671875,
      "regularization/w1": 1.425220012664795,
      "rewards/chosen": 0.5224020037106318,
      "rewards/margins": 0.7634551162941196,
      "rewards/rejected": -0.24105311258348772,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 22.18784523010254,
      "kl": 16.881107330322266,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35078514.87179487,
      "logits/rejected": -36275562.146341465,
      "logps/chosen": -498.1405248397436,
      "logps/rejected": -367.7579554115854,
      "loss": 0.6019,
      "loss/base_kto": 3.342583417892456,
      "metrics/advantage_var": 527.4892578125,
      "regularization/lipschitz_norm": 1521.4276123046875,
      "regularization/w1": 1.472741961479187,
      "rewards/chosen": 0.4546708327073317,
      "rewards/margins": 0.7038324134211156,
      "rewards/rejected": -0.24916158071378383,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 17.10660743713379,
      "kl": 18.817310333251953,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -36048223.52238806,
      "logits/rejected": -37401690.64918033,
      "logps/chosen": -495.32285447761194,
      "logps/rejected": -388.59574795081966,
      "loss": 0.6225,
      "loss/base_kto": 3.279219150543213,
      "metrics/advantage_var": 737.9172973632812,
      "regularization/lipschitz_norm": 1756.8636474609375,
      "regularization/w1": 1.7006438970565796,
      "rewards/chosen": 0.609760887943097,
      "rewards/margins": 0.7567953172743919,
      "rewards/rejected": -0.14703442933129482,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 18.01519203186035,
      "kl": 8.799586296081543,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -34808334.266253866,
      "logits/rejected": -36906529.91798107,
      "logps/chosen": -470.0831075851393,
      "logps/rejected": -366.04475552050474,
      "loss": 0.6028,
      "loss/base_kto": 3.363507032394409,
      "metrics/advantage_var": 511.4743347167969,
      "regularization/lipschitz_norm": 1507.081298828125,
      "regularization/w1": 1.4588545560836792,
      "rewards/chosen": 0.36784457274634774,
      "rewards/margins": 0.6944655471215941,
      "rewards/rejected": -0.32662097437524645,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 17.3218936920166,
      "kl": 15.690451622009277,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35773232.096969694,
      "logits/rejected": -35636138.11612903,
      "logps/chosen": -501.4535984848485,
      "logps/rejected": -365.37310987903226,
      "loss": 0.6102,
      "loss/base_kto": 3.2086281776428223,
      "metrics/advantage_var": 698.2100830078125,
      "regularization/lipschitz_norm": 1728.182861328125,
      "regularization/w1": 1.672881007194519,
      "rewards/chosen": 0.6377998120857008,
      "rewards/margins": 0.8835186880937653,
      "rewards/rejected": -0.24571887600806452,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 17.114648818969727,
      "kl": 23.868871688842773,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35142348.96075353,
      "logits/rejected": -37201375.353032656,
      "logps/chosen": -504.3203492935636,
      "logps/rejected": -371.99927099533437,
      "loss": 0.6056,
      "loss/base_kto": 3.1933350563049316,
      "metrics/advantage_var": 700.6327514648438,
      "regularization/lipschitz_norm": 1706.2183837890625,
      "regularization/w1": 1.6516193151474,
      "rewards/chosen": 0.7034476143973214,
      "rewards/margins": 0.9059251244135418,
      "rewards/rejected": -0.20247751001622036,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 18.362234115600586,
      "kl": 20.849035263061523,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -34272197.44049459,
      "logits/rejected": -35951044.95418642,
      "logps/chosen": -470.51294435857807,
      "logps/rejected": -372.9975809636651,
      "loss": 0.6039,
      "loss/base_kto": 3.3228371143341064,
      "metrics/advantage_var": 573.0205688476562,
      "regularization/lipschitz_norm": 1558.5523681640625,
      "regularization/w1": 1.5086787939071655,
      "rewards/chosen": 0.530442911469402,
      "rewards/margins": 0.6922132665493783,
      "rewards/rejected": -0.1617703550799763,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 23.748563766479492,
      "kl": 17.160327911376953,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34102851.62264151,
      "logits/rejected": -35446984.347826086,
      "logps/chosen": -465.8104363207547,
      "logps/rejected": -373.1613451086956,
      "loss": 0.6057,
      "loss/base_kto": 3.3180763721466064,
      "metrics/advantage_var": 569.592041015625,
      "regularization/lipschitz_norm": 1578.3482666015625,
      "regularization/w1": 1.5278409719467163,
      "rewards/chosen": 0.5636146593393769,
      "rewards/margins": 0.7727055378437806,
      "rewards/rejected": -0.20909087850440364,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 23.817169189453125,
      "kl": 13.761589050292969,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -33891055.47086614,
      "logits/rejected": -35839542.77209302,
      "logps/chosen": -475.5876968503937,
      "logps/rejected": -379.225,
      "loss": 0.5958,
      "loss/base_kto": 3.3010895252227783,
      "metrics/advantage_var": 584.8517456054688,
      "regularization/lipschitz_norm": 1513.895751953125,
      "regularization/w1": 1.4654511213302612,
      "rewards/chosen": 0.46517670398622046,
      "rewards/margins": 0.74915045697647,
      "rewards/rejected": -0.2839737529902495,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 16.948810577392578,
      "kl": 7.98611307144165,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34369874.56726094,
      "logits/rejected": -37922723.330316745,
      "logps/chosen": -480.9890599675851,
      "logps/rejected": -365.08790535444945,
      "loss": 0.6066,
      "loss/base_kto": 3.2714080810546875,
      "metrics/advantage_var": 632.8931274414062,
      "regularization/lipschitz_norm": 1633.6640625,
      "regularization/w1": 1.5813868045806885,
      "rewards/chosen": 0.37567811344585694,
      "rewards/margins": 0.805177997083206,
      "rewards/rejected": -0.42949988363734914,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 18.99588394165039,
      "kl": 8.318107604980469,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34767724.291734196,
      "logits/rejected": -35584108.11463047,
      "logps/chosen": -490.5239566450567,
      "logps/rejected": -373.4036104826546,
      "loss": 0.6046,
      "loss/base_kto": 3.2944839000701904,
      "metrics/advantage_var": 566.6473388671875,
      "regularization/lipschitz_norm": 1593.2259521484375,
      "regularization/w1": 1.5422428846359253,
      "rewards/chosen": 0.3456378361780794,
      "rewards/margins": 0.7647938718645896,
      "rewards/rejected": -0.41915603568651016,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 18.505903244018555,
      "kl": 8.197784423828125,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34011293.055214725,
      "logits/rejected": -36369544.968152866,
      "logps/chosen": -490.1889378834356,
      "logps/rejected": -384.5946705812102,
      "loss": 0.6172,
      "loss/base_kto": 3.3246331214904785,
      "metrics/advantage_var": 656.5321655273438,
      "regularization/lipschitz_norm": 1666.1669921875,
      "regularization/w1": 1.6128495931625366,
      "rewards/chosen": 0.3669387255709595,
      "rewards/margins": 0.7547161634006272,
      "rewards/rejected": -0.3877774378296676,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 16.069843292236328,
      "kl": 3.699871063232422,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -33936347.14241002,
      "logits/rejected": -34886366.85179407,
      "logps/chosen": -475.2531298904538,
      "logps/rejected": -387.1373098673947,
      "loss": 0.6161,
      "loss/base_kto": 3.3539459705352783,
      "metrics/advantage_var": 633.36083984375,
      "regularization/lipschitz_norm": 1627.1385498046875,
      "regularization/w1": 1.5750700235366821,
      "rewards/chosen": 0.13495619270909748,
      "rewards/margins": 0.7070479484496397,
      "rewards/rejected": -0.5720917557405422,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 18.5937557220459,
      "kl": 13.287925720214844,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34599509.60252366,
      "logits/rejected": -34751373.86996904,
      "logps/chosen": -473.21520110410097,
      "logps/rejected": -411.38694852941177,
      "loss": 0.6095,
      "loss/base_kto": 3.303356170654297,
      "metrics/advantage_var": 738.0794067382812,
      "regularization/lipschitz_norm": 1624.4622802734375,
      "regularization/w1": 1.572479486465454,
      "rewards/chosen": 0.41713139386583203,
      "rewards/margins": 0.786764929717467,
      "rewards/rejected": -0.36963353585163505,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 14.898553848266602,
      "kl": 9.77316951751709,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34896962.22885572,
      "logits/rejected": -36254104.389955685,
      "logps/chosen": -495.8976471807629,
      "logps/rejected": -352.47267355982274,
      "loss": 0.6091,
      "loss/base_kto": 3.2609596252441406,
      "metrics/advantage_var": 734.3327026367188,
      "regularization/lipschitz_norm": 1664.761962890625,
      "regularization/w1": 1.6114895343780518,
      "rewards/chosen": 0.3713784795100021,
      "rewards/margins": 0.8420873744509179,
      "rewards/rejected": -0.4707088949409158,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 17.896150588989258,
      "kl": 6.074289798736572,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -33776399.15022762,
      "logits/rejected": -33075975.00805153,
      "logps/chosen": -508.469034522003,
      "logps/rejected": -412.82168377616745,
      "loss": 0.6019,
      "loss/base_kto": 3.3112874031066895,
      "metrics/advantage_var": 567.4923706054688,
      "regularization/lipschitz_norm": 1553.6488037109375,
      "regularization/w1": 1.503931999206543,
      "rewards/chosen": 0.2563087104484778,
      "rewards/margins": 0.7786207948983319,
      "rewards/rejected": -0.5223120844498541,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 17.405357360839844,
      "kl": 8.466923713684082,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34845189.66192733,
      "logits/rejected": -35608800.741885625,
      "logps/chosen": -495.66903633491313,
      "logps/rejected": -385.2935181607419,
      "loss": 0.6216,
      "loss/base_kto": 3.321855306625366,
      "metrics/advantage_var": 709.4938354492188,
      "regularization/lipschitz_norm": 1705.716796875,
      "regularization/w1": 1.6511337757110596,
      "rewards/chosen": 0.3443122851716775,
      "rewards/margins": 0.7471894560233245,
      "rewards/rejected": -0.40287717085164704,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 15.573775291442871,
      "kl": 7.134391784667969,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33920080.67878788,
      "logits/rejected": -36426282.94193549,
      "logps/chosen": -500.8137784090909,
      "logps/rejected": -360.63009072580644,
      "loss": 0.6084,
      "loss/base_kto": 3.3668205738067627,
      "metrics/advantage_var": 544.8922729492188,
      "regularization/lipschitz_norm": 1549.573486328125,
      "regularization/w1": 1.499987006187439,
      "rewards/chosen": 0.2990838253136837,
      "rewards/margins": 0.704384433302595,
      "rewards/rejected": -0.40530060798891127,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 19.988651275634766,
      "kl": 19.997989654541016,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -31784119.87616099,
      "logits/rejected": -35308534.30914827,
      "logps/chosen": -502.8266253869969,
      "logps/rejected": -358.9458300473186,
      "loss": 0.6203,
      "loss/base_kto": 3.3223488330841064,
      "metrics/advantage_var": 1186.4998779296875,
      "regularization/lipschitz_norm": 1694.468017578125,
      "regularization/w1": 1.6402450799942017,
      "rewards/chosen": 0.5360499520788989,
      "rewards/margins": 0.769386824937855,
      "rewards/rejected": -0.23333687285895605,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 21.27825927734375,
      "kl": 14.778144836425781,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -33192780.406153847,
      "logits/rejected": -35539620.16507936,
      "logps/chosen": -487.72826923076923,
      "logps/rejected": -365.61153273809526,
      "loss": 0.5938,
      "loss/base_kto": 3.2722222805023193,
      "metrics/advantage_var": 570.6756591796875,
      "regularization/lipschitz_norm": 1527.1158447265625,
      "regularization/w1": 1.478248119354248,
      "rewards/chosen": 0.5232102614182692,
      "rewards/margins": 0.751742186680236,
      "rewards/rejected": -0.22853192526196678,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 18.707075119018555,
      "kl": 14.053110122680664,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -34841998.31181102,
      "logits/rejected": -36095298.28217054,
      "logps/chosen": -475.1556594488189,
      "logps/rejected": -374.0540697674419,
      "loss": 0.6004,
      "loss/base_kto": 3.334726333618164,
      "metrics/advantage_var": 629.5460815429688,
      "regularization/lipschitz_norm": 1516.635986328125,
      "regularization/w1": 1.4681035280227661,
      "rewards/chosen": 0.48633317121370573,
      "rewards/margins": 0.7661481353307116,
      "rewards/rejected": -0.2798149641170058,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 22.795734405517578,
      "kl": 10.226655960083008,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34476848.64797508,
      "logits/rejected": -34940799.59874608,
      "logps/chosen": -486.3995813862928,
      "logps/rejected": -359.81235305642633,
      "loss": 0.5896,
      "loss/base_kto": 3.316878080368042,
      "metrics/advantage_var": 529.7022705078125,
      "regularization/lipschitz_norm": 1446.478759765625,
      "regularization/w1": 1.4001914262771606,
      "rewards/chosen": 0.39251145692629236,
      "rewards/margins": 0.7439681498999404,
      "rewards/rejected": -0.3514566929736481,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 17.954349517822266,
      "kl": 17.41472816467285,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -35201938.6407767,
      "logits/rejected": -35623357.486404836,
      "logps/chosen": -483.64370954692555,
      "logps/rejected": -387.6900490936556,
      "loss": 0.6029,
      "loss/base_kto": 3.2610299587249756,
      "metrics/advantage_var": 612.34765625,
      "regularization/lipschitz_norm": 1613.9171142578125,
      "regularization/w1": 1.5622718334197998,
      "rewards/chosen": 0.5058932011181483,
      "rewards/margins": 0.8012901269377527,
      "rewards/rejected": -0.2953969258196044,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 17.581260681152344,
      "kl": 11.968046188354492,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -33651710.41731066,
      "logits/rejected": -33495272.9478673,
      "logps/chosen": -456.492658423493,
      "logps/rejected": -381.71936710110583,
      "loss": 0.595,
      "loss/base_kto": 3.2871201038360596,
      "metrics/advantage_var": 583.8927612304688,
      "regularization/lipschitz_norm": 1521.3782958984375,
      "regularization/w1": 1.4726942777633667,
      "rewards/chosen": 0.4107969304693537,
      "rewards/margins": 0.7617834051558663,
      "rewards/rejected": -0.3509864746865126,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 20.793994903564453,
      "kl": 14.662152290344238,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -35222220.96075353,
      "logits/rejected": -36583083.197511666,
      "logps/chosen": -492.78120094191524,
      "logps/rejected": -378.14959175738727,
      "loss": 0.6018,
      "loss/base_kto": 3.297760486602783,
      "metrics/advantage_var": 564.8939208984375,
      "regularization/lipschitz_norm": 1567.152587890625,
      "regularization/w1": 1.5170036554336548,
      "rewards/chosen": 0.4784900419753238,
      "rewards/margins": 0.7377271860539834,
      "rewards/rejected": -0.2592371440786596,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 18.106521606445312,
      "kl": 17.453224182128906,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34577372.8,
      "logits/rejected": -35849900.8,
      "logps/chosen": -478.12919921875,
      "logps/rejected": -374.3378173828125,
      "loss": 0.6005,
      "loss/base_kto": 3.2170441150665283,
      "metrics/advantage_var": 673.6275634765625,
      "regularization/lipschitz_norm": 1639.2894287109375,
      "regularization/w1": 1.5868322849273682,
      "rewards/chosen": 0.5354191780090332,
      "rewards/margins": 0.8580185413360595,
      "rewards/rejected": -0.3225993633270264,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 19.869972229003906,
      "kl": 19.689435958862305,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -35870056.2962963,
      "logits/rejected": -35619895.088607594,
      "logps/chosen": -462.14351851851853,
      "logps/rejected": -369.37663172468353,
      "loss": 0.6031,
      "loss/base_kto": 3.3086509704589844,
      "metrics/advantage_var": 570.68212890625,
      "regularization/lipschitz_norm": 1566.245361328125,
      "regularization/w1": 1.5161254405975342,
      "rewards/chosen": 0.5847580521195023,
      "rewards/margins": 0.7637615986290639,
      "rewards/rejected": -0.17900354650956166,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 17.84902000427246,
      "kl": 20.60800552368164,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -33869262.048780486,
      "logits/rejected": -35129416.20512821,
      "logps/chosen": -472.5958936737805,
      "logps/rejected": -374.59900841346155,
      "loss": 0.6061,
      "loss/base_kto": 3.276806354522705,
      "metrics/advantage_var": 604.2709350585938,
      "regularization/lipschitz_norm": 1623.892822265625,
      "regularization/w1": 1.5719282627105713,
      "rewards/chosen": 0.6462965244200172,
      "rewards/margins": 0.810724391722545,
      "rewards/rejected": -0.16442786730252779,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 16.47085189819336,
      "kl": 17.133132934570312,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -34565411.8647343,
      "logits/rejected": -37085233.72382397,
      "logps/chosen": -497.7308776167472,
      "logps/rejected": -355.6858402883156,
      "loss": 0.5882,
      "loss/base_kto": 3.306173324584961,
      "metrics/advantage_var": 500.00372314453125,
      "regularization/lipschitz_norm": 1446.0572509765625,
      "regularization/w1": 1.3997834920883179,
      "rewards/chosen": 0.4956057636058273,
      "rewards/margins": 0.7162988164157784,
      "rewards/rejected": -0.22069305280995116,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 16.752580642700195,
      "kl": 14.875192642211914,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -33914434.37037037,
      "logits/rejected": -35248309.468354434,
      "logps/chosen": -442.65928819444446,
      "logps/rejected": -374.595332278481,
      "loss": 0.597,
      "loss/base_kto": 3.3261163234710693,
      "metrics/advantage_var": 571.9568481445312,
      "regularization/lipschitz_norm": 1497.6256103515625,
      "regularization/w1": 1.4497016668319702,
      "rewards/chosen": 0.4288687529387297,
      "rewards/margins": 0.7263318923399362,
      "rewards/rejected": -0.2974631394012065,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 14.579239845275879,
      "kl": 16.715412139892578,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35149984.104234524,
      "logits/rejected": -35314502.93975904,
      "logps/chosen": -450.90482491856676,
      "logps/rejected": -354.9996234939759,
      "loss": 0.5681,
      "loss/base_kto": 3.2480804920196533,
      "metrics/advantage_var": 559.4739379882812,
      "regularization/lipschitz_norm": 1339.9051513671875,
      "regularization/w1": 1.2970281839370728,
      "rewards/chosen": 0.4743180166238294,
      "rewards/margins": 0.7994848337948572,
      "rewards/rejected": -0.32516681717102786,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 16.029247283935547,
      "kl": 13.296783447265625,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34697117.56899225,
      "logits/rejected": -35621518.71496063,
      "logps/chosen": -481.5494186046512,
      "logps/rejected": -385.4054625984252,
      "loss": 0.5587,
      "loss/base_kto": 3.2124602794647217,
      "metrics/advantage_var": 428.8362731933594,
      "regularization/lipschitz_norm": 1298.7864990234375,
      "regularization/w1": 1.2572253942489624,
      "rewards/chosen": 0.5423459075218023,
      "rewards/margins": 0.8369628710660322,
      "rewards/rejected": -0.29461696354422984,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 13.329975128173828,
      "kl": 13.94482707977295,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -34472352.1015873,
      "logits/rejected": -36389752.51692308,
      "logps/chosen": -486.62390873015875,
      "logps/rejected": -357.45927884615384,
      "loss": 0.5597,
      "loss/base_kto": 3.1015193462371826,
      "metrics/advantage_var": 529.8474731445312,
      "regularization/lipschitz_norm": 1421.7562255859375,
      "regularization/w1": 1.3762601613998413,
      "rewards/chosen": 0.5987052796378968,
      "rewards/margins": 0.9748003677163222,
      "rewards/rejected": -0.37609508807842545,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 15.356056213378906,
      "kl": 11.886622428894043,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34654014.91190108,
      "logits/rejected": -35670407.48183254,
      "logps/chosen": -464.8797333848532,
      "logps/rejected": -384.8746297393365,
      "loss": 0.5671,
      "loss/base_kto": 3.201629638671875,
      "metrics/advantage_var": 484.70343017578125,
      "regularization/lipschitz_norm": 1378.9920654296875,
      "regularization/w1": 1.3348641395568848,
      "rewards/chosen": 0.5062245954133259,
      "rewards/margins": 0.8540332645653672,
      "rewards/rejected": -0.3478086691520414,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 16.912874221801758,
      "kl": 14.003283500671387,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34599419.414925374,
      "logits/rejected": -35416252.01311475,
      "logps/chosen": -485.9853078358209,
      "logps/rejected": -386.86342213114756,
      "loss": 0.5719,
      "loss/base_kto": 3.1810734272003174,
      "metrics/advantage_var": 468.3973693847656,
      "regularization/lipschitz_norm": 1440.0457763671875,
      "regularization/w1": 1.3939642906188965,
      "rewards/chosen": 0.5261796638147155,
      "rewards/margins": 0.8673378209132042,
      "rewards/rejected": -0.34115815709848873,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 13.959023475646973,
      "kl": 18.680160522460938,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -32944703.795527156,
      "logits/rejected": -36012818.00611621,
      "logps/chosen": -466.327875399361,
      "logps/rejected": -365.87370986238534,
      "loss": 0.5582,
      "loss/base_kto": 3.122089147567749,
      "metrics/advantage_var": 494.7021484375,
      "regularization/lipschitz_norm": 1387.5946044921875,
      "regularization/w1": 1.3431915044784546,
      "rewards/chosen": 0.6593199903591753,
      "rewards/margins": 0.9139462265544043,
      "rewards/rejected": -0.2546262361952289,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 17.987796783447266,
      "kl": 15.049530029296875,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34199591.44296296,
      "logits/rejected": -34292866.32727273,
      "logps/chosen": -487.01222222222225,
      "logps/rejected": -378.01012396694216,
      "loss": 0.5543,
      "loss/base_kto": 3.1539852619171143,
      "metrics/advantage_var": 451.4657287597656,
      "regularization/lipschitz_norm": 1322.9229736328125,
      "regularization/w1": 1.2805893421173096,
      "rewards/chosen": 0.616356517650463,
      "rewards/margins": 0.8641814970700136,
      "rewards/rejected": -0.24782497941955062,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 19.532485961914062,
      "kl": 15.837020874023438,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34549902.58227848,
      "logits/rejected": -35127526.71604938,
      "logps/chosen": -485.71073971518985,
      "logps/rejected": -370.6300877700617,
      "loss": 0.5568,
      "loss/base_kto": 3.135880708694458,
      "metrics/advantage_var": 483.2705078125,
      "regularization/lipschitz_norm": 1361.748046875,
      "regularization/w1": 1.3181720972061157,
      "rewards/chosen": 0.5477567262287382,
      "rewards/margins": 0.9015852270470615,
      "rewards/rejected": -0.35382850081832323,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 19.168087005615234,
      "kl": 13.672842979431152,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -33957715.434022255,
      "logits/rejected": -36874866.039938554,
      "logps/chosen": -496.45399443561206,
      "logps/rejected": -351.21346966205834,
      "loss": 0.5529,
      "loss/base_kto": 3.1434247493743896,
      "metrics/advantage_var": 430.8213806152344,
      "regularization/lipschitz_norm": 1322.162109375,
      "regularization/w1": 1.2798528671264648,
      "rewards/chosen": 0.5340297680780505,
      "rewards/margins": 0.883097256834831,
      "rewards/rejected": -0.34906748875678045,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 20.155269622802734,
      "kl": 13.380441665649414,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -34002827.95712098,
      "logits/rejected": -34305286.94098884,
      "logps/chosen": -480.2856527565084,
      "logps/rejected": -387.265899122807,
      "loss": 0.5584,
      "loss/base_kto": 3.1513402462005615,
      "metrics/advantage_var": 502.35137939453125,
      "regularization/lipschitz_norm": 1359.7581787109375,
      "regularization/w1": 1.3162459135055542,
      "rewards/chosen": 0.5978419258620071,
      "rewards/margins": 0.9165808967021936,
      "rewards/rejected": -0.31873897084018643,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 14.516063690185547,
      "kl": 14.525212287902832,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -33392608.0,
      "logits/rejected": -35159033.6,
      "logps/chosen": -503.3990234375,
      "logps/rejected": -400.6938720703125,
      "loss": 0.5642,
      "loss/base_kto": 3.163792133331299,
      "metrics/advantage_var": 471.1412048339844,
      "regularization/lipschitz_norm": 1394.1875,
      "regularization/w1": 1.3495734930038452,
      "rewards/chosen": 0.5471538543701172,
      "rewards/margins": 0.8902327299118042,
      "rewards/rejected": -0.34307887554168703,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 15.617125511169434,
      "kl": 12.250290870666504,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -35133218.24203822,
      "logits/rejected": -35582127.90184049,
      "logps/chosen": -496.52164609872614,
      "logps/rejected": -370.25340299079755,
      "loss": 0.5481,
      "loss/base_kto": 3.1153676509857178,
      "metrics/advantage_var": 467.2294616699219,
      "regularization/lipschitz_norm": 1310.98681640625,
      "regularization/w1": 1.2690353393554688,
      "rewards/chosen": 0.5348491790188346,
      "rewards/margins": 0.9311423510945391,
      "rewards/rejected": -0.3962931720757046,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 19.886005401611328,
      "kl": 13.618847846984863,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34087156.72955975,
      "logits/rejected": -36125654.65838509,
      "logps/chosen": -480.1741843553459,
      "logps/rejected": -372.6374708850932,
      "loss": 0.5528,
      "loss/base_kto": 3.146395683288574,
      "metrics/advantage_var": 431.99267578125,
      "regularization/lipschitz_norm": 1317.941162109375,
      "regularization/w1": 1.2757669687271118,
      "rewards/chosen": 0.5684257843209513,
      "rewards/margins": 0.8884076614287493,
      "rewards/rejected": -0.31998187710779796,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 17.793807983398438,
      "kl": 13.24231243133545,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -34019655.41272431,
      "logits/rejected": -36141445.94902549,
      "logps/chosen": -463.381015497553,
      "logps/rejected": -380.266726011994,
      "loss": 0.5516,
      "loss/base_kto": 3.1404495239257812,
      "metrics/advantage_var": 454.6252136230469,
      "regularization/lipschitz_norm": 1314.20654296875,
      "regularization/w1": 1.2721518278121948,
      "rewards/chosen": 0.48604117015446574,
      "rewards/margins": 0.9000160064667784,
      "rewards/rejected": -0.4139748363123126,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 16.166582107543945,
      "kl": 15.547632217407227,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -33855733.76,
      "logits/rejected": -35213627.32698413,
      "logps/chosen": -477.28346153846155,
      "logps/rejected": -368.85835813492065,
      "loss": 0.5536,
      "loss/base_kto": 3.20768141746521,
      "metrics/advantage_var": 407.5632629394531,
      "regularization/lipschitz_norm": 1261.4091796875,
      "regularization/w1": 1.2210441827774048,
      "rewards/chosen": 0.486441650390625,
      "rewards/margins": 0.8268258279467386,
      "rewards/rejected": -0.3403841775561136,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 18.32387351989746,
      "kl": 14.501152038574219,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -33962063.47462687,
      "logits/rejected": -36594066.885245904,
      "logps/chosen": -481.04869402985076,
      "logps/rejected": -392.35432889344264,
      "loss": 0.5656,
      "loss/base_kto": 3.1842072010040283,
      "metrics/advantage_var": 451.8145446777344,
      "regularization/lipschitz_norm": 1384.595947265625,
      "regularization/w1": 1.340288758277893,
      "rewards/chosen": 0.5720315050722947,
      "rewards/margins": 0.8596475719268029,
      "rewards/rejected": -0.2876160668545082,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 21.38368034362793,
      "kl": 19.178592681884766,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -33770083.48815166,
      "logits/rejected": -34194788.10510046,
      "logps/chosen": -498.29260466034754,
      "logps/rejected": -371.72930351622875,
      "loss": 0.5602,
      "loss/base_kto": 3.1773126125335693,
      "metrics/advantage_var": 452.7632751464844,
      "regularization/lipschitz_norm": 1347.4056396484375,
      "regularization/w1": 1.3042887449264526,
      "rewards/chosen": 0.5842705074268119,
      "rewards/margins": 0.8374482504806904,
      "rewards/rejected": -0.25317774305387847,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 16.915109634399414,
      "kl": 16.03028678894043,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35029415.72413793,
      "logits/rejected": -34908601.004893966,
      "logps/chosen": -485.9177286356822,
      "logps/rejected": -388.0621431484502,
      "loss": 0.5582,
      "loss/base_kto": 3.2086143493652344,
      "metrics/advantage_var": 425.7274475097656,
      "regularization/lipschitz_norm": 1298.6712646484375,
      "regularization/w1": 1.2571138143539429,
      "rewards/chosen": 0.56939697265625,
      "rewards/margins": 0.8277747697192344,
      "rewards/rejected": -0.2583777970629843,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 24.30204200744629,
      "kl": 14.940142631530762,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34003794.85990338,
      "logits/rejected": -35394291.18057663,
      "logps/chosen": -465.68599033816423,
      "logps/rejected": -373.41426403641884,
      "loss": 0.5592,
      "loss/base_kto": 3.2054061889648438,
      "metrics/advantage_var": 439.9033203125,
      "regularization/lipschitz_norm": 1309.96923828125,
      "regularization/w1": 1.268050193786621,
      "rewards/chosen": 0.5584115290987319,
      "rewards/margins": 0.8335664185426692,
      "rewards/rejected": -0.2751548894439373,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 16.94352149963379,
      "kl": 16.40333366394043,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -35490728.04907975,
      "logits/rejected": -36010857.98726115,
      "logps/chosen": -466.27779907975463,
      "logps/rejected": -394.1244775079618,
      "loss": 0.558,
      "loss/base_kto": 3.1873714923858643,
      "metrics/advantage_var": 437.2276916503906,
      "regularization/lipschitz_norm": 1318.5452880859375,
      "regularization/w1": 1.276351809501648,
      "rewards/chosen": 0.5535883991264858,
      "rewards/margins": 0.8391093536230274,
      "rewards/rejected": -0.2855209544965416,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 17.528411865234375,
      "kl": 16.821489334106445,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -35454871.643312104,
      "logits/rejected": -36273387.582822084,
      "logps/chosen": -486.5666301751592,
      "logps/rejected": -375.32254121932516,
      "loss": 0.5601,
      "loss/base_kto": 3.201885938644409,
      "metrics/advantage_var": 461.4477233886719,
      "regularization/lipschitz_norm": 1321.3861083984375,
      "regularization/w1": 1.2791017293930054,
      "rewards/chosen": 0.5660655999639231,
      "rewards/margins": 0.8268592273693822,
      "rewards/rejected": -0.26079362740545914,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 17.622358322143555,
      "kl": 14.854166030883789,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -34862397.47112462,
      "logits/rejected": -36349115.67845659,
      "logps/chosen": -487.986797112462,
      "logps/rejected": -382.080159766881,
      "loss": 0.5516,
      "loss/base_kto": 3.13665771484375,
      "metrics/advantage_var": 469.099365234375,
      "regularization/lipschitz_norm": 1318.103515625,
      "regularization/w1": 1.2759240865707397,
      "rewards/chosen": 0.5873298992864266,
      "rewards/margins": 0.9101484714013282,
      "rewards/rejected": -0.3228185721149015,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 18.09905433654785,
      "kl": 12.205322265625,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -33807141.66966967,
      "logits/rejected": -35700736.0,
      "logps/chosen": -485.51783033033036,
      "logps/rejected": -370.8291174674267,
      "loss": 0.5639,
      "loss/base_kto": 3.148710012435913,
      "metrics/advantage_var": 479.14111328125,
      "regularization/lipschitz_norm": 1407.3203125,
      "regularization/w1": 1.3622859716415405,
      "rewards/chosen": 0.5770113375093844,
      "rewards/margins": 0.9141302256436729,
      "rewards/rejected": -0.3371188881342885,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 21.58671760559082,
      "kl": 13.516070365905762,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34692467.650842264,
      "logits/rejected": -35320722.57735247,
      "logps/chosen": -488.7352603369066,
      "logps/rejected": -369.4571371610845,
      "loss": 0.5547,
      "loss/base_kto": 3.1675968170166016,
      "metrics/advantage_var": 478.0860290527344,
      "regularization/lipschitz_norm": 1311.8924560546875,
      "regularization/w1": 1.269911766052246,
      "rewards/chosen": 0.5942296353817716,
      "rewards/margins": 0.8910569364558347,
      "rewards/rejected": -0.296827301074063,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 16.513416290283203,
      "kl": 13.972925186157227,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -34216062.180094786,
      "logits/rejected": -35726261.61360124,
      "logps/chosen": -454.08101303317534,
      "logps/rejected": -376.26019126738794,
      "loss": 0.558,
      "loss/base_kto": 3.2071805000305176,
      "metrics/advantage_var": 427.6632995605469,
      "regularization/lipschitz_norm": 1298.3660888671875,
      "regularization/w1": 1.256818413734436,
      "rewards/chosen": 0.5536692831753555,
      "rewards/margins": 0.8289034289397609,
      "rewards/rejected": -0.2752341457644054,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 16.91568374633789,
      "kl": 14.527670860290527,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34835875.566718996,
      "logits/rejected": -36801327.377916016,
      "logps/chosen": -482.87509811616957,
      "logps/rejected": -383.3812694401244,
      "loss": 0.555,
      "loss/base_kto": 3.2087466716766357,
      "metrics/advantage_var": 423.06982421875,
      "regularization/lipschitz_norm": 1272.2606201171875,
      "regularization/w1": 1.2315481901168823,
      "rewards/chosen": 0.5398905468136774,
      "rewards/margins": 0.8093527169770351,
      "rewards/rejected": -0.2694621701633578,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 20.491783142089844,
      "kl": 16.086137771606445,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -33287701.90808241,
      "logits/rejected": -36201230.59476117,
      "logps/chosen": -492.1697702060222,
      "logps/rejected": -364.98613251155626,
      "loss": 0.5539,
      "loss/base_kto": 3.196310520172119,
      "metrics/advantage_var": 497.85693359375,
      "regularization/lipschitz_norm": 1275.878173828125,
      "regularization/w1": 1.2350499629974365,
      "rewards/chosen": 0.5651610747759014,
      "rewards/margins": 0.8654061276366727,
      "rewards/rejected": -0.30024505286077136,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 16.039255142211914,
      "kl": 17.113508224487305,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -34850634.73291925,
      "logits/rejected": -35910920.05031446,
      "logps/chosen": -482.5238742236025,
      "logps/rejected": -383.0166814072327,
      "loss": 0.562,
      "loss/base_kto": 3.234588384628296,
      "metrics/advantage_var": 443.8955993652344,
      "regularization/lipschitz_norm": 1303.3660888671875,
      "regularization/w1": 1.2616583108901978,
      "rewards/chosen": 0.5121984067170516,
      "rewards/margins": 0.7845558675547457,
      "rewards/rejected": -0.2723574608376941,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 16.25307846069336,
      "kl": 14.860127449035645,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34488639.80780781,
      "logits/rejected": -36423980.19543974,
      "logps/chosen": -488.32624812312315,
      "logps/rejected": -366.43907776872965,
      "loss": 0.5473,
      "loss/base_kto": 3.14351749420166,
      "metrics/advantage_var": 442.5341491699219,
      "regularization/lipschitz_norm": 1275.9339599609375,
      "regularization/w1": 1.2351040840148926,
      "rewards/chosen": 0.5904500233876454,
      "rewards/margins": 0.8832285604047718,
      "rewards/rejected": -0.2927785370171264,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 15.692973136901855,
      "kl": 16.274581909179688,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -33666748.283870965,
      "logits/rejected": -35972406.303030305,
      "logps/chosen": -486.7703629032258,
      "logps/rejected": -382.0893465909091,
      "loss": 0.5602,
      "loss/base_kto": 3.148393392562866,
      "metrics/advantage_var": 456.3197326660156,
      "regularization/lipschitz_norm": 1376.9764404296875,
      "regularization/w1": 1.3329132795333862,
      "rewards/chosen": 0.583331544937626,
      "rewards/margins": 0.9028989998825834,
      "rewards/rejected": -0.31956745494495736,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 14.642328262329102,
      "kl": 15.017115592956543,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34592242.87179487,
      "logits/rejected": -36308703.222042136,
      "logps/chosen": -491.4352375565611,
      "logps/rejected": -387.99298521069693,
      "loss": 0.5568,
      "loss/base_kto": 3.1582484245300293,
      "metrics/advantage_var": 435.5256042480469,
      "regularization/lipschitz_norm": 1339.2635498046875,
      "regularization/w1": 1.2964071035385132,
      "rewards/chosen": 0.5732358354249152,
      "rewards/margins": 0.8721980992772247,
      "rewards/rejected": -0.29896226385230956,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 18.6578369140625,
      "kl": 15.550450325012207,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35819927.643312104,
      "logits/rejected": -36710268.073619634,
      "logps/chosen": -490.93197651273886,
      "logps/rejected": -369.9531729294479,
      "loss": 0.5568,
      "loss/base_kto": 3.1798367500305176,
      "metrics/advantage_var": 442.4677734375,
      "regularization/lipschitz_norm": 1317.0311279296875,
      "regularization/w1": 1.274886131286621,
      "rewards/chosen": 0.5815582275390625,
      "rewards/margins": 0.8676361832881998,
      "rewards/rejected": -0.28607795574913725,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 14.409995079040527,
      "kl": 14.426848411560059,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35907821.54231975,
      "logits/rejected": -36522507.16510903,
      "logps/chosen": -483.8010384012539,
      "logps/rejected": -366.04461156542055,
      "loss": 0.5507,
      "loss/base_kto": 3.1684281826019287,
      "metrics/advantage_var": 424.572998046875,
      "regularization/lipschitz_norm": 1278.3421630859375,
      "regularization/w1": 1.237435221672058,
      "rewards/chosen": 0.5785415792913646,
      "rewards/margins": 0.8696636830686232,
      "rewards/rejected": -0.29112210377725856,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 17.155927658081055,
      "kl": 18.289228439331055,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -33932250.074074075,
      "logits/rejected": -35081798.70106222,
      "logps/chosen": -489.66928341384863,
      "logps/rejected": -382.46156581942336,
      "loss": 0.5602,
      "loss/base_kto": 3.201462507247925,
      "metrics/advantage_var": 452.5195007324219,
      "regularization/lipschitz_norm": 1322.109375,
      "regularization/w1": 1.279801845550537,
      "rewards/chosen": 0.5439907694400412,
      "rewards/margins": 0.8164065325888106,
      "rewards/rejected": -0.2724157631487694,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 16.177122116088867,
      "kl": 16.64619255065918,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -34925743.63754045,
      "logits/rejected": -35050489.81268882,
      "logps/chosen": -453.9710254854369,
      "logps/rejected": -389.2960253021148,
      "loss": 0.5608,
      "loss/base_kto": 3.158618688583374,
      "metrics/advantage_var": 503.90240478515625,
      "regularization/lipschitz_norm": 1371.7763671875,
      "regularization/w1": 1.327879548072815,
      "rewards/chosen": 0.5825636286565787,
      "rewards/margins": 0.9102988610991061,
      "rewards/rejected": -0.3277352324425274,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 15.875944137573242,
      "kl": 17.56245994567871,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -34364978.949429035,
      "logits/rejected": -36102045.74512744,
      "logps/chosen": -469.6697593800979,
      "logps/rejected": -350.4022441904048,
      "loss": 0.5606,
      "loss/base_kto": 3.177584409713745,
      "metrics/advantage_var": 445.9990234375,
      "regularization/lipschitz_norm": 1350.1268310546875,
      "regularization/w1": 1.3069227933883667,
      "rewards/chosen": 0.5573854928693414,
      "rewards/margins": 0.8269274034326144,
      "rewards/rejected": -0.26954191056327303,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 14.446081161499023,
      "kl": 14.45959186553955,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35607916.36809816,
      "logits/rejected": -34189116.33121019,
      "logps/chosen": -443.21386119631904,
      "logps/rejected": -357.3136942675159,
      "loss": 0.5593,
      "loss/base_kto": 3.21616792678833,
      "metrics/advantage_var": 415.7434997558594,
      "regularization/lipschitz_norm": 1300.151611328125,
      "regularization/w1": 1.2585468292236328,
      "rewards/chosen": 0.508663294505488,
      "rewards/margins": 0.8264906042779552,
      "rewards/rejected": -0.31782730977246715,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 16.99323081970215,
      "kl": 14.399162292480469,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34767189.87086614,
      "logits/rejected": -36187744.0496124,
      "logps/chosen": -468.33233267716537,
      "logps/rejected": -375.66630329457365,
      "loss": 0.5592,
      "loss/base_kto": 3.196188449859619,
      "metrics/advantage_var": 417.1357421875,
      "regularization/lipschitz_norm": 1319.377197265625,
      "regularization/w1": 1.2771570682525635,
      "rewards/chosen": 0.5302483506090059,
      "rewards/margins": 0.8518294828915882,
      "rewards/rejected": -0.32158113228258234,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.97585917447635e+17,
      "train_loss": 0.5860847269726952,
      "train_runtime": 11055.2237,
      "train_samples_per_second": 13.407,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.97585917447635e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}