{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 18.355131149291992,
      "kl": 8.59229564666748,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35677106.2278481,
      "logits/rejected": -37380674.37037037,
      "logps/chosen": -501.69076344936707,
      "logps/rejected": -375.19931520061726,
      "loss": 0.5987,
      "loss/base_kto": 3.899252414703369,
      "metrics/advantage_var": 239.6805419921875,
      "regularization/mmd": 0.8906540870666504,
      "regularization/rkhs_norm": 171.60964965820312,
      "rewards/chosen": 0.050869072539896906,
      "rewards/margins": 0.09678698972084128,
      "rewards/rejected": -0.04591791718094437,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 14.383731842041016,
      "kl": 18.936019897460938,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35806462.02773498,
      "logits/rejected": -37790854.69413629,
      "logps/chosen": -491.8996533127889,
      "logps/rejected": -360.08897087955626,
      "loss": 0.589,
      "loss/base_kto": 3.903554916381836,
      "metrics/advantage_var": 188.86729431152344,
      "regularization/mmd": 0.8083402514457703,
      "regularization/rkhs_norm": 155.7495574951172,
      "rewards/chosen": 0.26754955885406634,
      "rewards/margins": 0.07950186263619541,
      "rewards/rejected": 0.18804769621787093,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 20.89337921142578,
      "kl": 3.5445687770843506,
      "learning_rate": 5.9e-07,
      "logits/chosen": -34724693.333333336,
      "logits/rejected": -35590390.63414634,
      "logps/chosen": -470.2059294871795,
      "logps/rejected": -364.9865186737805,
      "loss": 0.576,
      "loss/base_kto": 3.835980176925659,
      "metrics/advantage_var": 184.19546508789062,
      "regularization/mmd": 0.7723119258880615,
      "regularization/rkhs_norm": 148.80770874023438,
      "rewards/chosen": 0.042210071514814325,
      "rewards/margins": 0.16762212860054937,
      "rewards/rejected": -0.12541205708573505,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 16.099123001098633,
      "kl": 7.2627739906311035,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37767945.27244095,
      "logits/rejected": -37982627.1255814,
      "logps/chosen": -505.8322342519685,
      "logps/rejected": -385.1265988372093,
      "loss": 0.5774,
      "loss/base_kto": 3.831442356109619,
      "metrics/advantage_var": 175.14170837402344,
      "regularization/mmd": 0.7881031036376953,
      "regularization/rkhs_norm": 151.85031127929688,
      "rewards/chosen": 0.11390412097840798,
      "rewards/margins": 0.16512695915102488,
      "rewards/rejected": -0.051222838172616886,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 15.046182632446289,
      "kl": 9.304082870483398,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36750334.43185299,
      "logits/rejected": -38120995.11323764,
      "logps/chosen": -490.5138782542113,
      "logps/rejected": -373.44328149920256,
      "loss": 0.5912,
      "loss/base_kto": 3.834928274154663,
      "metrics/advantage_var": 241.1676025390625,
      "regularization/mmd": 0.8943428993225098,
      "regularization/rkhs_norm": 172.32041931152344,
      "rewards/chosen": 0.07395970109410892,
      "rewards/margins": 0.18977724413606217,
      "rewards/rejected": -0.11581754304195326,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 14.61628246307373,
      "kl": 12.464029312133789,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -37586362.43053435,
      "logits/rejected": -37443043.328,
      "logps/chosen": -491.85190839694656,
      "logps/rejected": -361.0565,
      "loss": 0.5865,
      "loss/base_kto": 3.87249755859375,
      "metrics/advantage_var": 199.67323303222656,
      "regularization/mmd": 0.8192858099937439,
      "regularization/rkhs_norm": 157.85855102539062,
      "rewards/chosen": 0.2136779086280415,
      "rewards/margins": 0.11438920013194775,
      "rewards/rejected": 0.09928870849609375,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 14.474100112915039,
      "kl": 12.758808135986328,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35743580.66257669,
      "logits/rejected": -38627249.73248408,
      "logps/chosen": -485.7898773006135,
      "logps/rejected": -384.0504329219745,
      "loss": 0.5778,
      "loss/base_kto": 3.8082337379455566,
      "metrics/advantage_var": 186.1129150390625,
      "regularization/mmd": 0.8143102526664734,
      "regularization/rkhs_norm": 156.8998565673828,
      "rewards/chosen": 0.16835581305568204,
      "rewards/margins": 0.155821749722523,
      "rewards/rejected": 0.01253406333315904,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 18.011737823486328,
      "kl": 13.618170738220215,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -37305216.575712144,
      "logits/rejected": -36834078.48613377,
      "logps/chosen": -495.27886056971516,
      "logps/rejected": -373.70554649265904,
      "loss": 0.5813,
      "loss/base_kto": 3.7884018421173096,
      "metrics/advantage_var": 216.3523712158203,
      "regularization/mmd": 0.8617948889732361,
      "regularization/rkhs_norm": 166.049072265625,
      "rewards/chosen": 0.30625135430331707,
      "rewards/margins": 0.18292163163591546,
      "rewards/rejected": 0.12332972266740161,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 17.904626846313477,
      "kl": 17.634511947631836,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35702719.90461997,
      "logits/rejected": -36885959.67159277,
      "logps/chosen": -466.5516952309985,
      "logps/rejected": -359.201867816092,
      "loss": 0.5749,
      "loss/base_kto": 3.762676239013672,
      "metrics/advantage_var": 196.781494140625,
      "regularization/mmd": 0.8366878628730774,
      "regularization/rkhs_norm": 161.21153259277344,
      "rewards/chosen": 0.31675417767907044,
      "rewards/margins": 0.21232553088360656,
      "rewards/rejected": 0.10442864679546388,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 13.703529357910156,
      "kl": 15.65002727508545,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -37641179.537091985,
      "logits/rejected": -39380167.39273927,
      "logps/chosen": -488.34597551928783,
      "logps/rejected": -382.0031456270627,
      "loss": 0.5825,
      "loss/base_kto": 3.7953789234161377,
      "metrics/advantage_var": 218.0191650390625,
      "regularization/mmd": 0.8642555475234985,
      "regularization/rkhs_norm": 166.52322387695312,
      "rewards/chosen": 0.28980716965672526,
      "rewards/margins": 0.1635241472199601,
      "rewards/rejected": 0.12628302243676517,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 24.622936248779297,
      "kl": 11.670283317565918,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37326730.5767285,
      "logits/rejected": -38053563.06259097,
      "logps/chosen": -479.42780354131537,
      "logps/rejected": -380.7495906113537,
      "loss": 0.5852,
      "loss/base_kto": 3.7580573558807373,
      "metrics/advantage_var": 249.5292205810547,
      "regularization/mmd": 0.9234267473220825,
      "regularization/rkhs_norm": 177.92422485351562,
      "rewards/chosen": 0.13914695598222754,
      "rewards/margins": 0.2099495529614937,
      "rewards/rejected": -0.07080259697926618,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 15.743287086486816,
      "kl": 8.136269569396973,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -35933149.9207607,
      "logits/rejected": -36145742.10169491,
      "logps/chosen": -466.4460677496038,
      "logps/rejected": -377.88212634822804,
      "loss": 0.5811,
      "loss/base_kto": 3.7892513275146484,
      "metrics/advantage_var": 212.422607421875,
      "regularization/mmd": 0.8593311309814453,
      "regularization/rkhs_norm": 165.5743865966797,
      "rewards/chosen": 0.029862895064799798,
      "rewards/margins": 0.2177220761441647,
      "rewards/rejected": -0.1878591810793649,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 16.04319953918457,
      "kl": 6.755073547363281,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -35730253.35825545,
      "logits/rejected": -35731314.75862069,
      "logps/chosen": -479.7824182242991,
      "logps/rejected": -379.6119710031348,
      "loss": 0.5825,
      "loss/base_kto": 3.7791366577148438,
      "metrics/advantage_var": 230.1333465576172,
      "regularization/mmd": 0.880833625793457,
      "regularization/rkhs_norm": 169.7174835205078,
      "rewards/chosen": 0.06319318753536617,
      "rewards/margins": 0.20973908798186036,
      "rewards/rejected": -0.14654590044649418,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 14.483595848083496,
      "kl": 8.073613166809082,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -35130597.945072696,
      "logits/rejected": -37963147.03782148,
      "logps/chosen": -505.0274131663974,
      "logps/rejected": -364.7469270045386,
      "loss": 0.5763,
      "loss/base_kto": 3.7302238941192627,
      "metrics/advantage_var": 218.50735473632812,
      "regularization/mmd": 0.880548894405365,
      "regularization/rkhs_norm": 169.66261291503906,
      "rewards/chosen": 0.14740888752729325,
      "rewards/margins": 0.2510804938248656,
      "rewards/rejected": -0.10367160629757233,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 12.326614379882812,
      "kl": 6.938302516937256,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -35278198.035928145,
      "logits/rejected": -37310162.823529415,
      "logps/chosen": -463.1574663173653,
      "logps/rejected": -374.53079044117646,
      "loss": 0.5839,
      "loss/base_kto": 3.781132459640503,
      "metrics/advantage_var": 229.2920379638672,
      "regularization/mmd": 0.8904251456260681,
      "regularization/rkhs_norm": 171.56553649902344,
      "rewards/chosen": 0.06592190337038326,
      "rewards/margins": 0.21471668841493494,
      "rewards/rejected": -0.14879478504455168,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 14.155945777893066,
      "kl": 13.264572143554688,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -36832059.14054927,
      "logits/rejected": -37249687.04387292,
      "logps/chosen": -522.9134693053312,
      "logps/rejected": -394.5444875189107,
      "loss": 0.5837,
      "loss/base_kto": 3.7714180946350098,
      "metrics/advantage_var": 222.8312225341797,
      "regularization/mmd": 0.8983078002929688,
      "regularization/rkhs_norm": 173.0843505859375,
      "rewards/chosen": 0.2278742874958981,
      "rewards/margins": 0.1922227962989959,
      "rewards/rejected": 0.03565149119690218,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 14.930360794067383,
      "kl": 5.002163887023926,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -34289873.38019169,
      "logits/rejected": -35831310.09174312,
      "logps/chosen": -471.3543330670926,
      "logps/rejected": -364.5137136850153,
      "loss": 0.5842,
      "loss/base_kto": 3.77946400642395,
      "metrics/advantage_var": 251.5850067138672,
      "regularization/mmd": 0.8943830728530884,
      "regularization/rkhs_norm": 172.32815551757812,
      "rewards/chosen": 0.019760937736438104,
      "rewards/margins": 0.21380272358031355,
      "rewards/rejected": -0.19404178584387544,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 17.12563133239746,
      "kl": 5.303887367248535,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -34518550.33279483,
      "logits/rejected": -35829217.79122542,
      "logps/chosen": -496.31078352180936,
      "logps/rejected": -365.0048458774584,
      "loss": 0.5784,
      "loss/base_kto": 3.7350640296936035,
      "metrics/advantage_var": 225.28868103027344,
      "regularization/mmd": 0.8921697735786438,
      "regularization/rkhs_norm": 171.90170288085938,
      "rewards/chosen": 0.03668622400533402,
      "rewards/margins": 0.2573953971183966,
      "rewards/rejected": -0.2207091731130626,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 16.593212127685547,
      "kl": 5.176967620849609,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35902288.77864992,
      "logits/rejected": -36061680.87091757,
      "logps/chosen": -482.09399529042383,
      "logps/rejected": -355.24023133748057,
      "loss": 0.5873,
      "loss/base_kto": 3.784255266189575,
      "metrics/advantage_var": 238.3084716796875,
      "regularization/mmd": 0.9138066172599792,
      "regularization/rkhs_norm": 176.0706329345703,
      "rewards/chosen": 0.01627715303909086,
      "rewards/margins": 0.2028148763727077,
      "rewards/rejected": -0.18653772333361684,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 15.823249816894531,
      "kl": 4.430182933807373,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36196202.463492066,
      "logits/rejected": -36116335.064615384,
      "logps/chosen": -482.8965277777778,
      "logps/rejected": -389.58752403846154,
      "loss": 0.5814,
      "loss/base_kto": 3.7527215480804443,
      "metrics/advantage_var": 236.9881134033203,
      "regularization/mmd": 0.8983505368232727,
      "regularization/rkhs_norm": 173.0925750732422,
      "rewards/chosen": -0.05178233555385044,
      "rewards/margins": 0.2334475322346111,
      "rewards/rejected": -0.28522986778846154,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 13.52650260925293,
      "kl": 8.73814582824707,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -37322299.760252364,
      "logits/rejected": -38061487.15789474,
      "logps/chosen": -478.4477523659306,
      "logps/rejected": -358.9422407120743,
      "loss": 0.5795,
      "loss/base_kto": 3.7514564990997314,
      "metrics/advantage_var": 219.9439697265625,
      "regularization/mmd": 0.8847070932388306,
      "regularization/rkhs_norm": 170.4637908935547,
      "rewards/chosen": 0.07475940210962145,
      "rewards/margins": 0.2192280343854656,
      "rewards/rejected": -0.14446863227584414,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 14.851932525634766,
      "kl": 10.129216194152832,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -35613759.209876545,
      "logits/rejected": -36460612.05063291,
      "logps/chosen": -454.60947145061726,
      "logps/rejected": -376.8286194620253,
      "loss": 0.5739,
      "loss/base_kto": 3.726673126220703,
      "metrics/advantage_var": 211.911865234375,
      "regularization/mmd": 0.8641713261604309,
      "regularization/rkhs_norm": 166.5070037841797,
      "rewards/chosen": 0.17373162728768807,
      "rewards/margins": 0.23917970017988915,
      "rewards/rejected": -0.0654480728922011,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 16.98316764831543,
      "kl": 11.698954582214355,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35815370.845801525,
      "logits/rejected": -36148592.64,
      "logps/chosen": -465.42123091603054,
      "logps/rejected": -356.472725,
      "loss": 0.578,
      "loss/base_kto": 3.681044816970825,
      "metrics/advantage_var": 263.0597839355469,
      "regularization/mmd": 0.9432243704795837,
      "regularization/rkhs_norm": 181.73878479003906,
      "rewards/chosen": 0.2568790348431536,
      "rewards/margins": 0.3083677311322161,
      "rewards/rejected": -0.0514886962890625,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 18.69508934020996,
      "kl": 14.229891777038574,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -37249244.21505377,
      "logits/rejected": -37633168.89030207,
      "logps/chosen": -494.7801459293395,
      "logps/rejected": -388.1153616852146,
      "loss": 0.5859,
      "loss/base_kto": 3.6975953578948975,
      "metrics/advantage_var": 285.19891357421875,
      "regularization/mmd": 0.9894009828567505,
      "regularization/rkhs_norm": 190.6360321044922,
      "rewards/chosen": 0.29050962386592744,
      "rewards/margins": 0.29268515360062536,
      "rewards/rejected": -0.0021755297346979334,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 13.925016403198242,
      "kl": 16.193500518798828,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35656988.706605226,
      "logits/rejected": -37467951.61844197,
      "logps/chosen": -465.0112807219662,
      "logps/rejected": -384.5026331478537,
      "loss": 0.5786,
      "loss/base_kto": 3.7520835399627686,
      "metrics/advantage_var": 226.3187255859375,
      "regularization/mmd": 0.876621425151825,
      "regularization/rkhs_norm": 168.90586853027344,
      "rewards/chosen": 0.2240577381327405,
      "rewards/margins": 0.21612428372532297,
      "rewards/rejected": 0.007933454407417528,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 15.746620178222656,
      "kl": 9.902660369873047,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36418136.83280757,
      "logits/rejected": -37338102.489164084,
      "logps/chosen": -465.4365634858044,
      "logps/rejected": -379.03957043343655,
      "loss": 0.5871,
      "loss/base_kto": 3.785489797592163,
      "metrics/advantage_var": 242.4727783203125,
      "regularization/mmd": 0.9116144180297852,
      "regularization/rkhs_norm": 175.6482391357422,
      "rewards/chosen": 0.1177103060653157,
      "rewards/margins": 0.18170398443876468,
      "rewards/rejected": -0.06399367837344899,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 15.057689666748047,
      "kl": 15.38354206085205,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36851674.59360731,
      "logits/rejected": -36642947.492776886,
      "logps/chosen": -500.0726788432268,
      "logps/rejected": -359.93649678972713,
      "loss": 0.5677,
      "loss/base_kto": 3.6362855434417725,
      "metrics/advantage_var": 236.0905303955078,
      "regularization/mmd": 0.9051849246025085,
      "regularization/rkhs_norm": 174.40945434570312,
      "rewards/chosen": 0.3106789784888699,
      "rewards/margins": 0.3464053420367936,
      "rewards/rejected": -0.03572636354792367,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 14.786884307861328,
      "kl": 16.252817153930664,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36130636.29652043,
      "logits/rejected": -36427646.96607431,
      "logps/chosen": -496.38705559757943,
      "logps/rejected": -385.2689317447496,
      "loss": 0.5856,
      "loss/base_kto": 3.7102839946746826,
      "metrics/advantage_var": 268.5868225097656,
      "regularization/mmd": 0.9748560190200806,
      "regularization/rkhs_norm": 187.8335418701172,
      "rewards/chosen": 0.3056913020931827,
      "rewards/margins": 0.2599889825551094,
      "rewards/rejected": 0.04570231953807331,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 14.099896430969238,
      "kl": 19.939496994018555,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -36829003.85185185,
      "logits/rejected": -37800435.037974685,
      "logps/chosen": -504.62191358024694,
      "logps/rejected": -367.6693285205696,
      "loss": 0.5723,
      "loss/base_kto": 3.695143938064575,
      "metrics/advantage_var": 217.6491241455078,
      "regularization/mmd": 0.8835013508796692,
      "regularization/rkhs_norm": 170.23147583007812,
      "rewards/chosen": 0.3421223251907914,
      "rewards/margins": 0.23950373349589918,
      "rewards/rejected": 0.10261859169489221,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 16.136192321777344,
      "kl": 9.32910442352295,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36956643.55555555,
      "logits/rejected": -37741010.63291139,
      "logps/chosen": -524.8196855709876,
      "logps/rejected": -366.2413963607595,
      "loss": 0.5775,
      "loss/base_kto": 3.747807264328003,
      "metrics/advantage_var": 220.1378173828125,
      "regularization/mmd": 0.8723760843276978,
      "regularization/rkhs_norm": 168.087890625,
      "rewards/chosen": 0.1329013447702667,
      "rewards/margins": 0.23484869460833036,
      "rewards/rejected": -0.10194734983806368,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 21.093189239501953,
      "kl": 10.337899208068848,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -34430201.952755906,
      "logits/rejected": -36404139.85736434,
      "logps/chosen": -476.6104330708661,
      "logps/rejected": -383.88822674418606,
      "loss": 0.5831,
      "loss/base_kto": 3.7550950050354004,
      "metrics/advantage_var": 256.0594787597656,
      "regularization/mmd": 0.9093719720840454,
      "regularization/rkhs_norm": 175.21617126464844,
      "rewards/chosen": 0.10829953171136811,
      "rewards/margins": 0.18434801160477896,
      "rewards/rejected": -0.07604847989341085,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 16.082395553588867,
      "kl": 8.053885459899902,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35425285.04433498,
      "logits/rejected": -37677919.76154993,
      "logps/chosen": -481.14706486042695,
      "logps/rejected": -369.09719634873323,
      "loss": 0.5824,
      "loss/base_kto": 3.744581937789917,
      "metrics/advantage_var": 239.15542602539062,
      "regularization/mmd": 0.9146170020103455,
      "regularization/rkhs_norm": 176.22679138183594,
      "rewards/chosen": 0.05715729411208179,
      "rewards/margins": 0.23221122745041195,
      "rewards/rejected": -0.17505393333833016,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 14.41285228729248,
      "kl": 9.332486152648926,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -34698492.01246106,
      "logits/rejected": -35375726.7460815,
      "logps/chosen": -454.7192367601246,
      "logps/rejected": -385.7163989028213,
      "loss": 0.5881,
      "loss/base_kto": 3.765181064605713,
      "metrics/advantage_var": 250.76968383789062,
      "regularization/mmd": 0.9399226307868958,
      "regularization/rkhs_norm": 181.10263061523438,
      "rewards/chosen": 0.11088201932817976,
      "rewards/margins": 0.2165571336877813,
      "rewards/rejected": -0.10567511435960154,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 14.423428535461426,
      "kl": 14.862100601196289,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35039565.84326019,
      "logits/rejected": -36648494.25545172,
      "logps/chosen": -494.4254506269593,
      "logps/rejected": -377.2711253894081,
      "loss": 0.5786,
      "loss/base_kto": 3.728292942047119,
      "metrics/advantage_var": 232.7931671142578,
      "regularization/mmd": 0.9003773927688599,
      "regularization/rkhs_norm": 173.4831085205078,
      "rewards/chosen": 0.25897384213055935,
      "rewards/margins": 0.22975997244746302,
      "rewards/rejected": 0.029213869683096343,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 14.714305877685547,
      "kl": 10.654457092285156,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -35113917.88346457,
      "logits/rejected": -36209268.68837209,
      "logps/chosen": -488.20344488188977,
      "logps/rejected": -345.6830910852713,
      "loss": 0.5819,
      "loss/base_kto": 3.7753243446350098,
      "metrics/advantage_var": 226.36376953125,
      "regularization/mmd": 0.8800527453422546,
      "regularization/rkhs_norm": 169.56700134277344,
      "rewards/chosen": 0.08953093280942422,
      "rewards/margins": 0.18774708573986876,
      "rewards/rejected": -0.09821615293044453,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 16.04747772216797,
      "kl": 10.848597526550293,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36346454.25656878,
      "logits/rejected": -37313227.02053712,
      "logps/chosen": -500.8554385625966,
      "logps/rejected": -379.5542061611374,
      "loss": 0.5857,
      "loss/base_kto": 3.7539193630218506,
      "metrics/advantage_var": 289.1528015136719,
      "regularization/mmd": 0.9315635561943054,
      "regularization/rkhs_norm": 179.4920196533203,
      "rewards/chosen": 0.12201917079353627,
      "rewards/margins": 0.21513198979623918,
      "rewards/rejected": -0.0931128190027029,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 17.332683563232422,
      "kl": 12.433494567871094,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36929741.46278317,
      "logits/rejected": -37516421.027190335,
      "logps/chosen": -492.72314927184465,
      "logps/rejected": -386.1748017371601,
      "loss": 0.5809,
      "loss/base_kto": 3.727473020553589,
      "metrics/advantage_var": 243.08250427246094,
      "regularization/mmd": 0.9198707938194275,
      "regularization/rkhs_norm": 177.2390594482422,
      "rewards/chosen": 0.15290815236113217,
      "rewards/margins": 0.2302717687513028,
      "rewards/rejected": -0.07736361639017064,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 14.800250053405762,
      "kl": 8.272868156433105,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35740746.76825397,
      "logits/rejected": -37316790.74461538,
      "logps/chosen": -477.70992063492065,
      "logps/rejected": -360.88326923076926,
      "loss": 0.5816,
      "loss/base_kto": 3.7273449897766113,
      "metrics/advantage_var": 246.2641143798828,
      "regularization/mmd": 0.9251508712768555,
      "regularization/rkhs_norm": 178.2564239501953,
      "rewards/chosen": 0.00815635635739281,
      "rewards/margins": 0.2315292529920082,
      "rewards/rejected": -0.22337289663461538,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 8.796030044555664,
      "kl": 6.298320293426514,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -34560059.65048543,
      "logits/rejected": -35027130.18181818,
      "logps/chosen": -467.2739178802589,
      "logps/rejected": -387.8612215909091,
      "loss": 0.5722,
      "loss/base_kto": 3.5254478454589844,
      "metrics/advantage_var": 362.2047424316406,
      "regularization/mmd": 1.0523730516433716,
      "regularization/rkhs_norm": 202.76939392089844,
      "rewards/chosen": 0.14243385784062754,
      "rewards/margins": 0.4932251647697824,
      "rewards/rejected": -0.35079130692915483,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 16.780546188354492,
      "kl": 10.918546676635742,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -35739378.27871363,
      "logits/rejected": -36621637.00159489,
      "logps/chosen": -463.666586906585,
      "logps/rejected": -387.4616726475279,
      "loss": 0.5696,
      "loss/base_kto": 3.2663350105285645,
      "metrics/advantage_var": 559.0786743164062,
      "regularization/mmd": 1.2902926206588745,
      "regularization/rkhs_norm": 248.61131286621094,
      "rewards/chosen": 0.449957340818817,
      "rewards/margins": 0.8422214073294205,
      "rewards/rejected": -0.3922640665106036,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 13.12932014465332,
      "kl": 18.999629974365234,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -34947555.018867925,
      "logits/rejected": -36489925.16770186,
      "logps/chosen": -442.7131977201258,
      "logps/rejected": -391.82603843167703,
      "loss": 0.5645,
      "loss/base_kto": 3.246511220932007,
      "metrics/advantage_var": 518.4884643554688,
      "regularization/mmd": 1.2694826126098633,
      "regularization/rkhs_norm": 244.60166931152344,
      "rewards/chosen": 0.5303995384360259,
      "rewards/margins": 0.7891794314463254,
      "rewards/rejected": -0.2587798930102994,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 9.967662811279297,
      "kl": 6.526576519012451,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35417250.03588907,
      "logits/rejected": -35428815.64017991,
      "logps/chosen": -511.6195962479608,
      "logps/rejected": -366.17344452773614,
      "loss": 0.5673,
      "loss/base_kto": 3.1759274005889893,
      "metrics/advantage_var": 602.5060424804688,
      "regularization/mmd": 1.3626421689987183,
      "regularization/rkhs_norm": 262.5514831542969,
      "rewards/chosen": 0.3481656894403803,
      "rewards/margins": 0.9205143112720425,
      "rewards/rejected": -0.5723486218316622,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 11.86622142791748,
      "kl": 8.370628356933594,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -36123243.83151326,
      "logits/rejected": -36291444.58215962,
      "logps/chosen": -461.5860471918877,
      "logps/rejected": -374.5848982785603,
      "loss": 0.5643,
      "loss/base_kto": 3.240628957748413,
      "metrics/advantage_var": 543.981689453125,
      "regularization/mmd": 1.2740906476974487,
      "regularization/rkhs_norm": 245.48953247070312,
      "rewards/chosen": 0.3063705849015211,
      "rewards/margins": 0.820691638992997,
      "rewards/rejected": -0.514321054091476,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 10.228275299072266,
      "kl": 20.769678115844727,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35856915.54198473,
      "logits/rejected": -36818753.9456,
      "logps/chosen": -459.0350190839695,
      "logps/rejected": -369.253525,
      "loss": 0.5539,
      "loss/base_kto": 3.094128370285034,
      "metrics/advantage_var": 562.14501953125,
      "regularization/mmd": 1.3369877338409424,
      "regularization/rkhs_norm": 257.6084289550781,
      "rewards/chosen": 0.6964062872733778,
      "rewards/margins": 0.9727445929374403,
      "rewards/rejected": -0.2763383056640625,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 12.683265686035156,
      "kl": 13.78448486328125,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -37744494.59818731,
      "logits/rejected": -37460488.284789644,
      "logps/chosen": -507.40563632930514,
      "logps/rejected": -383.04634405339806,
      "loss": 0.5758,
      "loss/base_kto": 3.207179307937622,
      "metrics/advantage_var": 653.6361694335938,
      "regularization/mmd": 1.3994256258010864,
      "regularization/rkhs_norm": 269.63885498046875,
      "rewards/chosen": 0.5503196140070337,
      "rewards/margins": 0.8896718235982048,
      "rewards/rejected": -0.33935220959117113,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 11.519844055175781,
      "kl": 10.403990745544434,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -35192848.84210526,
      "logits/rejected": -35591631.23809524,
      "logps/chosen": -488.888671875,
      "logps/rejected": -376.9961867559524,
      "loss": 0.5679,
      "loss/base_kto": 3.162940263748169,
      "metrics/advantage_var": 678.7843627929688,
      "regularization/mmd": 1.3800307512283325,
      "regularization/rkhs_norm": 265.9018859863281,
      "rewards/chosen": 0.44200319992868525,
      "rewards/margins": 0.9566270689617722,
      "rewards/rejected": -0.5146238690330869,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 14.86784839630127,
      "kl": 12.996451377868652,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -36329521.23076923,
      "logits/rejected": -37399982.82926829,
      "logps/chosen": -488.44190705128204,
      "logps/rejected": -372.8462271341463,
      "loss": 0.5731,
      "loss/base_kto": 3.1849658489227295,
      "metrics/advantage_var": 616.439453125,
      "regularization/mmd": 1.400101900100708,
      "regularization/rkhs_norm": 269.7691345214844,
      "rewards/chosen": 0.5117607116699219,
      "rewards/margins": 0.909477513010909,
      "rewards/rejected": -0.397716801340987,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 12.045408248901367,
      "kl": 11.645041465759277,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36871137.835658915,
      "logits/rejected": -38185995.28818898,
      "logps/chosen": -492.06763565891475,
      "logps/rejected": -380.04018208661415,
      "loss": 0.5669,
      "loss/base_kto": 3.208451986312866,
      "metrics/advantage_var": 539.9119873046875,
      "regularization/mmd": 1.3271362781524658,
      "regularization/rkhs_norm": 255.7102508544922,
      "rewards/chosen": 0.45313749091569766,
      "rewards/margins": 0.8564526610836356,
      "rewards/rejected": -0.403315170167938,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 13.22069263458252,
      "kl": 12.721824645996094,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35281307.51401869,
      "logits/rejected": -36470151.623824455,
      "logps/chosen": -474.9979556074766,
      "logps/rejected": -394.079006661442,
      "loss": 0.5703,
      "loss/base_kto": 3.217174530029297,
      "metrics/advantage_var": 539.5971069335938,
      "regularization/mmd": 1.3451206684112549,
      "regularization/rkhs_norm": 259.17547607421875,
      "rewards/chosen": 0.49599230995059385,
      "rewards/margins": 0.8761570273826937,
      "rewards/rejected": -0.38016471743209984,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 13.768370628356934,
      "kl": 14.06915283203125,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -34588707.5892575,
      "logits/rejected": -34524313.52086553,
      "logps/chosen": -470.2716725908373,
      "logps/rejected": -405.7633307573416,
      "loss": 0.5818,
      "loss/base_kto": 3.2592437267303467,
      "metrics/advantage_var": 650.3941040039062,
      "regularization/mmd": 1.3952311277389526,
      "regularization/rkhs_norm": 268.8306579589844,
      "rewards/chosen": 0.43066994424491506,
      "rewards/margins": 0.8314947018400658,
      "rewards/rejected": -0.4008247575951507,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 15.858077049255371,
      "kl": 17.79615592956543,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35441875.54491018,
      "logits/rejected": -35059216.732026145,
      "logps/chosen": -476.67215568862275,
      "logps/rejected": -385.0362796160131,
      "loss": 0.5624,
      "loss/base_kto": 3.196995496749878,
      "metrics/advantage_var": 541.07373046875,
      "regularization/mmd": 1.3020113706588745,
      "regularization/rkhs_norm": 250.86924743652344,
      "rewards/chosen": 0.5837789295676226,
      "rewards/margins": 0.8488349412911596,
      "rewards/rejected": -0.26505601172353704,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 14.126097679138184,
      "kl": 20.838333129882812,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -34667620.09022556,
      "logits/rejected": -36130579.56422764,
      "logps/chosen": -509.91494360902254,
      "logps/rejected": -364.2094004065041,
      "loss": 0.5678,
      "loss/base_kto": 3.2881786823272705,
      "metrics/advantage_var": 461.0966796875,
      "regularization/mmd": 1.2540638446807861,
      "regularization/rkhs_norm": 241.6307830810547,
      "rewards/chosen": 0.5731052943638393,
      "rewards/margins": 0.7670012204904589,
      "rewards/rejected": -0.19389592612661966,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 13.985225677490234,
      "kl": 16.84873390197754,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35178023.133757964,
      "logits/rejected": -35479306.99386503,
      "logps/chosen": -491.51462977707007,
      "logps/rejected": -377.34854294478527,
      "loss": 0.5766,
      "loss/base_kto": 3.1906678676605225,
      "metrics/advantage_var": 649.16259765625,
      "regularization/mmd": 1.4221806526184082,
      "regularization/rkhs_norm": 274.0232849121094,
      "rewards/chosen": 0.6344547636190038,
      "rewards/margins": 0.9031659941251986,
      "rewards/rejected": -0.2687112305061949,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 13.469324111938477,
      "kl": 10.265970230102539,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -36038125.278219394,
      "logits/rejected": -37385028.817204304,
      "logps/chosen": -503.97322138314786,
      "logps/rejected": -367.8606950844854,
      "loss": 0.5608,
      "loss/base_kto": 3.1485276222229004,
      "metrics/advantage_var": 526.8321533203125,
      "regularization/mmd": 1.3380202054977417,
      "regularization/rkhs_norm": 257.8074035644531,
      "rewards/chosen": 0.47844770034280604,
      "rewards/margins": 0.9144246341165676,
      "rewards/rejected": -0.4359769337737615,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 9.046934127807617,
      "kl": 16.51395034790039,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34729330.04361371,
      "logits/rejected": -36002616.97805642,
      "logps/chosen": -469.17416277258565,
      "logps/rejected": -377.32381465517244,
      "loss": 0.5561,
      "loss/base_kto": 3.212134599685669,
      "metrics/advantage_var": 491.69171142578125,
      "regularization/mmd": 1.2363241910934448,
      "regularization/rkhs_norm": 238.21275329589844,
      "rewards/chosen": 0.4971886750693633,
      "rewards/margins": 0.8227839687422179,
      "rewards/rejected": -0.3255952936728546,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 13.699871063232422,
      "kl": 12.540992736816406,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35314302.55250404,
      "logits/rejected": -36094472.5204236,
      "logps/chosen": -469.674071082391,
      "logps/rejected": -378.0487660741301,
      "loss": 0.56,
      "loss/base_kto": 3.1778485774993896,
      "metrics/advantage_var": 510.32958984375,
      "regularization/mmd": 1.3021544218063354,
      "regularization/rkhs_norm": 250.8968048095703,
      "rewards/chosen": 0.4937756958993841,
      "rewards/margins": 0.8691603512732848,
      "rewards/rejected": -0.3753846553739008,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 13.748052597045898,
      "kl": 7.949827671051025,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34263613.04928458,
      "logits/rejected": -35533221.55453149,
      "logps/chosen": -486.01952503974564,
      "logps/rejected": -376.6563220046083,
      "loss": 0.5637,
      "loss/base_kto": 3.226621389389038,
      "metrics/advantage_var": 541.7657470703125,
      "regularization/mmd": 1.2831403017044067,
      "regularization/rkhs_norm": 247.2332000732422,
      "rewards/chosen": 0.3904226330391743,
      "rewards/margins": 0.8421150882313067,
      "rewards/rejected": -0.4516924551921323,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 16.558958053588867,
      "kl": 7.912838935852051,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34892940.613893375,
      "logits/rejected": -36078274.42057489,
      "logps/chosen": -483.1467588852989,
      "logps/rejected": -374.8048411497731,
      "loss": 0.5645,
      "loss/base_kto": 3.155876636505127,
      "metrics/advantage_var": 615.82958984375,
      "regularization/mmd": 1.3601619005203247,
      "regularization/rkhs_norm": 262.0735778808594,
      "rewards/chosen": 0.43474848782688813,
      "rewards/margins": 0.9497797570331098,
      "rewards/rejected": -0.5150312692062217,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 13.430770874023438,
      "kl": 13.892699241638184,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34218263.87138264,
      "logits/rejected": -35506754.91793313,
      "logps/chosen": -497.61223874598073,
      "logps/rejected": -345.22314304711244,
      "loss": 0.5528,
      "loss/base_kto": 3.1559383869171143,
      "metrics/advantage_var": 486.2275390625,
      "regularization/mmd": 1.2661854028701782,
      "regularization/rkhs_norm": 243.9663543701172,
      "rewards/chosen": 0.552090512977919,
      "rewards/margins": 0.8996528912786409,
      "rewards/rejected": -0.3475623783007219,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 15.6405668258667,
      "kl": 14.188281059265137,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34322806.30014859,
      "logits/rejected": -37109854.47116969,
      "logps/chosen": -503.85944465081724,
      "logps/rejected": -364.56785420098845,
      "loss": 0.5777,
      "loss/base_kto": 3.1833126544952393,
      "metrics/advantage_var": 741.0894165039062,
      "regularization/mmd": 1.4379055500030518,
      "regularization/rkhs_norm": 277.0531005859375,
      "rewards/chosen": 0.5156612311217265,
      "rewards/margins": 0.8787353688266919,
      "rewards/rejected": -0.3630741377049655,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 28.624948501586914,
      "kl": 12.644123077392578,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34743139.68847352,
      "logits/rejected": -36302250.93416928,
      "logps/chosen": -463.32496105919006,
      "logps/rejected": -376.3301332288401,
      "loss": 0.5646,
      "loss/base_kto": 3.245013475418091,
      "metrics/advantage_var": 545.5440673828125,
      "regularization/mmd": 1.271781325340271,
      "regularization/rkhs_norm": 245.04458618164062,
      "rewards/chosen": 0.5009703829281056,
      "rewards/margins": 0.8233771158520378,
      "rewards/rejected": -0.32240673292393224,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 11.098860740661621,
      "kl": 13.221571922302246,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34350759.41587301,
      "logits/rejected": -36409989.907692306,
      "logps/chosen": -507.50555555555553,
      "logps/rejected": -376.8789903846154,
      "loss": 0.5695,
      "loss/base_kto": 3.155639171600342,
      "metrics/advantage_var": 640.3905029296875,
      "regularization/mmd": 1.4001741409301758,
      "regularization/rkhs_norm": 269.7830810546875,
      "rewards/chosen": 0.5849505227709574,
      "rewards/margins": 0.9541778881180127,
      "rewards/rejected": -0.3692273653470553,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 13.457465171813965,
      "kl": 13.062397956848145,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -34188224.50874404,
      "logits/rejected": -35642448.22119816,
      "logps/chosen": -482.57839825119237,
      "logps/rejected": -355.75254416282644,
      "loss": 0.5626,
      "loss/base_kto": 3.2334468364715576,
      "metrics/advantage_var": 501.9878845214844,
      "regularization/mmd": 1.2672839164733887,
      "regularization/rkhs_norm": 244.1780242919922,
      "rewards/chosen": 0.4810851662640352,
      "rewards/margins": 0.8089145067168242,
      "rewards/rejected": -0.327829340452789,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 9.959589958190918,
      "kl": 10.90673828125,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -35522215.98744113,
      "logits/rejected": -36263321.281493,
      "logps/chosen": -472.88196624803766,
      "logps/rejected": -369.7211314152411,
      "loss": 0.5723,
      "loss/base_kto": 3.2004776000976562,
      "metrics/advantage_var": 601.2073364257812,
      "regularization/mmd": 1.3777668476104736,
      "regularization/rkhs_norm": 265.4656677246094,
      "rewards/chosen": 0.44117609859448587,
      "rewards/margins": 0.8610656999504345,
      "rewards/rejected": -0.41988960135594866,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 10.555566787719727,
      "kl": 14.242239952087402,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33768006.42967542,
      "logits/rejected": -35118029.85150079,
      "logps/chosen": -469.5661707882535,
      "logps/rejected": -368.74012638230647,
      "loss": 0.5527,
      "loss/base_kto": 3.1599979400634766,
      "metrics/advantage_var": 506.82470703125,
      "regularization/mmd": 1.2614637613296509,
      "regularization/rkhs_norm": 243.0565948486328,
      "rewards/chosen": 0.531270942511109,
      "rewards/margins": 0.8937158758796171,
      "rewards/rejected": -0.3624449333685081,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 11.619686126708984,
      "kl": 9.854671478271484,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -35102347.63636363,
      "logits/rejected": -35890413.4939759,
      "logps/chosen": -484.1734983766234,
      "logps/rejected": -387.3549510542169,
      "loss": 0.5709,
      "loss/base_kto": 3.2288010120391846,
      "metrics/advantage_var": 594.1427612304688,
      "regularization/mmd": 1.3385165929794312,
      "regularization/rkhs_norm": 257.90301513671875,
      "rewards/chosen": 0.46015707238928066,
      "rewards/margins": 0.8614431401221261,
      "rewards/rejected": -0.4012860677328454,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 15.014073371887207,
      "kl": 9.039198875427246,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -35080624.8085758,
      "logits/rejected": -36249443.215311006,
      "logps/chosen": -483.1822358346095,
      "logps/rejected": -375.1662679425837,
      "loss": 0.5656,
      "loss/base_kto": 3.224367618560791,
      "metrics/advantage_var": 550.4187622070312,
      "regularization/mmd": 1.300495982170105,
      "regularization/rkhs_norm": 250.57725524902344,
      "rewards/chosen": 0.4142883674656633,
      "rewards/margins": 0.865131606600432,
      "rewards/rejected": -0.45084323913476876,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 11.960787773132324,
      "kl": 13.760507583618164,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -34482436.711656444,
      "logits/rejected": -35953145.477707006,
      "logps/chosen": -483.8002300613497,
      "logps/rejected": -383.39570063694265,
      "loss": 0.5548,
      "loss/base_kto": 3.189375638961792,
      "metrics/advantage_var": 481.8376159667969,
      "regularization/mmd": 1.249145269393921,
      "regularization/rkhs_norm": 240.68309020996094,
      "rewards/chosen": 0.49734628127396474,
      "rewards/margins": 0.8299500031355914,
      "rewards/rejected": -0.3326037218616267,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 15.612455368041992,
      "kl": 18.639978408813477,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35105800.03767661,
      "logits/rejected": -35919365.57387247,
      "logps/chosen": -473.2154140502355,
      "logps/rejected": -392.6224241835148,
      "loss": 0.5577,
      "loss/base_kto": 3.197514057159424,
      "metrics/advantage_var": 516.6060791015625,
      "regularization/mmd": 1.2640329599380493,
      "regularization/rkhs_norm": 243.5516357421875,
      "rewards/chosen": 0.5329540019132653,
      "rewards/margins": 0.8422518394272378,
      "rewards/rejected": -0.3092978375139726,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 13.539543151855469,
      "kl": 12.182269096374512,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -35346172.52488688,
      "logits/rejected": -35774855.67585089,
      "logps/chosen": -473.17444381598796,
      "logps/rejected": -350.419545178282,
      "loss": 0.5706,
      "loss/base_kto": 3.2329118251800537,
      "metrics/advantage_var": 542.537109375,
      "regularization/mmd": 1.3316736221313477,
      "regularization/rkhs_norm": 256.58453369140625,
      "rewards/chosen": 0.5181275334653092,
      "rewards/margins": 0.8476684600600974,
      "rewards/rejected": -0.3295409265947883,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 10.709891319274902,
      "kl": 9.94327449798584,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -35305236.82296651,
      "logits/rejected": -35859495.203675345,
      "logps/chosen": -484.4208034290271,
      "logps/rejected": -359.55044027565083,
      "loss": 0.5594,
      "loss/base_kto": 3.2328438758850098,
      "metrics/advantage_var": 489.0275573730469,
      "regularization/mmd": 1.242189645767212,
      "regularization/rkhs_norm": 239.3428955078125,
      "rewards/chosen": 0.4519747257993172,
      "rewards/margins": 0.8191801250832711,
      "rewards/rejected": -0.36720539928395385,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 13.026509284973145,
      "kl": 14.385235786437988,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -33401099.403993856,
      "logits/rejected": -34391699.33227345,
      "logps/chosen": -475.46548579109066,
      "logps/rejected": -352.3910472972973,
      "loss": 0.5597,
      "loss/base_kto": 3.1858298778533936,
      "metrics/advantage_var": 524.2139892578125,
      "regularization/mmd": 1.2919800281524658,
      "regularization/rkhs_norm": 248.93643188476562,
      "rewards/chosen": 0.5690245738227246,
      "rewards/margins": 0.8721156503082175,
      "rewards/rejected": -0.3030910764854928,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 9.928730010986328,
      "kl": 13.804104804992676,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34858126.6625387,
      "logits/rejected": -35866927.646687694,
      "logps/chosen": -468.6269349845201,
      "logps/rejected": -394.26695583596216,
      "loss": 0.5585,
      "loss/base_kto": 3.2086141109466553,
      "metrics/advantage_var": 477.46563720703125,
      "regularization/mmd": 1.2596110105514526,
      "regularization/rkhs_norm": 242.69961547851562,
      "rewards/chosen": 0.5428633084607198,
      "rewards/margins": 0.849956281947202,
      "rewards/rejected": -0.3070929734864822,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 15.070842742919922,
      "kl": 11.10472297668457,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -34750306.461538464,
      "logits/rejected": -35542640.15238095,
      "logps/chosen": -494.4030769230769,
      "logps/rejected": -391.96760912698414,
      "loss": 0.5614,
      "loss/base_kto": 3.1633832454681396,
      "metrics/advantage_var": 531.6734619140625,
      "regularization/mmd": 1.3275425434112549,
      "regularization/rkhs_norm": 255.78855895996094,
      "rewards/chosen": 0.5454000150240385,
      "rewards/margins": 0.928437255188659,
      "rewards/rejected": -0.38303724016462054,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 14.644248008728027,
      "kl": 13.015934944152832,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35368389.299363054,
      "logits/rejected": -37573644.564417176,
      "logps/chosen": -474.7845342356688,
      "logps/rejected": -372.0687787576687,
      "loss": 0.5603,
      "loss/base_kto": 3.153552293777466,
      "metrics/advantage_var": 546.5127563476562,
      "regularization/mmd": 1.3292239904403687,
      "regularization/rkhs_norm": 256.1125183105469,
      "rewards/chosen": 0.5453437270632215,
      "rewards/margins": 0.9080075577794073,
      "rewards/rejected": -0.3626638307161858,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 13.781595230102539,
      "kl": 9.892565727233887,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -34171076.46511628,
      "logits/rejected": -35169127.783783786,
      "logps/chosen": -478.1899527616279,
      "logps/rejected": -376.4047719594595,
      "loss": 0.565,
      "loss/base_kto": 3.2333877086639404,
      "metrics/advantage_var": 520.9701538085938,
      "regularization/mmd": 1.2864378690719604,
      "regularization/rkhs_norm": 247.8685760498047,
      "rewards/chosen": 0.49371550803960756,
      "rewards/margins": 0.8383285575209737,
      "rewards/rejected": -0.3446130494813661,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 14.64671802520752,
      "kl": 12.811894416809082,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -36281189.3418124,
      "logits/rejected": -36737571.391705066,
      "logps/chosen": -486.2001689189189,
      "logps/rejected": -389.9088421658986,
      "loss": 0.5675,
      "loss/base_kto": 3.20735239982605,
      "metrics/advantage_var": 596.3021240234375,
      "regularization/mmd": 1.3325361013412476,
      "regularization/rkhs_norm": 256.7507019042969,
      "rewards/chosen": 0.5178320176650686,
      "rewards/margins": 0.8580788950277198,
      "rewards/rejected": -0.3402468773626512,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 10.71275520324707,
      "kl": 11.482906341552734,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34089391.686274506,
      "logits/rejected": -35108587.24324324,
      "logps/chosen": -487.71920955882354,
      "logps/rejected": -377.3284769144144,
      "loss": 0.5351,
      "loss/base_kto": 3.1168744564056396,
      "metrics/advantage_var": 399.5955505371094,
      "regularization/mmd": 1.1641170978546143,
      "regularization/rkhs_norm": 224.3000030517578,
      "rewards/chosen": 0.5034692303027982,
      "rewards/margins": 0.9092341566060547,
      "rewards/rejected": -0.4057649263032564,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 11.445816040039062,
      "kl": 10.277003288269043,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34275630.51107828,
      "logits/rejected": -34560315.86069652,
      "logps/chosen": -459.00415435745936,
      "logps/rejected": -382.97128938640134,
      "loss": 0.5324,
      "loss/base_kto": 3.1620447635650635,
      "metrics/advantage_var": 355.1131286621094,
      "regularization/mmd": 1.0971364974975586,
      "regularization/rkhs_norm": 211.3943328857422,
      "rewards/chosen": 0.47600524992499077,
      "rewards/margins": 0.8982038982037791,
      "rewards/rejected": -0.42219864827878834,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 10.892109870910645,
      "kl": 17.682931900024414,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33588075.87421384,
      "logits/rejected": -36002466.18633541,
      "logps/chosen": -497.57596305031444,
      "logps/rejected": -361.7705017468944,
      "loss": 0.5269,
      "loss/base_kto": 3.0482025146484375,
      "metrics/advantage_var": 400.92138671875,
      "regularization/mmd": 1.1668514013290405,
      "regularization/rkhs_norm": 224.8268585205078,
      "rewards/chosen": 0.7147384739521915,
      "rewards/margins": 0.9691219639045159,
      "rewards/rejected": -0.25438348995232435,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 11.56076717376709,
      "kl": 13.388104438781738,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34235619.381317,
      "logits/rejected": -36256815.362041466,
      "logps/chosen": -482.87964203675347,
      "logps/rejected": -386.31733452950556,
      "loss": 0.5271,
      "loss/base_kto": 3.081200361251831,
      "metrics/advantage_var": 383.1581115722656,
      "regularization/mmd": 1.135278344154358,
      "regularization/rkhs_norm": 218.743408203125,
      "rewards/chosen": 0.6020886361325134,
      "rewards/margins": 0.9377138785209329,
      "rewards/rejected": -0.33562524238841956,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 9.227808952331543,
      "kl": 13.091130256652832,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34775515.83045526,
      "logits/rejected": -34548301.23794712,
      "logps/chosen": -482.2095270800628,
      "logps/rejected": -374.9788102643857,
      "loss": 0.5275,
      "loss/base_kto": 3.0965659618377686,
      "metrics/advantage_var": 363.8895568847656,
      "regularization/mmd": 1.1230347156524658,
      "regularization/rkhs_norm": 216.3843231201172,
      "rewards/chosen": 0.5647210281348116,
      "rewards/margins": 0.929788576871811,
      "rewards/rejected": -0.3650675487369994,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 11.359593391418457,
      "kl": 12.2516508102417,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -34294916.98701299,
      "logits/rejected": -34889456.578313254,
      "logps/chosen": -472.3590198863636,
      "logps/rejected": -371.5574642319277,
      "loss": 0.5298,
      "loss/base_kto": 3.090224504470825,
      "metrics/advantage_var": 379.31390380859375,
      "regularization/mmd": 1.148317575454712,
      "regularization/rkhs_norm": 221.2557830810547,
      "rewards/chosen": 0.5197414051402699,
      "rewards/margins": 0.9446700932815066,
      "rewards/rejected": -0.4249286881412368,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 8.25497817993164,
      "kl": 9.440899848937988,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34211684.17391305,
      "logits/rejected": -36721248.60377359,
      "logps/chosen": -489.473262810559,
      "logps/rejected": -362.4254373034591,
      "loss": 0.5322,
      "loss/base_kto": 3.1154391765594482,
      "metrics/advantage_var": 390.202880859375,
      "regularization/mmd": 1.14195716381073,
      "regularization/rkhs_norm": 220.0302734375,
      "rewards/chosen": 0.4888434084305852,
      "rewards/margins": 0.9296738070861228,
      "rewards/rejected": -0.4408303986555375,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 10.409468650817871,
      "kl": 13.0885009765625,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34505938.58712716,
      "logits/rejected": -36608060.516329706,
      "logps/chosen": -523.9324960753532,
      "logps/rejected": -372.30574455676515,
      "loss": 0.5298,
      "loss/base_kto": 3.0309512615203857,
      "metrics/advantage_var": 430.7159729003906,
      "regularization/mmd": 1.2071322202682495,
      "regularization/rkhs_norm": 232.5880889892578,
      "rewards/chosen": 0.6278587832368524,
      "rewards/margins": 1.007938370043083,
      "rewards/rejected": -0.38007958680623055,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 9.557205200195312,
      "kl": 9.520075798034668,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -33973349.618320614,
      "logits/rejected": -35741296.2304,
      "logps/chosen": -473.8358778625954,
      "logps/rejected": -392.514525,
      "loss": 0.5327,
      "loss/base_kto": 3.0503900051116943,
      "metrics/advantage_var": 438.140625,
      "regularization/mmd": 1.211198091506958,
      "regularization/rkhs_norm": 233.37149047851562,
      "rewards/chosen": 0.5725689371123569,
      "rewards/margins": 1.0125099527373569,
      "rewards/rejected": -0.439941015625,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 12.640892028808594,
      "kl": 8.175743103027344,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -33956896.830534354,
      "logits/rejected": -35551094.3744,
      "logps/chosen": -492.9168893129771,
      "logps/rejected": -365.5969,
      "loss": 0.5307,
      "loss/base_kto": 3.129976987838745,
      "metrics/advantage_var": 352.7520446777344,
      "regularization/mmd": 1.1156553030014038,
      "regularization/rkhs_norm": 214.9624786376953,
      "rewards/chosen": 0.5004034843153626,
      "rewards/margins": 0.9101089530653627,
      "rewards/rejected": -0.40970546875,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 8.18465518951416,
      "kl": 9.268996238708496,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34128800.34303216,
      "logits/rejected": -34571063.119617224,
      "logps/chosen": -443.77038667687594,
      "logps/rejected": -373.73514752791067,
      "loss": 0.5254,
      "loss/base_kto": 3.1064136028289795,
      "metrics/advantage_var": 385.292236328125,
      "regularization/mmd": 1.0964959859848022,
      "regularization/rkhs_norm": 211.27090454101562,
      "rewards/chosen": 0.5316680861468702,
      "rewards/margins": 0.9256872054339017,
      "rewards/rejected": -0.3940191192870315,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 10.294926643371582,
      "kl": 12.416064262390137,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -35230866.741433024,
      "logits/rejected": -35737593.5799373,
      "logps/chosen": -487.93126947040497,
      "logps/rejected": -393.0354868730407,
      "loss": 0.5285,
      "loss/base_kto": 3.10258150100708,
      "metrics/advantage_var": 355.3718566894531,
      "regularization/mmd": 1.125684142112732,
      "regularization/rkhs_norm": 216.8948516845703,
      "rewards/chosen": 0.548384431737977,
      "rewards/margins": 0.9079973301310877,
      "rewards/rejected": -0.3596128983931108,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 11.296473503112793,
      "kl": 13.17083740234375,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -36390026.55054432,
      "logits/rejected": -34921644.00627943,
      "logps/chosen": -488.79840590979785,
      "logps/rejected": -368.5661057692308,
      "loss": 0.5294,
      "loss/base_kto": 3.0887959003448486,
      "metrics/advantage_var": 367.3639221191406,
      "regularization/mmd": 1.1464333534240723,
      "regularization/rkhs_norm": 220.8927459716797,
      "rewards/chosen": 0.5507928305428654,
      "rewards/margins": 0.9450203899184786,
      "rewards/rejected": -0.3942275593756132,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 12.347671508789062,
      "kl": 13.250162124633789,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -34845197.79397781,
      "logits/rejected": -36435696.61633282,
      "logps/chosen": -464.3315174326466,
      "logps/rejected": -356.19414483821265,
      "loss": 0.522,
      "loss/base_kto": 3.090343952178955,
      "metrics/advantage_var": 332.7373962402344,
      "regularization/mmd": 1.0859272480010986,
      "regularization/rkhs_norm": 209.23452758789062,
      "rewards/chosen": 0.5313877401941363,
      "rewards/margins": 0.9191312507181395,
      "rewards/rejected": -0.3877435105240033,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 12.698050498962402,
      "kl": 9.831679344177246,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -35244203.47551343,
      "logits/rejected": -36054669.65069552,
      "logps/chosen": -455.4088665086888,
      "logps/rejected": -397.23357805255023,
      "loss": 0.5281,
      "loss/base_kto": 3.0877957344055176,
      "metrics/advantage_var": 389.5352783203125,
      "regularization/mmd": 1.136988639831543,
      "regularization/rkhs_norm": 219.0729522705078,
      "rewards/chosen": 0.4912326324607524,
      "rewards/margins": 0.9503001171557388,
      "rewards/rejected": -0.4590674846949865,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 11.359013557434082,
      "kl": 8.866395950317383,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35714928.671850696,
      "logits/rejected": -35437547.10204082,
      "logps/chosen": -471.7073289269051,
      "logps/rejected": -394.2383241758242,
      "loss": 0.5306,
      "loss/base_kto": 3.0911548137664795,
      "metrics/advantage_var": 391.9338073730469,
      "regularization/mmd": 1.15353524684906,
      "regularization/rkhs_norm": 222.2611083984375,
      "rewards/chosen": 0.46505623397708495,
      "rewards/margins": 0.9473504333951335,
      "rewards/rejected": -0.48229419941804846,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 11.26817798614502,
      "kl": 9.509455680847168,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -32140284.809968848,
      "logits/rejected": -35454834.75862069,
      "logps/chosen": -467.07028816199374,
      "logps/rejected": -387.3153653996865,
      "loss": 0.5266,
      "loss/base_kto": 3.1238632202148438,
      "metrics/advantage_var": 345.85382080078125,
      "regularization/mmd": 1.08919358253479,
      "regularization/rkhs_norm": 209.86387634277344,
      "rewards/chosen": 0.48274901202905957,
      "rewards/margins": 0.9038101780569299,
      "rewards/rejected": -0.4210611660278703,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 11.294798851013184,
      "kl": 14.53909969329834,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -34184551.96147673,
      "logits/rejected": -35793104.852359205,
      "logps/chosen": -474.4423154093098,
      "logps/rejected": -384.1917332572298,
      "loss": 0.5304,
      "loss/base_kto": 3.1386189460754395,
      "metrics/advantage_var": 370.9787902832031,
      "regularization/mmd": 1.1045948266983032,
      "regularization/rkhs_norm": 212.8313446044922,
      "rewards/chosen": 0.5339062539187901,
      "rewards/margins": 0.857013690286156,
      "rewards/rejected": -0.3231074363673659,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.645028114318848,
      "kl": 11.492669105529785,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -35577545.430615164,
      "logits/rejected": -36086408.59208262,
      "logps/chosen": -481.560443490701,
      "logps/rejected": -389.5584391135973,
      "loss": 0.5324,
      "loss/base_kto": 3.1076765060424805,
      "metrics/advantage_var": 388.7484436035156,
      "regularization/mmd": 1.1512047052383423,
      "regularization/rkhs_norm": 221.8120880126953,
      "rewards/chosen": 0.5397454563299132,
      "rewards/margins": 0.9362181180426122,
      "rewards/rejected": -0.396472661712699,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 12.249076843261719,
      "kl": 15.316606521606445,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -35357905.45454545,
      "logits/rejected": -36790333.686746985,
      "logps/chosen": -529.6217532467532,
      "logps/rejected": -362.1625094126506,
      "loss": 0.5283,
      "loss/base_kto": 3.0658607482910156,
      "metrics/advantage_var": 402.5772399902344,
      "regularization/mmd": 1.1603775024414062,
      "regularization/rkhs_norm": 223.57949829101562,
      "rewards/chosen": 0.6278794523957488,
      "rewards/margins": 0.9634614272066133,
      "rewards/rejected": -0.33558197481086455,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 9.163156509399414,
      "kl": 10.477073669433594,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -33207286.309148263,
      "logits/rejected": -35143521.486068115,
      "logps/chosen": -472.6181979495268,
      "logps/rejected": -357.89870356037153,
      "loss": 0.5287,
      "loss/base_kto": 3.095052480697632,
      "metrics/advantage_var": 384.4367980957031,
      "regularization/mmd": 1.134222388267517,
      "regularization/rkhs_norm": 218.5399932861328,
      "rewards/chosen": 0.5564845713907236,
      "rewards/margins": 0.927981380177972,
      "rewards/rejected": -0.37149680878724844,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 10.682226181030273,
      "kl": 11.542448997497559,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35190588.49452269,
      "logits/rejected": -34718846.20280811,
      "logps/chosen": -460.3348493740219,
      "logps/rejected": -384.1492297191888,
      "loss": 0.5279,
      "loss/base_kto": 3.1000070571899414,
      "metrics/advantage_var": 357.6339416503906,
      "regularization/mmd": 1.1230379343032837,
      "regularization/rkhs_norm": 216.38499450683594,
      "rewards/chosen": 0.5276913351855927,
      "rewards/margins": 0.9278597198506668,
      "rewards/rejected": -0.4001683846650741,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 14.015803337097168,
      "kl": 12.815312385559082,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -35221306.30467571,
      "logits/rejected": -36127556.460291736,
      "logps/chosen": -473.0073058069382,
      "logps/rejected": -369.95715153970826,
      "loss": 0.5245,
      "loss/base_kto": 3.0746517181396484,
      "metrics/advantage_var": 359.9502868652344,
      "regularization/mmd": 1.121084213256836,
      "regularization/rkhs_norm": 216.00852966308594,
      "rewards/chosen": 0.562893506448553,
      "rewards/margins": 0.9334219640654239,
      "rewards/rejected": -0.3705284576168709,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 11.130681991577148,
      "kl": 12.841374397277832,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34324413.424148604,
      "logits/rejected": -35546018.32176656,
      "logps/chosen": -491.4020897832817,
      "logps/rejected": -352.90883773659306,
      "loss": 0.5267,
      "loss/base_kto": 3.0797278881073,
      "metrics/advantage_var": 376.87994384765625,
      "regularization/mmd": 1.1341991424560547,
      "regularization/rkhs_norm": 218.5354766845703,
      "rewards/chosen": 0.6077518049777477,
      "rewards/margins": 0.95141431964929,
      "rewards/rejected": -0.3436625146715423,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 11.328431129455566,
      "kl": 11.874162673950195,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35556658.89614243,
      "logits/rejected": -35123673.13531353,
      "logps/chosen": -468.75018545994067,
      "logps/rejected": -391.64253300330034,
      "loss": 0.5277,
      "loss/base_kto": 3.1278867721557617,
      "metrics/advantage_var": 333.9039001464844,
      "regularization/mmd": 1.0934902429580688,
      "regularization/rkhs_norm": 210.69175720214844,
      "rewards/chosen": 0.5320587610986183,
      "rewards/margins": 0.8782715699405377,
      "rewards/rejected": -0.34621280884191935,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 8.911983489990234,
      "kl": 15.25463581085205,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -35222911.174193546,
      "logits/rejected": -36076807.75757576,
      "logps/chosen": -480.13860887096774,
      "logps/rejected": -394.96796875,
      "loss": 0.5294,
      "loss/base_kto": 3.02984619140625,
      "metrics/advantage_var": 424.6706237792969,
      "regularization/mmd": 1.2054380178451538,
      "regularization/rkhs_norm": 232.2616729736328,
      "rewards/chosen": 0.6062917893932712,
      "rewards/margins": 1.0025537097442418,
      "rewards/rejected": -0.39626192035097063,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 11.520490646362305,
      "kl": 13.6519193649292,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -35128602.15479876,
      "logits/rejected": -36351912.78233439,
      "logps/chosen": -468.5253482972136,
      "logps/rejected": -383.8808162460568,
      "loss": 0.5307,
      "loss/base_kto": 3.1344125270843506,
      "metrics/advantage_var": 350.7089538574219,
      "regularization/mmd": 1.1107876300811768,
      "regularization/rkhs_norm": 214.02456665039062,
      "rewards/chosen": 0.527235096084075,
      "rewards/margins": 0.8736736754628951,
      "rewards/rejected": -0.34643857937882,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 10.270307540893555,
      "kl": 13.290621757507324,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -33216139.925465837,
      "logits/rejected": -35478682.56603774,
      "logps/chosen": -503.1929347826087,
      "logps/rejected": -354.3146373820755,
      "loss": 0.5281,
      "loss/base_kto": 3.0898449420928955,
      "metrics/advantage_var": 369.3459167480469,
      "regularization/mmd": 1.1351157426834106,
      "regularization/rkhs_norm": 218.7120819091797,
      "rewards/chosen": 0.5763542222680512,
      "rewards/margins": 0.9304242739291287,
      "rewards/rejected": -0.3540700516610775,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 7.135807037353516,
      "kl": 12.883994102478027,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34329645.801916935,
      "logits/rejected": -36721967.755351685,
      "logps/chosen": -490.8725039936102,
      "logps/rejected": -378.97720756880733,
      "loss": 0.5265,
      "loss/base_kto": 3.095879554748535,
      "metrics/advantage_var": 361.8744812011719,
      "regularization/mmd": 1.1162868738174438,
      "regularization/rkhs_norm": 215.0841827392578,
      "rewards/chosen": 0.5800625249600639,
      "rewards/margins": 0.9294950473284221,
      "rewards/rejected": -0.3494325223683582,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 10.335249900817871,
      "kl": 12.950479507446289,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -34357906.977917984,
      "logits/rejected": -35518349.86996904,
      "logps/chosen": -489.25621056782336,
      "logps/rejected": -372.02549342105266,
      "loss": 0.5207,
      "loss/base_kto": 3.0475614070892334,
      "metrics/advantage_var": 361.4256286621094,
      "regularization/mmd": 1.1180591583251953,
      "regularization/rkhs_norm": 215.42568969726562,
      "rewards/chosen": 0.5793135008225305,
      "rewards/margins": 0.9720364189757087,
      "rewards/rejected": -0.3927229181531782,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 12.101631164550781,
      "kl": 14.51342487335205,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34178817.240710825,
      "logits/rejected": -35369808.944024205,
      "logps/chosen": -498.0520496768982,
      "logps/rejected": -371.3050302571861,
      "loss": 0.5303,
      "loss/base_kto": 3.0979537963867188,
      "metrics/advantage_var": 576.8146362304688,
      "regularization/mmd": 1.1443445682525635,
      "regularization/rkhs_norm": 220.4902801513672,
      "rewards/chosen": 0.5742010014892972,
      "rewards/margins": 0.8867016940213273,
      "rewards/rejected": -0.31250069253203006,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 12.365646362304688,
      "kl": 13.501253128051758,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35068562.04590164,
      "logits/rejected": -37147907.82089552,
      "logps/chosen": -491.85056352459014,
      "logps/rejected": -379.8675839552239,
      "loss": 0.5213,
      "loss/base_kto": 3.0109212398529053,
      "metrics/advantage_var": 396.2356872558594,
      "regularization/mmd": 1.1596250534057617,
      "regularization/rkhs_norm": 223.43447875976562,
      "rewards/chosen": 0.5979486184042008,
      "rewards/margins": 0.9946309477243966,
      "rewards/rejected": -0.39668232932019587,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 7.931318759918213,
      "kl": 12.034379959106445,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -34815871.38164251,
      "logits/rejected": -36582507.21699545,
      "logps/chosen": -469.4171698872786,
      "logps/rejected": -372.40629742033383,
      "loss": 0.5308,
      "loss/base_kto": 3.097630023956299,
      "metrics/advantage_var": 385.7208557128906,
      "regularization/mmd": 1.1488761901855469,
      "regularization/rkhs_norm": 221.36343383789062,
      "rewards/chosen": 0.5448376445186695,
      "rewards/margins": 0.933819696663254,
      "rewards/rejected": -0.3889820521445846,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 13.937902450561523,
      "kl": 12.061327934265137,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35316257.267828844,
      "logits/rejected": -35816481.13405239,
      "logps/chosen": -467.92115689381933,
      "logps/rejected": -386.2687548151002,
      "loss": 0.5311,
      "loss/base_kto": 3.126477003097534,
      "metrics/advantage_var": 370.1043395996094,
      "regularization/mmd": 1.1222646236419678,
      "regularization/rkhs_norm": 216.23594665527344,
      "rewards/chosen": 0.518689138953546,
      "rewards/margins": 0.8781875258949943,
      "rewards/rejected": -0.35949838694144837,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 9.281513214111328,
      "kl": 12.302824974060059,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35342033.45454545,
      "logits/rejected": -36803324.915662654,
      "logps/chosen": -493.375101461039,
      "logps/rejected": -401.9391942771084,
      "loss": 0.5236,
      "loss/base_kto": 3.0720460414886475,
      "metrics/advantage_var": 353.0370178222656,
      "regularization/mmd": 1.1166185140609741,
      "regularization/rkhs_norm": 215.14805603027344,
      "rewards/chosen": 0.5612790987089082,
      "rewards/margins": 0.9515410130655535,
      "rewards/rejected": -0.3902619143566453,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 11.074307441711426,
      "kl": 10.97407054901123,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -34191000.47648903,
      "logits/rejected": -35948942.75389408,
      "logps/chosen": -464.1570336990596,
      "logps/rejected": -365.73203855140184,
      "loss": 0.5278,
      "loss/base_kto": 3.1273510456085205,
      "metrics/advantage_var": 343.7378845214844,
      "regularization/mmd": 1.0950945615768433,
      "regularization/rkhs_norm": 211.0008544921875,
      "rewards/chosen": 0.5045455406452047,
      "rewards/margins": 0.8785536242858054,
      "rewards/rejected": -0.3740080836406007,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 10.354769706726074,
      "kl": 11.708769798278809,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35112156.67596899,
      "logits/rejected": -36165696.503937006,
      "logps/chosen": -493.58856589147285,
      "logps/rejected": -394.29537401574805,
      "loss": 0.5327,
      "loss/base_kto": 3.140596389770508,
      "metrics/advantage_var": 366.3534240722656,
      "regularization/mmd": 1.1208542585372925,
      "regularization/rkhs_norm": 215.96421813964844,
      "rewards/chosen": 0.5278361948885659,
      "rewards/margins": 0.8934785923110561,
      "rewards/rejected": -0.36564239742249016,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 7.407812595367432,
      "kl": 9.797353744506836,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34317422.32473445,
      "logits/rejected": -33990174.50563607,
      "logps/chosen": -469.95428679817906,
      "logps/rejected": -343.463768115942,
      "loss": 0.5238,
      "loss/base_kto": 3.1010022163391113,
      "metrics/advantage_var": 348.421142578125,
      "regularization/mmd": 1.0893239974975586,
      "regularization/rkhs_norm": 209.88902282714844,
      "rewards/chosen": 0.5466918019134105,
      "rewards/margins": 0.9251473647813906,
      "rewards/rejected": -0.3784555628679801,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.975174191323546e+17,
      "train_loss": 0.5582944930938035,
      "train_runtime": 11062.9397,
      "train_samples_per_second": 13.398,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.975174191323546e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}