{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 17.28666877746582,
      "kl": 13.869549751281738,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36216319.19370079,
      "logits/rejected": -37507910.25116279,
      "logps/chosen": -481.15551181102364,
      "logps/rejected": -354.4021802325581,
      "loss": 0.6018,
      "loss/base_kto": 3.947984457015991,
      "metrics/advantage_var": 226.25376892089844,
      "regularization/mmd": 0.8666197061538696,
      "regularization/rkhs_norm": 166.97874450683594,
      "rewards/chosen": 0.1351126843550074,
      "rewards/margins": 0.039705591691338554,
      "rewards/rejected": 0.09540709266366884,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 20.494544982910156,
      "kl": 8.738840103149414,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36129686.4,
      "logits/rejected": -36837670.4,
      "logps/chosen": -475.59697265625,
      "logps/rejected": -411.471484375,
      "loss": 0.5951,
      "loss/base_kto": 3.9235122203826904,
      "metrics/advantage_var": 196.90664672851562,
      "regularization/mmd": 0.8370766043663025,
      "regularization/rkhs_norm": 161.28643798828125,
      "rewards/chosen": 0.08794127702713013,
      "rewards/margins": 0.07156163454055786,
      "rewards/rejected": 0.016379642486572265,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 17.606281280517578,
      "kl": 4.159315586090088,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35912865.00628931,
      "logits/rejected": -38222110.21118012,
      "logps/chosen": -488.298054245283,
      "logps/rejected": -373.7237480590062,
      "loss": 0.5808,
      "loss/base_kto": 3.898115634918213,
      "metrics/advantage_var": 159.5330047607422,
      "regularization/mmd": 0.7481209635734558,
      "regularization/rkhs_norm": 144.14663696289062,
      "rewards/chosen": -0.0006746038700799522,
      "rewards/margins": 0.08927492697428267,
      "rewards/rejected": -0.08994953084436262,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 18.236949920654297,
      "kl": 6.960988521575928,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37659216.50314465,
      "logits/rejected": -37593059.37888199,
      "logps/chosen": -504.82360455974845,
      "logps/rejected": -383.9125339673913,
      "loss": 0.5807,
      "loss/base_kto": 3.8643014430999756,
      "metrics/advantage_var": 173.45523071289062,
      "regularization/mmd": 0.7811560034751892,
      "regularization/rkhs_norm": 150.51174926757812,
      "rewards/chosen": 0.08751846409443789,
      "rewards/margins": 0.11913866150904934,
      "rewards/rejected": -0.03162019741461144,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 32.01507568359375,
      "kl": 13.415864944458008,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36480816.64797508,
      "logits/rejected": -37147622.319749214,
      "logps/chosen": -507.1674941588785,
      "logps/rejected": -378.25230211598745,
      "loss": 0.5863,
      "loss/base_kto": 3.8215203285217285,
      "metrics/advantage_var": 216.98648071289062,
      "regularization/mmd": 0.8688768744468689,
      "regularization/rkhs_norm": 167.4136505126953,
      "rewards/chosen": 0.22570339690116337,
      "rewards/margins": 0.1469141373284018,
      "rewards/rejected": 0.07878925957276156,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 18.021196365356445,
      "kl": 9.468263626098633,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35707987.88253478,
      "logits/rejected": -37240802.88151659,
      "logps/chosen": -469.1993817619784,
      "logps/rejected": -370.0141686413902,
      "loss": 0.5801,
      "loss/base_kto": 3.831249237060547,
      "metrics/advantage_var": 192.60733032226562,
      "regularization/mmd": 0.8097471594810486,
      "regularization/rkhs_norm": 156.02061462402344,
      "rewards/chosen": 0.1122589111328125,
      "rewards/margins": 0.1417243198195905,
      "rewards/rejected": -0.02946540868677799,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 18.51321792602539,
      "kl": 7.499949932098389,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36242857.88379205,
      "logits/rejected": -37610872.23003195,
      "logps/chosen": -479.3987958715596,
      "logps/rejected": -384.4207767571885,
      "loss": 0.5826,
      "loss/base_kto": 3.8097259998321533,
      "metrics/advantage_var": 211.20962524414062,
      "regularization/mmd": 0.8511638045310974,
      "regularization/rkhs_norm": 164.00074768066406,
      "rewards/chosen": 0.16908476476640147,
      "rewards/margins": 0.1780336560740007,
      "rewards/rejected": -0.008948891307599247,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 14.697526931762695,
      "kl": 8.385546684265137,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -36088274.313846156,
      "logits/rejected": -37605951.39047619,
      "logps/chosen": -493.81947115384617,
      "logps/rejected": -360.7446428571429,
      "loss": 0.5799,
      "loss/base_kto": 3.7885797023773193,
      "metrics/advantage_var": 223.79861450195312,
      "regularization/mmd": 0.8502904176712036,
      "regularization/rkhs_norm": 163.8324432373047,
      "rewards/chosen": 0.15470606877253607,
      "rewards/margins": 0.19735971872332042,
      "rewards/rejected": -0.04265364995078435,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 15.981952667236328,
      "kl": 7.477453708648682,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35705480.586583465,
      "logits/rejected": -36716213.88419405,
      "logps/chosen": -526.872513650546,
      "logps/rejected": -367.46322378716746,
      "loss": 0.5841,
      "loss/base_kto": 3.7968509197235107,
      "metrics/advantage_var": 212.843994140625,
      "regularization/mmd": 0.8761966824531555,
      "regularization/rkhs_norm": 168.8240203857422,
      "rewards/chosen": 0.1464927423390881,
      "rewards/margins": 0.1853673282026582,
      "rewards/rejected": -0.038874585863570094,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 14.56324577331543,
      "kl": 9.05606746673584,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -35997238.022047244,
      "logits/rejected": -37437624.16124031,
      "logps/chosen": -461.77288385826773,
      "logps/rejected": -374.12756782945735,
      "loss": 0.5846,
      "loss/base_kto": 3.8013081550598145,
      "metrics/advantage_var": 252.80130004882812,
      "regularization/mmd": 0.875170886516571,
      "regularization/rkhs_norm": 168.6263885498047,
      "rewards/chosen": 0.1368635282741757,
      "rewards/margins": 0.18972407808264832,
      "rewards/rejected": -0.05286054980847263,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 17.85694694519043,
      "kl": 5.026250839233398,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -36942050.119873814,
      "logits/rejected": -38242938.05572756,
      "logps/chosen": -477.1010449526814,
      "logps/rejected": -379.43817724458205,
      "loss": 0.5799,
      "loss/base_kto": 3.8143177032470703,
      "metrics/advantage_var": 197.2425994873047,
      "regularization/mmd": 0.8247326016426086,
      "regularization/rkhs_norm": 158.9080352783203,
      "rewards/chosen": 0.08102270782181893,
      "rewards/margins": 0.17888955745538124,
      "rewards/rejected": -0.0978668496335623,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 16.530935287475586,
      "kl": 2.5676889419555664,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -34537841.365539454,
      "logits/rejected": -37187565.353566006,
      "logps/chosen": -483.52556360708536,
      "logps/rejected": -351.68771339150226,
      "loss": 0.5756,
      "loss/base_kto": 3.769634246826172,
      "metrics/advantage_var": 197.66934204101562,
      "regularization/mmd": 0.8351051211357117,
      "regularization/rkhs_norm": 160.90658569335938,
      "rewards/chosen": -0.024460631292223353,
      "rewards/margins": 0.21369419535681877,
      "rewards/rejected": -0.2381548266490421,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 17.145092010498047,
      "kl": 5.858470439910889,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -37138496.60567823,
      "logits/rejected": -37570468.0619195,
      "logps/chosen": -485.2397969242902,
      "logps/rejected": -390.5070626934984,
      "loss": 0.5842,
      "loss/base_kto": 3.8405380249023438,
      "metrics/advantage_var": 198.77114868164062,
      "regularization/mmd": 0.8329442143440247,
      "regularization/rkhs_norm": 160.49021911621094,
      "rewards/chosen": -0.03661429769233198,
      "rewards/margins": 0.14798289484165117,
      "rewards/rejected": -0.18459719253398316,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 13.552000999450684,
      "kl": 6.439029693603516,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -37463869.847094804,
      "logits/rejected": -38733771.65495208,
      "logps/chosen": -493.7569285168196,
      "logps/rejected": -376.71270966453676,
      "loss": 0.5815,
      "loss/base_kto": 3.77490234375,
      "metrics/advantage_var": 218.3898468017578,
      "regularization/mmd": 0.8774310946464539,
      "regularization/rkhs_norm": 169.06187438964844,
      "rewards/chosen": 0.14522417167640242,
      "rewards/margins": 0.22872841888477527,
      "rewards/rejected": -0.08350424720837285,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 16.38521957397461,
      "kl": 9.403640747070312,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -34959307.5712,
      "logits/rejected": -36243984.41526718,
      "logps/chosen": -496.1666,
      "logps/rejected": -396.7485448473282,
      "loss": 0.5902,
      "loss/base_kto": 3.8247368335723877,
      "metrics/advantage_var": 240.5669403076172,
      "regularization/mmd": 0.8972446322441101,
      "regularization/rkhs_norm": 172.8795166015625,
      "rewards/chosen": 0.12554970703125,
      "rewards/margins": 0.13659369605260951,
      "rewards/rejected": -0.011043989021359508,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 15.341755867004395,
      "kl": 8.521685600280762,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -35781301.36677116,
      "logits/rejected": -36972237.75700934,
      "logps/chosen": -471.6827978056426,
      "logps/rejected": -365.81563960280374,
      "loss": 0.5803,
      "loss/base_kto": 3.7325761318206787,
      "metrics/advantage_var": 240.1177520751953,
      "regularization/mmd": 0.9098437428474426,
      "regularization/rkhs_norm": 175.3070831298828,
      "rewards/chosen": 0.09740401958597117,
      "rewards/margins": 0.2508578108388917,
      "rewards/rejected": -0.15345379125292055,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.72157096862793,
      "kl": 10.334311485290527,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36954460.98654708,
      "logits/rejected": -38009522.48772504,
      "logps/chosen": -456.5036434977578,
      "logps/rejected": -359.93443126022913,
      "loss": 0.5742,
      "loss/base_kto": 3.677534580230713,
      "metrics/advantage_var": 242.1745147705078,
      "regularization/mmd": 0.9159858822822571,
      "regularization/rkhs_norm": 176.49053955078125,
      "rewards/chosen": 0.1850318965712292,
      "rewards/margins": 0.3090293366908461,
      "rewards/rejected": -0.12399744011961691,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 21.72677993774414,
      "kl": 10.107943534851074,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -36391666.87179487,
      "logits/rejected": -36863719.02439024,
      "logps/chosen": -485.4851262019231,
      "logps/rejected": -374.4798971036585,
      "loss": 0.5877,
      "loss/base_kto": 3.8004074096679688,
      "metrics/advantage_var": 234.7443389892578,
      "regularization/mmd": 0.9010725021362305,
      "regularization/rkhs_norm": 173.6170654296875,
      "rewards/chosen": 0.08377578319647373,
      "rewards/margins": 0.15731556360389085,
      "rewards/rejected": -0.07353978040741711,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 13.693880081176758,
      "kl": 9.090913772583008,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -38133170.37672282,
      "logits/rejected": -38013989.56299841,
      "logps/chosen": -482.5569008422665,
      "logps/rejected": -372.1973185805423,
      "loss": 0.5878,
      "loss/base_kto": 3.7692184448242188,
      "metrics/advantage_var": 261.0716247558594,
      "regularization/mmd": 0.9331462979316711,
      "regularization/rkhs_norm": 179.79698181152344,
      "rewards/chosen": 0.18396464606339133,
      "rewards/margins": 0.20749051227831997,
      "rewards/rejected": -0.023525866214928633,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 30.564823150634766,
      "kl": 6.068899154663086,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36919501.136289,
      "logits/rejected": -38799903.284649774,
      "logps/chosen": -492.0698891625616,
      "logps/rejected": -375.616663561848,
      "loss": 0.5856,
      "loss/base_kto": 3.774129629135132,
      "metrics/advantage_var": 229.98155212402344,
      "regularization/mmd": 0.9104791879653931,
      "regularization/rkhs_norm": 175.4295196533203,
      "rewards/chosen": 0.07467767875182804,
      "rewards/margins": 0.20593059861548768,
      "rewards/rejected": -0.13125291986365964,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 15.081184387207031,
      "kl": 7.291744232177734,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -36202234.88786482,
      "logits/rejected": -36638667.90461049,
      "logps/chosen": -490.5248655913978,
      "logps/rejected": -367.88473767885534,
      "loss": 0.5782,
      "loss/base_kto": 3.7068536281585693,
      "metrics/advantage_var": 250.29248046875,
      "regularization/mmd": 0.9191361665725708,
      "regularization/rkhs_norm": 177.09751892089844,
      "rewards/chosen": 0.11651709771925403,
      "rewards/margins": 0.2766628800444043,
      "rewards/rejected": -0.16014578232515028,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 16.892141342163086,
      "kl": 8.64151668548584,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -34951623.11111111,
      "logits/rejected": -38028521.48534202,
      "logps/chosen": -507.9345908408408,
      "logps/rejected": -362.2352911237785,
      "loss": 0.5804,
      "loss/base_kto": 3.7336881160736084,
      "metrics/advantage_var": 251.204833984375,
      "regularization/mmd": 0.9091903567314148,
      "regularization/rkhs_norm": 175.18118286132812,
      "rewards/chosen": 0.13266144715271913,
      "rewards/margins": 0.26913717484287847,
      "rewards/rejected": -0.1364757276901593,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 18.60469627380371,
      "kl": 13.147854804992676,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35052245.53945249,
      "logits/rejected": -36744482.57359636,
      "logps/chosen": -472.6548913043478,
      "logps/rejected": -353.2754172989378,
      "loss": 0.581,
      "loss/base_kto": 3.7372944355010986,
      "metrics/advantage_var": 239.1718292236328,
      "regularization/mmd": 0.9105936288833618,
      "regularization/rkhs_norm": 175.4515380859375,
      "rewards/chosen": 0.21827007299844,
      "rewards/margins": 0.23776968821762567,
      "rewards/rejected": -0.019499615219185674,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 14.794768333435059,
      "kl": 6.379868030548096,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36755732.57877813,
      "logits/rejected": -37569875.258358665,
      "logps/chosen": -484.35289389067526,
      "logps/rejected": -388.01676481762917,
      "loss": 0.5774,
      "loss/base_kto": 3.7156808376312256,
      "metrics/advantage_var": 239.91152954101562,
      "regularization/mmd": 0.9038274884223938,
      "regularization/rkhs_norm": 174.1478729248047,
      "rewards/chosen": 0.07609347745153298,
      "rewards/margins": 0.27110358442816673,
      "rewards/rejected": -0.19501010697663373,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 16.114152908325195,
      "kl": 14.015588760375977,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -36049948.53250774,
      "logits/rejected": -37586472.3785489,
      "logps/chosen": -497.7021575077399,
      "logps/rejected": -355.047417192429,
      "loss": 0.5867,
      "loss/base_kto": 3.7359397411346436,
      "metrics/advantage_var": 264.5102844238281,
      "regularization/mmd": 0.9575252532958984,
      "regularization/rkhs_norm": 184.49424743652344,
      "rewards/chosen": 0.16994176339069755,
      "rewards/margins": 0.2273095274515032,
      "rewards/rejected": -0.05736776406080565,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 12.509623527526855,
      "kl": 8.933459281921387,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -35814398.422187984,
      "logits/rejected": -36157615.26465927,
      "logps/chosen": -486.26926040061636,
      "logps/rejected": -367.7392531695721,
      "loss": 0.5831,
      "loss/base_kto": 3.7047085762023926,
      "metrics/advantage_var": 290.3509216308594,
      "regularization/mmd": 0.9603023529052734,
      "regularization/rkhs_norm": 185.0293426513672,
      "rewards/chosen": 0.12054084813098878,
      "rewards/margins": 0.2591782034457484,
      "rewards/rejected": -0.13863735531475957,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 18.955705642700195,
      "kl": 16.17554473876953,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37009599.22891566,
      "logits/rejected": -36987328.83116883,
      "logps/chosen": -480.6894766566265,
      "logps/rejected": -357.87243810876623,
      "loss": 0.5762,
      "loss/base_kto": 3.6945130825042725,
      "metrics/advantage_var": 248.5550537109375,
      "regularization/mmd": 0.9150959849357605,
      "regularization/rkhs_norm": 176.31907653808594,
      "rewards/chosen": 0.2735217450612999,
      "rewards/margins": 0.28345770929885983,
      "rewards/rejected": -0.009935964237559925,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 14.894756317138672,
      "kl": 13.262564659118652,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36067758.82926829,
      "logits/rejected": -37874825.84615385,
      "logps/chosen": -467.3940548780488,
      "logps/rejected": -348.3900741185897,
      "loss": 0.5851,
      "loss/base_kto": 3.7631022930145264,
      "metrics/advantage_var": 239.29248046875,
      "regularization/mmd": 0.9173154830932617,
      "regularization/rkhs_norm": 176.74673461914062,
      "rewards/chosen": 0.17195061939518627,
      "rewards/margins": 0.2147787238300555,
      "rewards/rejected": -0.04282810443486923,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 16.39813232421875,
      "kl": 18.698083877563477,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -37496603.105882354,
      "logits/rejected": -39177465.17333333,
      "logps/chosen": -473.9681985294118,
      "logps/rejected": -385.46098958333334,
      "loss": 0.5892,
      "loss/base_kto": 3.7126572132110596,
      "metrics/advantage_var": 292.2877502441406,
      "regularization/mmd": 1.0008432865142822,
      "regularization/rkhs_norm": 192.84071350097656,
      "rewards/chosen": 0.31195447585161995,
      "rewards/margins": 0.23376629174924365,
      "rewards/rejected": 0.0781881841023763,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 15.629815101623535,
      "kl": 17.760190963745117,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37011374.08,
      "logits/rejected": -36616374.04444444,
      "logps/chosen": -449.8775961538461,
      "logps/rejected": -393.846378968254,
      "loss": 0.5801,
      "loss/base_kto": 3.748232364654541,
      "metrics/advantage_var": 219.3304901123047,
      "regularization/mmd": 0.8923266530036926,
      "regularization/rkhs_norm": 171.93191528320312,
      "rewards/chosen": 0.2888657789963942,
      "rewards/margins": 0.2183362590524303,
      "rewards/rejected": 0.07052951994396392,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 17.80891227722168,
      "kl": 19.24934196472168,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -35685240.75471698,
      "logits/rejected": -37087193.838509314,
      "logps/chosen": -485.37578616352204,
      "logps/rejected": -387.72481560559004,
      "loss": 0.5818,
      "loss/base_kto": 3.7200429439544678,
      "metrics/advantage_var": 246.68115234375,
      "regularization/mmd": 0.9344601631164551,
      "regularization/rkhs_norm": 180.0501251220703,
      "rewards/chosen": 0.3106190423545597,
      "rewards/margins": 0.23427756733575633,
      "rewards/rejected": 0.07634147501880338,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 13.532238960266113,
      "kl": 17.658248901367188,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35347474.197092086,
      "logits/rejected": -36609608.036308624,
      "logps/chosen": -492.469810177706,
      "logps/rejected": -366.7897834720121,
      "loss": 0.5756,
      "loss/base_kto": 3.7010133266448975,
      "metrics/advantage_var": 232.38818359375,
      "regularization/mmd": 0.9040445685386658,
      "regularization/rkhs_norm": 174.18968200683594,
      "rewards/chosen": 0.21897417355045942,
      "rewards/margins": 0.253385691918625,
      "rewards/rejected": -0.03441151836816554,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 15.496357917785645,
      "kl": 11.838883399963379,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36601564.7706422,
      "logits/rejected": -37832164.19169329,
      "logps/chosen": -495.41910359327215,
      "logps/rejected": -371.149660543131,
      "loss": 0.5739,
      "loss/base_kto": 3.676666021347046,
      "metrics/advantage_var": 237.0503387451172,
      "regularization/mmd": 0.91443932056427,
      "regularization/rkhs_norm": 176.19253540039062,
      "rewards/chosen": 0.2042687558981986,
      "rewards/margins": 0.3039905486859881,
      "rewards/rejected": -0.09972179278778953,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 20.567485809326172,
      "kl": 16.626239776611328,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -34921041.1008,
      "logits/rejected": -37017825.123664126,
      "logps/chosen": -473.7086,
      "logps/rejected": -375.5192032442748,
      "loss": 0.5753,
      "loss/base_kto": 3.697774887084961,
      "metrics/advantage_var": 232.10256958007812,
      "regularization/mmd": 0.9045900702476501,
      "regularization/rkhs_norm": 174.2947998046875,
      "rewards/chosen": 0.1565810546875,
      "rewards/margins": 0.2124826243160335,
      "rewards/rejected": -0.055901569628533515,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 12.992965698242188,
      "kl": 9.039730072021484,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -34760422.645367414,
      "logits/rejected": -37357768.41590214,
      "logps/chosen": -484.9376497603834,
      "logps/rejected": -363.50365538990826,
      "loss": 0.5773,
      "loss/base_kto": 3.666606903076172,
      "metrics/advantage_var": 261.6567077636719,
      "regularization/mmd": 0.9521806836128235,
      "regularization/rkhs_norm": 183.46450805664062,
      "rewards/chosen": 0.11359065485457642,
      "rewards/margins": 0.3118521608214629,
      "rewards/rejected": -0.19826150596688646,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 17.397127151489258,
      "kl": 4.5755157470703125,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36470936.5451664,
      "logits/rejected": -37373705.861325115,
      "logps/chosen": -484.4300713153724,
      "logps/rejected": -371.74352369029276,
      "loss": 0.5889,
      "loss/base_kto": 3.7251832485198975,
      "metrics/advantage_var": 285.101806640625,
      "regularization/mmd": 0.9862076640129089,
      "regularization/rkhs_norm": 190.020751953125,
      "rewards/chosen": 0.010507079576350241,
      "rewards/margins": 0.2533064828801108,
      "rewards/rejected": -0.2427994033037606,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 15.375116348266602,
      "kl": 14.448086738586426,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -37374692.628930815,
      "logits/rejected": -38540908.124223605,
      "logps/chosen": -514.5865762578617,
      "logps/rejected": -353.1389266304348,
      "loss": 0.5739,
      "loss/base_kto": 3.628591299057007,
      "metrics/advantage_var": 268.5576477050781,
      "regularization/mmd": 0.9626501202583313,
      "regularization/rkhs_norm": 185.48171997070312,
      "rewards/chosen": 0.27646538596483144,
      "rewards/margins": 0.3209132078248191,
      "rewards/rejected": -0.04444782185998763,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 16.5850772857666,
      "kl": 8.80722713470459,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -37816466.985645935,
      "logits/rejected": -37114213.53751914,
      "logps/chosen": -489.84370015948963,
      "logps/rejected": -397.30455589586524,
      "loss": 0.5806,
      "loss/base_kto": 3.75372314453125,
      "metrics/advantage_var": 228.4040985107422,
      "regularization/mmd": 0.8908863067626953,
      "regularization/rkhs_norm": 171.6544189453125,
      "rewards/chosen": 0.06759945323402612,
      "rewards/margins": 0.19316826957821281,
      "rewards/rejected": -0.12556881634418668,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 13.62050724029541,
      "kl": 11.619669914245605,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -37364197.218461536,
      "logits/rejected": -37631634.75159235,
      "logps/chosen": -479.99701923076924,
      "logps/rejected": -398.9910429936306,
      "loss": 0.5686,
      "loss/base_kto": 3.5261247158050537,
      "metrics/advantage_var": 315.3587951660156,
      "regularization/mmd": 1.022654414176941,
      "regularization/rkhs_norm": 197.04324340820312,
      "rewards/chosen": 0.25338146503155046,
      "rewards/margins": 0.45228655944555074,
      "rewards/rejected": -0.1989050944140003,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 12.98912239074707,
      "kl": 14.082290649414062,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -37224238.761609904,
      "logits/rejected": -36818217.18611988,
      "logps/chosen": -465.79005417956654,
      "logps/rejected": -384.6505323343849,
      "loss": 0.5685,
      "loss/base_kto": 3.2370293140411377,
      "metrics/advantage_var": 594.7805786132812,
      "regularization/mmd": 1.3111907243728638,
      "regularization/rkhs_norm": 252.6378936767578,
      "rewards/chosen": 0.5340818044940016,
      "rewards/margins": 0.8299278470668895,
      "rewards/rejected": -0.29584604257288794,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 14.384129524230957,
      "kl": 10.371628761291504,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -35496298.596721314,
      "logits/rejected": -37267306.22089552,
      "logps/chosen": -490.1142418032787,
      "logps/rejected": -370.1044776119403,
      "loss": 0.5711,
      "loss/base_kto": 3.2188737392425537,
      "metrics/advantage_var": 599.1707763671875,
      "regularization/mmd": 1.350122332572937,
      "regularization/rkhs_norm": 260.13916015625,
      "rewards/chosen": 0.4529114770107582,
      "rewards/margins": 0.8901385952039718,
      "rewards/rejected": -0.43722711819321364,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 13.116654396057129,
      "kl": 7.182905673980713,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36326894.85167464,
      "logits/rejected": -37312913.44563553,
      "logps/chosen": -470.3089114832536,
      "logps/rejected": -356.1921659647779,
      "loss": 0.5686,
      "loss/base_kto": 3.2808244228363037,
      "metrics/advantage_var": 541.5362548828125,
      "regularization/mmd": 1.268153429031372,
      "regularization/rkhs_norm": 244.34556579589844,
      "rewards/chosen": 0.31684688099643643,
      "rewards/margins": 0.7840647396775238,
      "rewards/rejected": -0.4672178586810873,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 11.740735054016113,
      "kl": 8.199923515319824,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -34742351.27741936,
      "logits/rejected": -35920880.484848484,
      "logps/chosen": -494.820564516129,
      "logps/rejected": -370.13219696969696,
      "loss": 0.5634,
      "loss/base_kto": 3.1991069316864014,
      "metrics/advantage_var": 542.0139770507812,
      "regularization/mmd": 1.307726263999939,
      "regularization/rkhs_norm": 251.9704132080078,
      "rewards/chosen": 0.3672663042622228,
      "rewards/margins": 0.8765179855033449,
      "rewards/rejected": -0.5092516812411222,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 13.875117301940918,
      "kl": 11.704268455505371,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35606088.700906344,
      "logits/rejected": -36732961.13915858,
      "logps/chosen": -491.577416918429,
      "logps/rejected": -385.93433960355986,
      "loss": 0.5744,
      "loss/base_kto": 3.2271511554718018,
      "metrics/advantage_var": 604.9378051757812,
      "regularization/mmd": 1.368111252784729,
      "regularization/rkhs_norm": 263.605224609375,
      "rewards/chosen": 0.5205691703496743,
      "rewards/margins": 0.858607642645639,
      "rewards/rejected": -0.3380384722959648,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 13.730972290039062,
      "kl": 15.659111022949219,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -35645711.802469134,
      "logits/rejected": -36658872.70886076,
      "logps/chosen": -471.9669174382716,
      "logps/rejected": -395.3960393591772,
      "loss": 0.5674,
      "loss/base_kto": 3.1399471759796143,
      "metrics/advantage_var": 651.7294921875,
      "regularization/mmd": 1.3993967771530151,
      "regularization/rkhs_norm": 269.63330078125,
      "rewards/chosen": 0.5837679262514468,
      "rewards/margins": 0.9759252032258955,
      "rewards/rejected": -0.3921572769744487,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 13.653825759887695,
      "kl": 16.984235763549805,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -35353943.473354235,
      "logits/rejected": -36772886.33021807,
      "logps/chosen": -472.08801920062695,
      "logps/rejected": -407.0686818535826,
      "loss": 0.5731,
      "loss/base_kto": 3.203996419906616,
      "metrics/advantage_var": 627.2606201171875,
      "regularization/mmd": 1.380798578262329,
      "regularization/rkhs_norm": 266.0498352050781,
      "rewards/chosen": 0.5989840038144103,
      "rewards/margins": 0.8975086880668076,
      "rewards/rejected": -0.2985246842523973,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 14.916421890258789,
      "kl": 18.980070114135742,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35682870.4,
      "logits/rejected": -37265344.0,
      "logps/chosen": -469.505517578125,
      "logps/rejected": -385.406591796875,
      "loss": 0.5838,
      "loss/base_kto": 3.1923668384552,
      "metrics/advantage_var": 793.8643188476562,
      "regularization/mmd": 1.4783775806427002,
      "regularization/rkhs_norm": 284.8511657714844,
      "rewards/chosen": 0.6252220153808594,
      "rewards/margins": 0.9321086883544922,
      "rewards/rejected": -0.3068866729736328,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 13.798221588134766,
      "kl": 8.175427436828613,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -35857702.835913315,
      "logits/rejected": -36561348.239747636,
      "logps/chosen": -480.4967105263158,
      "logps/rejected": -369.85996648264984,
      "loss": 0.5657,
      "loss/base_kto": 3.243157148361206,
      "metrics/advantage_var": 499.720703125,
      "regularization/mmd": 1.2824822664260864,
      "regularization/rkhs_norm": 247.10643005371094,
      "rewards/chosen": 0.3872545555280089,
      "rewards/margins": 0.8420906578421847,
      "rewards/rejected": -0.45483610231417587,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 9.352314949035645,
      "kl": 23.32759666442871,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -37269526.47021943,
      "logits/rejected": -36725230.45482866,
      "logps/chosen": -470.67584247648904,
      "logps/rejected": -372.3608352803738,
      "loss": 0.5672,
      "loss/base_kto": 3.1387908458709717,
      "metrics/advantage_var": 631.537841796875,
      "regularization/mmd": 1.399050235748291,
      "regularization/rkhs_norm": 269.5664978027344,
      "rewards/chosen": 0.7032140175750636,
      "rewards/margins": 0.9272310330738103,
      "rewards/rejected": -0.22401701549874659,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 12.985386848449707,
      "kl": 12.752604484558105,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -35332991.60856269,
      "logits/rejected": -35242435.47603834,
      "logps/chosen": -481.59121750764524,
      "logps/rejected": -364.62460063897765,
      "loss": 0.56,
      "loss/base_kto": 3.198040008544922,
      "metrics/advantage_var": 520.0338134765625,
      "regularization/mmd": 1.2815788984298706,
      "regularization/rkhs_norm": 246.9323272705078,
      "rewards/chosen": 0.5117893510637662,
      "rewards/margins": 0.8754246244700661,
      "rewards/rejected": -0.3636352734062999,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 14.315081596374512,
      "kl": 15.352499961853027,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -36456540.78964401,
      "logits/rejected": -38448242.465256795,
      "logps/chosen": -486.46677791262135,
      "logps/rejected": -364.0645770392749,
      "loss": 0.5649,
      "loss/base_kto": 3.1950738430023193,
      "metrics/advantage_var": 526.9627075195312,
      "regularization/mmd": 1.3240159749984741,
      "regularization/rkhs_norm": 255.1090850830078,
      "rewards/chosen": 0.5178439933505259,
      "rewards/margins": 0.8641094807909291,
      "rewards/rejected": -0.34626548744040314,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 11.546377182006836,
      "kl": 13.232007026672363,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -36010176.60188088,
      "logits/rejected": -37478313.86915888,
      "logps/chosen": -479.94494514106583,
      "logps/rejected": -396.6027550623053,
      "loss": 0.5692,
      "loss/base_kto": 3.1710774898529053,
      "metrics/advantage_var": 709.99072265625,
      "regularization/mmd": 1.382637858390808,
      "regularization/rkhs_norm": 266.4042053222656,
      "rewards/chosen": 0.5016709091521355,
      "rewards/margins": 0.8928089562248818,
      "rewards/rejected": -0.3911380470727463,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 10.159113883972168,
      "kl": 12.148736953735352,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -36512554.73449921,
      "logits/rejected": -38063398.14439324,
      "logps/chosen": -474.3939785373609,
      "logps/rejected": -374.65101766513055,
      "loss": 0.5712,
      "loss/base_kto": 3.201780080795288,
      "metrics/advantage_var": 608.3012084960938,
      "regularization/mmd": 1.3678390979766846,
      "regularization/rkhs_norm": 263.5528259277344,
      "rewards/chosen": 0.5240032153592011,
      "rewards/margins": 0.8929486403664015,
      "rewards/rejected": -0.36894542500720046,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 15.198099136352539,
      "kl": 12.118966102600098,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35891424.39506173,
      "logits/rejected": -36746252.962025315,
      "logps/chosen": -492.47202932098764,
      "logps/rejected": -367.29756724683546,
      "loss": 0.5676,
      "loss/base_kto": 3.21653413772583,
      "metrics/advantage_var": 601.02294921875,
      "regularization/mmd": 1.324424147605896,
      "regularization/rkhs_norm": 255.1876983642578,
      "rewards/chosen": 0.48597100340289834,
      "rewards/margins": 0.839893086810916,
      "rewards/rejected": -0.35392208340801773,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 23.502729415893555,
      "kl": 12.213374137878418,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -35316529.34939759,
      "logits/rejected": -36814023.48051948,
      "logps/chosen": -476.14664909638554,
      "logps/rejected": -362.5178064123377,
      "loss": 0.566,
      "loss/base_kto": 3.2537143230438232,
      "metrics/advantage_var": 508.3929748535156,
      "regularization/mmd": 1.2744741439819336,
      "regularization/rkhs_norm": 245.5634307861328,
      "rewards/chosen": 0.5508395275437689,
      "rewards/margins": 0.8143075887184341,
      "rewards/rejected": -0.2634680611746652,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 13.538008689880371,
      "kl": 15.613471031188965,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34546725.236363634,
      "logits/rejected": -36143731.61290322,
      "logps/chosen": -473.5277935606061,
      "logps/rejected": -386.2426411290323,
      "loss": 0.5634,
      "loss/base_kto": 3.24967885017395,
      "metrics/advantage_var": 471.4722595214844,
      "regularization/mmd": 1.257878303527832,
      "regularization/rkhs_norm": 242.3657684326172,
      "rewards/chosen": 0.5266698663884943,
      "rewards/margins": 0.7974940772629904,
      "rewards/rejected": -0.270824210874496,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 16.916542053222656,
      "kl": 15.56737995147705,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34568906.49517685,
      "logits/rejected": -35735589.34954407,
      "logps/chosen": -492.60651125401927,
      "logps/rejected": -358.8301671732523,
      "loss": 0.5585,
      "loss/base_kto": 3.189795732498169,
      "metrics/advantage_var": 529.7860717773438,
      "regularization/mmd": 1.2780075073242188,
      "regularization/rkhs_norm": 246.2442169189453,
      "rewards/chosen": 0.5155350173017986,
      "rewards/margins": 0.8675446601144173,
      "rewards/rejected": -0.3520096428126187,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 12.518861770629883,
      "kl": 7.532243251800537,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34510080.40829346,
      "logits/rejected": -36870633.2618683,
      "logps/chosen": -492.5997807017544,
      "logps/rejected": -385.3143903139357,
      "loss": 0.5723,
      "loss/base_kto": 3.241284132003784,
      "metrics/advantage_var": 557.99072265625,
      "regularization/mmd": 1.3371669054031372,
      "regularization/rkhs_norm": 257.6429443359375,
      "rewards/chosen": 0.408393920703748,
      "rewards/margins": 0.8317354468929709,
      "rewards/rejected": -0.4233415261892228,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 13.507658004760742,
      "kl": 11.165776252746582,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34689529.7939394,
      "logits/rejected": -37316611.30322581,
      "logps/chosen": -496.69389204545456,
      "logps/rejected": -379.5487147177419,
      "loss": 0.5758,
      "loss/base_kto": 3.2763335704803467,
      "metrics/advantage_var": 578.8201293945312,
      "regularization/mmd": 1.330436110496521,
      "regularization/rkhs_norm": 256.3460388183594,
      "rewards/chosen": 0.46783918900923294,
      "rewards/margins": 0.8089585245529578,
      "rewards/rejected": -0.3411193355437248,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 12.720839500427246,
      "kl": 14.162638664245605,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34116387.395498395,
      "logits/rejected": -37287929.77507599,
      "logps/chosen": -483.574959807074,
      "logps/rejected": -360.15197568389056,
      "loss": 0.5673,
      "loss/base_kto": 3.2285149097442627,
      "metrics/advantage_var": 568.7373657226562,
      "regularization/mmd": 1.309622049331665,
      "regularization/rkhs_norm": 252.3356475830078,
      "rewards/chosen": 0.5008895726832546,
      "rewards/margins": 0.843100522056355,
      "rewards/rejected": -0.3422109493731003,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 10.554686546325684,
      "kl": 15.69724178314209,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34917303.66091052,
      "logits/rejected": -35976059.81959564,
      "logps/chosen": -457.6415816326531,
      "logps/rejected": -373.05538005443236,
      "loss": 0.5586,
      "loss/base_kto": 3.2105298042297363,
      "metrics/advantage_var": 482.99664306640625,
      "regularization/mmd": 1.2578706741333008,
      "regularization/rkhs_norm": 242.36428833007812,
      "rewards/chosen": 0.5146402930913953,
      "rewards/margins": 0.851425189823328,
      "rewards/rejected": -0.33678489673193285,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 12.624411582946777,
      "kl": 18.55557632446289,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -35022171.62519201,
      "logits/rejected": -35563142.308426075,
      "logps/chosen": -483.1249039938556,
      "logps/rejected": -375.14288553259144,
      "loss": 0.5618,
      "loss/base_kto": 3.19071626663208,
      "metrics/advantage_var": 546.0989379882812,
      "regularization/mmd": 1.3034061193466187,
      "regularization/rkhs_norm": 251.1379852294922,
      "rewards/chosen": 0.6021119570402506,
      "rewards/margins": 0.8761229352174167,
      "rewards/rejected": -0.27401097817716613,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 18.09439468383789,
      "kl": 11.00549602508545,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -36292341.85736925,
      "logits/rejected": -36865038.20030817,
      "logps/chosen": -484.70701267828844,
      "logps/rejected": -381.20040446841296,
      "loss": 0.5723,
      "loss/base_kto": 3.2627198696136475,
      "metrics/advantage_var": 536.9502563476562,
      "regularization/mmd": 1.3156884908676147,
      "regularization/rkhs_norm": 253.50454711914062,
      "rewards/chosen": 0.43750696439183834,
      "rewards/margins": 0.7933569254948574,
      "rewards/rejected": -0.3558499611030191,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 15.669052124023438,
      "kl": 14.620813369750977,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -35399084.8,
      "logits/rejected": -36422662.4,
      "logps/chosen": -490.25322265625,
      "logps/rejected": -379.8054931640625,
      "loss": 0.5619,
      "loss/base_kto": 3.240927219390869,
      "metrics/advantage_var": 489.51904296875,
      "regularization/mmd": 1.2539480924606323,
      "regularization/rkhs_norm": 241.6085205078125,
      "rewards/chosen": 0.4669234275817871,
      "rewards/margins": 0.7803276538848878,
      "rewards/rejected": -0.3134042263031006,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 13.812790870666504,
      "kl": 16.63750648498535,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -36057573.86186186,
      "logits/rejected": -36731967.37459283,
      "logps/chosen": -467.51318506006004,
      "logps/rejected": -372.22892915309444,
      "loss": 0.5633,
      "loss/base_kto": 3.2285385131835938,
      "metrics/advantage_var": 585.1842041015625,
      "regularization/mmd": 1.277657151222229,
      "regularization/rkhs_norm": 246.1767120361328,
      "rewards/chosen": 0.6083782757366741,
      "rewards/margins": 0.8179173599071368,
      "rewards/rejected": -0.20953908417046263,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 12.299062728881836,
      "kl": 12.596667289733887,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -36139868.28840125,
      "logits/rejected": -38455142.87850467,
      "logps/chosen": -481.437059169279,
      "logps/rejected": -380.82138337227417,
      "loss": 0.5702,
      "loss/base_kto": 3.2609164714813232,
      "metrics/advantage_var": 542.4650268554688,
      "regularization/mmd": 1.301024317741394,
      "regularization/rkhs_norm": 250.6790313720703,
      "rewards/chosen": 0.5080097162611432,
      "rewards/margins": 0.7714715333518997,
      "rewards/rejected": -0.26346181709075644,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 14.116256713867188,
      "kl": 15.492639541625977,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34781736.23420647,
      "logits/rejected": -37028602.725832015,
      "logps/chosen": -481.3108628659476,
      "logps/rejected": -358.75680962757525,
      "loss": 0.5497,
      "loss/base_kto": 3.203596830368042,
      "metrics/advantage_var": 437.26300048828125,
      "regularization/mmd": 1.193867564201355,
      "regularization/rkhs_norm": 230.0322723388672,
      "rewards/chosen": 0.5390303366356896,
      "rewards/margins": 0.8165800440151592,
      "rewards/rejected": -0.2775497073794696,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 15.030481338500977,
      "kl": 11.008419036865234,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -35722243.210031345,
      "logits/rejected": -35698132.93457944,
      "logps/chosen": -470.43172021943576,
      "logps/rejected": -367.1655957943925,
      "loss": 0.5676,
      "loss/base_kto": 3.234706163406372,
      "metrics/advantage_var": 517.1394653320312,
      "regularization/mmd": 1.306067705154419,
      "regularization/rkhs_norm": 251.6508026123047,
      "rewards/chosen": 0.47645769747073374,
      "rewards/margins": 0.8401498652341318,
      "rewards/rejected": -0.3636921677633981,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 13.965100288391113,
      "kl": 10.82063102722168,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -36113708.39546191,
      "logits/rejected": -36955693.56259427,
      "logps/chosen": -496.9145563209076,
      "logps/rejected": -382.8325791855204,
      "loss": 0.5697,
      "loss/base_kto": 3.221923589706421,
      "metrics/advantage_var": 579.80224609375,
      "regularization/mmd": 1.3358854055404663,
      "regularization/rkhs_norm": 257.39605712890625,
      "rewards/chosen": 0.4650991183239212,
      "rewards/margins": 0.8364370549784508,
      "rewards/rejected": -0.3713379366545296,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 13.135164260864258,
      "kl": 11.971667289733887,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -37069436.937799044,
      "logits/rejected": -36610599.987748854,
      "logps/chosen": -501.1855562200957,
      "logps/rejected": -386.27837863705975,
      "loss": 0.5612,
      "loss/base_kto": 3.246180772781372,
      "metrics/advantage_var": 493.3094787597656,
      "regularization/mmd": 1.2436574697494507,
      "regularization/rkhs_norm": 239.62571716308594,
      "rewards/chosen": 0.45027951570599084,
      "rewards/margins": 0.791760629577964,
      "rewards/rejected": -0.3414811138719731,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 12.561501502990723,
      "kl": 8.76456356048584,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -34343353.35193799,
      "logits/rejected": -35630023.55905512,
      "logps/chosen": -477.0470930232558,
      "logps/rejected": -397.0420275590551,
      "loss": 0.5587,
      "loss/base_kto": 3.1628811359405518,
      "metrics/advantage_var": 565.871337890625,
      "regularization/mmd": 1.3064866065979004,
      "regularization/rkhs_norm": 251.73155212402344,
      "rewards/chosen": 0.4099231808684593,
      "rewards/margins": 0.9257905433038678,
      "rewards/rejected": -0.5158673624354084,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 27.34815216064453,
      "kl": 16.869966506958008,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -35392244.38184664,
      "logits/rejected": -35469979.756630264,
      "logps/chosen": -476.41035798122067,
      "logps/rejected": -366.750609399376,
      "loss": 0.5573,
      "loss/base_kto": 3.191033124923706,
      "metrics/advantage_var": 543.9557495117188,
      "regularization/mmd": 1.2676764726638794,
      "regularization/rkhs_norm": 244.253662109375,
      "rewards/chosen": 0.5922295655443075,
      "rewards/margins": 0.8774939125195538,
      "rewards/rejected": -0.2852643469752462,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 10.379315376281738,
      "kl": 10.070303916931152,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -36206361.6,
      "logits/rejected": -35660636.8,
      "logps/chosen": -488.068408203125,
      "logps/rejected": -367.74365234375,
      "loss": 0.5692,
      "loss/base_kto": 3.191129684448242,
      "metrics/advantage_var": 589.2523803710938,
      "regularization/mmd": 1.3624193668365479,
      "regularization/rkhs_norm": 262.508544921875,
      "rewards/chosen": 0.4839737892150879,
      "rewards/margins": 0.8736880779266357,
      "rewards/rejected": -0.38971428871154784,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 10.132719039916992,
      "kl": 8.817431449890137,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -35998789.96273292,
      "logits/rejected": -35578049.20754717,
      "logps/chosen": -475.36733307453414,
      "logps/rejected": -383.6253439465409,
      "loss": 0.5637,
      "loss/base_kto": 3.250783920288086,
      "metrics/advantage_var": 499.1070251464844,
      "regularization/mmd": 1.258469820022583,
      "regularization/rkhs_norm": 242.479736328125,
      "rewards/chosen": 0.4158513797736316,
      "rewards/margins": 0.8038303713960766,
      "rewards/rejected": -0.387978991622445,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 11.175532341003418,
      "kl": 12.814590454101562,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -34154521.20615385,
      "logits/rejected": -35270460.952380955,
      "logps/chosen": -494.93375,
      "logps/rejected": -365.18846726190475,
      "loss": 0.5665,
      "loss/base_kto": 3.188995361328125,
      "metrics/advantage_var": 550.5657958984375,
      "regularization/mmd": 1.343163013458252,
      "regularization/rkhs_norm": 258.79827880859375,
      "rewards/chosen": 0.5832626108022837,
      "rewards/margins": 0.8485274307195084,
      "rewards/rejected": -0.2652648199172247,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 10.874167442321777,
      "kl": 10.524325370788574,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -36449446.84984025,
      "logits/rejected": -37267568.73394495,
      "logps/chosen": -480.68310702875397,
      "logps/rejected": -368.5727255351682,
      "loss": 0.5643,
      "loss/base_kto": 3.2181408405303955,
      "metrics/advantage_var": 512.5886840820312,
      "regularization/mmd": 1.2961418628692627,
      "regularization/rkhs_norm": 249.73828125,
      "rewards/chosen": 0.42515188588882785,
      "rewards/margins": 0.8297866007445236,
      "rewards/rejected": -0.40463471485569574,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 12.937446594238281,
      "kl": 11.89905834197998,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -35112049.06998445,
      "logits/rejected": -35724656.1255887,
      "logps/chosen": -496.1260206065319,
      "logps/rejected": -373.39602138932497,
      "loss": 0.5667,
      "loss/base_kto": 3.214163303375244,
      "metrics/advantage_var": 615.8258056640625,
      "regularization/mmd": 1.3195157051086426,
      "regularization/rkhs_norm": 254.241943359375,
      "rewards/chosen": 0.4987821920107164,
      "rewards/margins": 0.8687606850230054,
      "rewards/rejected": -0.369978493012289,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 10.794614791870117,
      "kl": 13.87277889251709,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35004384.540706605,
      "logits/rejected": -36439240.880382776,
      "logps/chosen": -493.66599462365593,
      "logps/rejected": -364.7004336124402,
      "loss": 0.5356,
      "loss/base_kto": 3.0849404335021973,
      "metrics/advantage_var": 414.8458557128906,
      "regularization/mmd": 1.2002458572387695,
      "regularization/rkhs_norm": 231.26126098632812,
      "rewards/chosen": 0.5897390714255712,
      "rewards/margins": 0.9555537427974936,
      "rewards/rejected": -0.36581467137192236,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 12.27173137664795,
      "kl": 11.178915977478027,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34765633.084745765,
      "logits/rejected": -35100467.524564184,
      "logps/chosen": -476.7357473035439,
      "logps/rejected": -405.81658577654514,
      "loss": 0.5304,
      "loss/base_kto": 3.0577104091644287,
      "metrics/advantage_var": 396.5050048828125,
      "regularization/mmd": 1.1852072477340698,
      "regularization/rkhs_norm": 228.3636474609375,
      "rewards/chosen": 0.5489674995786787,
      "rewards/margins": 0.9826593314304022,
      "rewards/rejected": -0.43369183185172344,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 13.18808650970459,
      "kl": 15.312929153442383,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33659943.203675345,
      "logits/rejected": -35243408.12759171,
      "logps/chosen": -491.76722817764164,
      "logps/rejected": -389.0051086523126,
      "loss": 0.5251,
      "loss/base_kto": 3.044113874435425,
      "metrics/advantage_var": 385.6358642578125,
      "regularization/mmd": 1.1567519903182983,
      "regularization/rkhs_norm": 222.88096618652344,
      "rewards/chosen": 0.6191242679520004,
      "rewards/margins": 0.9529156083190757,
      "rewards/rejected": -0.33379134036707536,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 13.835762023925781,
      "kl": 12.04203987121582,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -36170506.720268,
      "logits/rejected": -36780595.724743776,
      "logps/chosen": -481.39175041876047,
      "logps/rejected": -405.1716233528551,
      "loss": 0.5212,
      "loss/base_kto": 3.049400806427002,
      "metrics/advantage_var": 360.25384521484375,
      "regularization/mmd": 1.1205371618270874,
      "regularization/rkhs_norm": 215.9031219482422,
      "rewards/chosen": 0.5269861269236809,
      "rewards/margins": 0.9735436368751815,
      "rewards/rejected": -0.44655750995150073,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 10.22216510772705,
      "kl": 9.879226684570312,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -33514864.252365932,
      "logits/rejected": -35528998.835913315,
      "logps/chosen": -479.18661277602524,
      "logps/rejected": -365.45714009287923,
      "loss": 0.5247,
      "loss/base_kto": 3.052150011062622,
      "metrics/advantage_var": 372.58612060546875,
      "regularization/mmd": 1.1455343961715698,
      "regularization/rkhs_norm": 220.7195281982422,
      "rewards/chosen": 0.5359188043732749,
      "rewards/margins": 0.9770049504887934,
      "rewards/rejected": -0.44108614611551855,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 12.031999588012695,
      "kl": 11.795473098754883,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -35341678.05984252,
      "logits/rejected": -34914402.43100775,
      "logps/chosen": -479.39207677165354,
      "logps/rejected": -380.02703488372094,
      "loss": 0.5262,
      "loss/base_kto": 3.060995578765869,
      "metrics/advantage_var": 370.60595703125,
      "regularization/mmd": 1.1483268737792969,
      "regularization/rkhs_norm": 221.25759887695312,
      "rewards/chosen": 0.5401754544475886,
      "rewards/margins": 0.9658052047806798,
      "rewards/rejected": -0.4256297503330911,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 7.272770881652832,
      "kl": 12.063289642333984,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -35471808.28785607,
      "logits/rejected": -37075782.57748777,
      "logps/chosen": -483.42789542728633,
      "logps/rejected": -373.52492862969007,
      "loss": 0.5359,
      "loss/base_kto": 3.1197516918182373,
      "metrics/advantage_var": 397.1454772949219,
      "regularization/mmd": 1.1671723127365112,
      "regularization/rkhs_norm": 224.88870239257812,
      "rewards/chosen": 0.5531095536216267,
      "rewards/margins": 0.9306453005087342,
      "rewards/rejected": -0.37753574688710745,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 10.803357124328613,
      "kl": 18.992996215820312,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -35726013.05863708,
      "logits/rejected": -35897009.50385208,
      "logps/chosen": -477.5151545166403,
      "logps/rejected": -355.7731847072419,
      "loss": 0.5222,
      "loss/base_kto": 3.07306170463562,
      "metrics/advantage_var": 355.2674560546875,
      "regularization/mmd": 1.1047872304916382,
      "regularization/rkhs_norm": 212.8684539794922,
      "rewards/chosen": 0.6384236234492621,
      "rewards/margins": 0.9141885942727646,
      "rewards/rejected": -0.2757649708235025,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 12.077045440673828,
      "kl": 10.724763870239258,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -33900746.54945055,
      "logits/rejected": -34756350.009331256,
      "logps/chosen": -454.79341640502355,
      "logps/rejected": -357.78628013219287,
      "loss": 0.5241,
      "loss/base_kto": 3.111751079559326,
      "metrics/advantage_var": 340.6296081542969,
      "regularization/mmd": 1.0811723470687866,
      "regularization/rkhs_norm": 208.31838989257812,
      "rewards/chosen": 0.4972784260939953,
      "rewards/margins": 0.925645127211195,
      "rewards/rejected": -0.42836670111719966,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 13.94926643371582,
      "kl": 12.412457466125488,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -33723913.66037736,
      "logits/rejected": -36846881.39130435,
      "logps/chosen": -471.5267786949685,
      "logps/rejected": -375.14635093167703,
      "loss": 0.5255,
      "loss/base_kto": 3.0692825317382812,
      "metrics/advantage_var": 370.166015625,
      "regularization/mmd": 1.1345977783203125,
      "regularization/rkhs_norm": 218.6123046875,
      "rewards/chosen": 0.563676654167895,
      "rewards/margins": 0.9501858801435477,
      "rewards/rejected": -0.3865092259756527,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 11.850485801696777,
      "kl": 9.79004192352295,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34795158.5882353,
      "logits/rejected": -36215982.59907834,
      "logps/chosen": -492.7114467408585,
      "logps/rejected": -373.8477822580645,
      "loss": 0.5301,
      "loss/base_kto": 3.0690057277679443,
      "metrics/advantage_var": 401.89691162109375,
      "regularization/mmd": 1.1717764139175415,
      "regularization/rkhs_norm": 225.7758026123047,
      "rewards/chosen": 0.5096414199125596,
      "rewards/margins": 0.9560450991230292,
      "rewards/rejected": -0.4464036792104695,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 9.599963188171387,
      "kl": 11.842473030090332,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -36198355.616099074,
      "logits/rejected": -34933575.873817034,
      "logps/chosen": -469.62267801857587,
      "logps/rejected": -392.19395701892745,
      "loss": 0.528,
      "loss/base_kto": 3.074521780014038,
      "metrics/advantage_var": 387.1634216308594,
      "regularization/mmd": 1.1494674682617188,
      "regularization/rkhs_norm": 221.4773406982422,
      "rewards/chosen": 0.550881636770148,
      "rewards/margins": 0.9560462892124785,
      "rewards/rejected": -0.40516465244233046,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 12.557998657226562,
      "kl": 11.34885025024414,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34241445.46341463,
      "logits/rejected": -36624948.51282051,
      "logps/chosen": -456.6895960365854,
      "logps/rejected": -374.07882612179486,
      "loss": 0.5255,
      "loss/base_kto": 3.1066834926605225,
      "metrics/advantage_var": 336.653076171875,
      "regularization/mmd": 1.0975435972213745,
      "regularization/rkhs_norm": 211.47276306152344,
      "rewards/chosen": 0.5479634447795588,
      "rewards/margins": 0.8984530611736018,
      "rewards/rejected": -0.35048961639404297,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 9.489789009094238,
      "kl": 13.431262016296387,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -36068473.018181816,
      "logits/rejected": -37891844.95483871,
      "logps/chosen": -501.8767518939394,
      "logps/rejected": -363.4361391129032,
      "loss": 0.5271,
      "loss/base_kto": 3.0921616554260254,
      "metrics/advantage_var": 364.2037048339844,
      "regularization/mmd": 1.1243157386779785,
      "regularization/rkhs_norm": 216.6311492919922,
      "rewards/chosen": 0.5953232273910984,
      "rewards/margins": 0.9308806604886917,
      "rewards/rejected": -0.3355574330975932,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 13.395483016967773,
      "kl": 15.368057250976562,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -34993718.80122324,
      "logits/rejected": -36962902.69648562,
      "logps/chosen": -499.00095565749234,
      "logps/rejected": -394.44473841853033,
      "loss": 0.5285,
      "loss/base_kto": 3.074232339859009,
      "metrics/advantage_var": 380.4449462890625,
      "regularization/mmd": 1.1537588834762573,
      "regularization/rkhs_norm": 222.30422973632812,
      "rewards/chosen": 0.673077364580347,
      "rewards/margins": 0.9492099664798078,
      "rewards/rejected": -0.27613260189946087,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 9.622467041015625,
      "kl": 12.925310134887695,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35845182.43902439,
      "logits/rejected": -37073814.974358976,
      "logps/chosen": -488.638671875,
      "logps/rejected": -349.1306590544872,
      "loss": 0.5212,
      "loss/base_kto": 3.0903120040893555,
      "metrics/advantage_var": 329.059326171875,
      "regularization/mmd": 1.0790796279907227,
      "regularization/rkhs_norm": 207.91514587402344,
      "rewards/chosen": 0.5687211199504573,
      "rewards/margins": 0.9100569790046315,
      "rewards/rejected": -0.3413358590541742,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.48875617980957,
      "kl": 12.72642707824707,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -34771179.06918239,
      "logits/rejected": -36125533.81366459,
      "logps/chosen": -471.2513757861635,
      "logps/rejected": -365.42245729813664,
      "loss": 0.5281,
      "loss/base_kto": 3.0797269344329834,
      "metrics/advantage_var": 402.2024230957031,
      "regularization/mmd": 1.1452440023422241,
      "regularization/rkhs_norm": 220.66357421875,
      "rewards/chosen": 0.5790888948260613,
      "rewards/margins": 0.9379967010329955,
      "rewards/rejected": -0.3589078062069342,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 11.197590827941895,
      "kl": 12.252760887145996,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35302467.637480795,
      "logits/rejected": -36032420.83306836,
      "logps/chosen": -480.44945276497697,
      "logps/rejected": -385.88409181240064,
      "loss": 0.5272,
      "loss/base_kto": 3.123046636581421,
      "metrics/advantage_var": 339.9273376464844,
      "regularization/mmd": 1.0945415496826172,
      "regularization/rkhs_norm": 210.8943328857422,
      "rewards/chosen": 0.5276418940812212,
      "rewards/margins": 0.8905536096872799,
      "rewards/rejected": -0.3629117156060587,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 10.127400398254395,
      "kl": 13.046516418457031,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -33633762.78678679,
      "logits/rejected": -36871071.27035831,
      "logps/chosen": -498.67966403903904,
      "logps/rejected": -379.32983000814335,
      "loss": 0.5316,
      "loss/base_kto": 3.103759288787842,
      "metrics/advantage_var": 375.3100280761719,
      "regularization/mmd": 1.1488044261932373,
      "regularization/rkhs_norm": 221.34959411621094,
      "rewards/chosen": 0.6095284126900338,
      "rewards/margins": 0.9271969919904078,
      "rewards/rejected": -0.31766857930037407,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 8.174375534057617,
      "kl": 14.385890007019043,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34574600.05031446,
      "logits/rejected": -36766421.06832298,
      "logps/chosen": -460.4575963050315,
      "logps/rejected": -373.6992915372671,
      "loss": 0.5264,
      "loss/base_kto": 3.0298900604248047,
      "metrics/advantage_var": 427.42755126953125,
      "regularization/mmd": 1.181685447692871,
      "regularization/rkhs_norm": 227.68505859375,
      "rewards/chosen": 0.6774577494687254,
      "rewards/margins": 1.0040204295413915,
      "rewards/rejected": -0.326562680072666,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 9.178069114685059,
      "kl": 14.07157039642334,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34065085.14649682,
      "logits/rejected": -34723500.7607362,
      "logps/chosen": -460.515226910828,
      "logps/rejected": -357.27255080521473,
      "loss": 0.5267,
      "loss/base_kto": 3.108546733856201,
      "metrics/advantage_var": 347.3668518066406,
      "regularization/mmd": 1.1048446893692017,
      "regularization/rkhs_norm": 212.87950134277344,
      "rewards/chosen": 0.5507369314789012,
      "rewards/margins": 0.9131097686625669,
      "rewards/rejected": -0.36237283718366564,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 11.285250663757324,
      "kl": 12.09557056427002,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -36527692.11893584,
      "logits/rejected": -37667141.89079563,
      "logps/chosen": -496.76066118935836,
      "logps/rejected": -353.9441302652106,
      "loss": 0.5238,
      "loss/base_kto": 3.1014554500579834,
      "metrics/advantage_var": 346.4856872558594,
      "regularization/mmd": 1.088848352432251,
      "regularization/rkhs_norm": 209.79734802246094,
      "rewards/chosen": 0.5277840818784233,
      "rewards/margins": 0.905094729113883,
      "rewards/rejected": -0.3773106472354597,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.444851875305176,
      "kl": 15.103259086608887,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -36318720.0,
      "logits/rejected": -36915378.60465116,
      "logps/chosen": -472.03832347972974,
      "logps/rejected": -412.3602834302326,
      "loss": 0.5276,
      "loss/base_kto": 3.086003065109253,
      "metrics/advantage_var": 379.9912414550781,
      "regularization/mmd": 1.134604811668396,
      "regularization/rkhs_norm": 218.6136474609375,
      "rewards/chosen": 0.5286396129711254,
      "rewards/margins": 0.9213921938356103,
      "rewards/rejected": -0.39275258086448495,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 11.337721824645996,
      "kl": 10.847223281860352,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35732298.73291925,
      "logits/rejected": -36615844.2264151,
      "logps/chosen": -485.6144701086956,
      "logps/rejected": -377.01110455974845,
      "loss": 0.526,
      "loss/base_kto": 3.108407974243164,
      "metrics/advantage_var": 344.0582580566406,
      "regularization/mmd": 1.0993179082870483,
      "regularization/rkhs_norm": 211.8146209716797,
      "rewards/chosen": 0.5150987139399748,
      "rewards/margins": 0.9081066281652352,
      "rewards/rejected": -0.39300791422526044,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 10.461943626403809,
      "kl": 12.795968055725098,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35372848.898876406,
      "logits/rejected": -36322177.75342466,
      "logps/chosen": -487.4076043338684,
      "logps/rejected": -386.2137081430746,
      "loss": 0.5227,
      "loss/base_kto": 3.0303561687469482,
      "metrics/advantage_var": 388.92620849609375,
      "regularization/mmd": 1.1511913537979126,
      "regularization/rkhs_norm": 221.8095245361328,
      "rewards/chosen": 0.6047032143484149,
      "rewards/margins": 1.0036851070662611,
      "rewards/rejected": -0.3989818927178463,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 11.451590538024902,
      "kl": 10.240742683410645,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -36673216.29675425,
      "logits/rejected": -37039791.519747235,
      "logps/chosen": -513.1583268933539,
      "logps/rejected": -368.4791172985782,
      "loss": 0.5303,
      "loss/base_kto": 3.0926804542541504,
      "metrics/advantage_var": 380.3329772949219,
      "regularization/mmd": 1.1495593786239624,
      "regularization/rkhs_norm": 221.4950714111328,
      "rewards/chosen": 0.5325430585575252,
      "rewards/margins": 0.9418068260831965,
      "rewards/rejected": -0.4092637675256714,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 12.47182846069336,
      "kl": 12.153326034545898,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -37296432.0,
      "logits/rejected": -36498425.6,
      "logps/chosen": -485.547412109375,
      "logps/rejected": -377.9694580078125,
      "loss": 0.5248,
      "loss/base_kto": 3.0201194286346436,
      "metrics/advantage_var": 419.91796875,
      "regularization/mmd": 1.1779553890228271,
      "regularization/rkhs_norm": 226.9663543701172,
      "rewards/chosen": 0.5866191864013672,
      "rewards/margins": 1.0076179504394531,
      "rewards/rejected": -0.42099876403808595,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 11.72092056274414,
      "kl": 13.146728515625,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -35183640.497607656,
      "logits/rejected": -36547529.114854515,
      "logps/chosen": -489.65081738437004,
      "logps/rejected": -365.09939701378255,
      "loss": 0.5224,
      "loss/base_kto": 3.085169553756714,
      "metrics/advantage_var": 342.1353454589844,
      "regularization/mmd": 1.0941046476364136,
      "regularization/rkhs_norm": 210.8101043701172,
      "rewards/chosen": 0.5412538275954446,
      "rewards/margins": 0.9198822881859883,
      "rewards/rejected": -0.37862846059054367,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 15.549859046936035,
      "kl": 10.180974006652832,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -33732301.12302839,
      "logits/rejected": -36279245.2755418,
      "logps/chosen": -458.020011829653,
      "logps/rejected": -383.98679373065016,
      "loss": 0.5301,
      "loss/base_kto": 3.090721368789673,
      "metrics/advantage_var": 371.25335693359375,
      "regularization/mmd": 1.1504759788513184,
      "regularization/rkhs_norm": 221.6716766357422,
      "rewards/chosen": 0.5378277895954505,
      "rewards/margins": 0.9477921473317605,
      "rewards/rejected": -0.40996435773630996,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 12.656947135925293,
      "kl": 11.5225191116333,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -36261612.8,
      "logits/rejected": -36237446.4,
      "logps/chosen": -469.375390625,
      "logps/rejected": -401.649951171875,
      "loss": 0.5229,
      "loss/base_kto": 3.0506389141082764,
      "metrics/advantage_var": 367.6473083496094,
      "regularization/mmd": 1.1327515840530396,
      "regularization/rkhs_norm": 218.25657653808594,
      "rewards/chosen": 0.5057161331176758,
      "rewards/margins": 0.9704075813293458,
      "rewards/rejected": -0.4646914482116699,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 8.844049453735352,
      "kl": 12.70142650604248,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -35099579.07692308,
      "logits/rejected": -36255481.75609756,
      "logps/chosen": -491.3915264423077,
      "logps/rejected": -377.7775342987805,
      "loss": 0.522,
      "loss/base_kto": 3.0711255073547363,
      "metrics/advantage_var": 336.4460754394531,
      "regularization/mmd": 1.1048855781555176,
      "regularization/rkhs_norm": 212.8874053955078,
      "rewards/chosen": 0.5291022276267027,
      "rewards/margins": 0.9443479350688235,
      "rewards/rejected": -0.4152457074421208,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 12.089055061340332,
      "kl": 11.538997650146484,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -34441206.33962264,
      "logits/rejected": -36088154.63354037,
      "logps/chosen": -470.83431603773585,
      "logps/rejected": -372.7272175854037,
      "loss": 0.5384,
      "loss/base_kto": 3.090852975845337,
      "metrics/advantage_var": 444.470703125,
      "regularization/mmd": 1.2166645526885986,
      "regularization/rkhs_norm": 234.4247589111328,
      "rewards/chosen": 0.5174438668496953,
      "rewards/margins": 0.9640238414803,
      "rewards/rejected": -0.4465799746306046,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 12.057787895202637,
      "kl": 10.492204666137695,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35121621.13112164,
      "logits/rejected": -35171453.03245749,
      "logps/chosen": -481.2192436808847,
      "logps/rejected": -381.671271251932,
      "loss": 0.5282,
      "loss/base_kto": 3.0740230083465576,
      "metrics/advantage_var": 378.0727233886719,
      "regularization/mmd": 1.1514508724212646,
      "regularization/rkhs_norm": 221.8594970703125,
      "rewards/chosen": 0.5253483439320449,
      "rewards/margins": 0.9607945156410141,
      "rewards/rejected": -0.43544617170896927,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 8.149927139282227,
      "kl": 9.867273330688477,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35843470.39513678,
      "logits/rejected": -34951922.00643087,
      "logps/chosen": -478.296210106383,
      "logps/rejected": -377.64585008038586,
      "loss": 0.5301,
      "loss/base_kto": 3.1066694259643555,
      "metrics/advantage_var": 368.0716247558594,
      "regularization/mmd": 1.1340526342391968,
      "regularization/rkhs_norm": 218.50723266601562,
      "rewards/chosen": 0.511175370143902,
      "rewards/margins": 0.9317512142738508,
      "rewards/rejected": -0.42057584412994875,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 9.785394668579102,
      "kl": 11.317770957946777,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -36203450.87598116,
      "logits/rejected": -37434726.32037325,
      "logps/chosen": -493.74921507064363,
      "logps/rejected": -365.62249708398133,
      "loss": 0.5311,
      "loss/base_kto": 3.1075687408447266,
      "metrics/advantage_var": 387.37799072265625,
      "regularization/mmd": 1.1412509679794312,
      "regularization/rkhs_norm": 219.8942108154297,
      "rewards/chosen": 0.5315692608173077,
      "rewards/margins": 0.9077749187288309,
      "rewards/rejected": -0.37620565791152316,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.527176856994629,
      "kl": 11.023845672607422,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -36198295.809220985,
      "logits/rejected": -36452307.95698924,
      "logps/chosen": -480.5086943561208,
      "logps/rejected": -389.5552515360983,
      "loss": 0.5263,
      "loss/base_kto": 3.082139730453491,
      "metrics/advantage_var": 357.8573303222656,
      "regularization/mmd": 1.1284321546554565,
      "regularization/rkhs_norm": 217.42430114746094,
      "rewards/chosen": 0.5544601464688493,
      "rewards/margins": 0.9547959242086247,
      "rewards/rejected": -0.40033577773977536,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 9.05074691772461,
      "kl": 13.44152545928955,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35885581.385620914,
      "logits/rejected": -36495476.50299401,
      "logps/chosen": -461.20680147058823,
      "logps/rejected": -371.7387022829341,
      "loss": 0.5211,
      "loss/base_kto": 3.0867555141448975,
      "metrics/advantage_var": 348.108154296875,
      "regularization/mmd": 1.0818575620651245,
      "regularization/rkhs_norm": 208.4503936767578,
      "rewards/chosen": 0.5207149530547897,
      "rewards/margins": 0.9165391798453492,
      "rewards/rejected": -0.3958242267905595,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 7.720363616943359,
      "kl": 7.960742473602295,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -35646991.23809524,
      "logits/rejected": -36893005.473684214,
      "logps/chosen": -502.2574869791667,
      "logps/rejected": -359.7631578947368,
      "loss": 0.5248,
      "loss/base_kto": 3.085153579711914,
      "metrics/advantage_var": 348.100341796875,
      "regularization/mmd": 1.1133981943130493,
      "regularization/rkhs_norm": 214.52757263183594,
      "rewards/chosen": 0.5423346019926525,
      "rewards/margins": 0.9594680336782508,
      "rewards/rejected": -0.4171334316855983,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.969850461966828e+17,
      "train_loss": 0.5584362290272029,
      "train_runtime": 11031.6005,
      "train_samples_per_second": 13.436,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.969850461966828e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}