{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 15.313405990600586,
      "kl": 6.287380218505859,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36423765.73023256,
      "logits/rejected": -36855343.571653545,
      "logps/chosen": -500.0248546511628,
      "logps/rejected": -362.0871062992126,
      "loss": 0.5883,
      "loss/base_kto": 3.9272773265838623,
      "metrics/advantage_var": 169.6308135986328,
      "regularization/mmd": 0.7794427871704102,
      "regularization/rkhs_norm": 150.18167114257812,
      "rewards/chosen": 0.057254383175872094,
      "rewards/margins": 0.06401568852682905,
      "rewards/rejected": -0.006761305350956954,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 19.27733039855957,
      "kl": 8.419197082519531,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35511164.47706422,
      "logits/rejected": -37369142.79872204,
      "logps/chosen": -485.55103211009174,
      "logps/rejected": -375.57071186102235,
      "loss": 0.5901,
      "loss/base_kto": 3.922342300415039,
      "metrics/advantage_var": 186.844970703125,
      "regularization/mmd": 0.798768937587738,
      "regularization/rkhs_norm": 153.9053955078125,
      "rewards/chosen": 0.09748388284572403,
      "rewards/margins": 0.0519025923294953,
      "rewards/rejected": 0.04558129051622872,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 18.518787384033203,
      "kl": 14.31860065460205,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36978193.22324159,
      "logits/rejected": -37764125.44408946,
      "logps/chosen": -468.082377675841,
      "logps/rejected": -352.98507388178916,
      "loss": 0.579,
      "loss/base_kto": 3.8540725708007812,
      "metrics/advantage_var": 169.3621826171875,
      "regularization/mmd": 0.7777101397514343,
      "regularization/rkhs_norm": 149.84780883789062,
      "rewards/chosen": 0.2522563875997467,
      "rewards/margins": 0.1583661334906962,
      "rewards/rejected": 0.09389025410905052,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 20.603500366210938,
      "kl": 5.978942394256592,
      "learning_rate": 7.9e-07,
      "logits/chosen": -36682372.453857794,
      "logits/rejected": -37832872.73667205,
      "logps/chosen": -502.98165658093797,
      "logps/rejected": -383.53435480613894,
      "loss": 0.5794,
      "loss/base_kto": 3.853320837020874,
      "metrics/advantage_var": 174.62965393066406,
      "regularization/mmd": 0.7816373109817505,
      "regularization/rkhs_norm": 150.60447692871094,
      "rewards/chosen": 0.09290631131216906,
      "rewards/margins": 0.13982160289653522,
      "rewards/rejected": -0.046915291584366164,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 21.397480010986328,
      "kl": 12.552741050720215,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35619206.0952381,
      "logits/rejected": -37989193.25538462,
      "logps/chosen": -494.6147817460317,
      "logps/rejected": -360.1650480769231,
      "loss": 0.5866,
      "loss/base_kto": 3.8501617908477783,
      "metrics/advantage_var": 204.6127166748047,
      "regularization/mmd": 0.8428762555122375,
      "regularization/rkhs_norm": 162.40391540527344,
      "rewards/chosen": 0.21830361986917163,
      "rewards/margins": 0.13981841854705623,
      "rewards/rejected": 0.07848520132211538,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 15.790153503417969,
      "kl": 15.475531578063965,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35700546.427921094,
      "logits/rejected": -37738123.336553946,
      "logps/chosen": -481.3101764036419,
      "logps/rejected": -352.2597876409018,
      "loss": 0.579,
      "loss/base_kto": 3.777491807937622,
      "metrics/advantage_var": 209.87998962402344,
      "regularization/mmd": 0.8547720313072205,
      "regularization/rkhs_norm": 164.69595336914062,
      "rewards/chosen": 0.2858772104173582,
      "rewards/margins": 0.19888717339919193,
      "rewards/rejected": 0.08699003701816627,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 39.783966064453125,
      "kl": 11.525238037109375,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35765464.42902208,
      "logits/rejected": -37715473.436532505,
      "logps/chosen": -489.69104889589903,
      "logps/rejected": -387.1947078173375,
      "loss": 0.5811,
      "loss/base_kto": 3.787822723388672,
      "metrics/advantage_var": 236.62136840820312,
      "regularization/mmd": 0.8610712289810181,
      "regularization/rkhs_norm": 165.90966796875,
      "rewards/chosen": 0.1821763432740413,
      "rewards/margins": 0.1777836617979089,
      "rewards/rejected": 0.00439268147613242,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 16.868717193603516,
      "kl": 3.994859457015991,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -35069476.27018122,
      "logits/rejected": -37011054.312035665,
      "logps/chosen": -492.60337726523886,
      "logps/rejected": -384.65583209509657,
      "loss": 0.5817,
      "loss/base_kto": 3.8289992809295654,
      "metrics/advantage_var": 194.04469299316406,
      "regularization/mmd": 0.8245702981948853,
      "regularization/rkhs_norm": 158.87673950195312,
      "rewards/chosen": -0.04185859139707964,
      "rewards/margins": 0.14970385146748433,
      "rewards/rejected": -0.19156244286456398,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 16.328460693359375,
      "kl": 3.0741677284240723,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -37287643.428571425,
      "logits/rejected": -38373464.22153846,
      "logps/chosen": -507.6699404761905,
      "logps/rejected": -377.856875,
      "loss": 0.5845,
      "loss/base_kto": 3.7946877479553223,
      "metrics/advantage_var": 219.6715545654297,
      "regularization/mmd": 0.8813549280166626,
      "regularization/rkhs_norm": 169.8179168701172,
      "rewards/chosen": -0.04087443276057168,
      "rewards/margins": 0.19756701104952448,
      "rewards/rejected": -0.23844144381009616,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 18.04525375366211,
      "kl": 5.2681803703308105,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -37049502.315789476,
      "logits/rejected": -38178852.571428575,
      "logps/chosen": -487.16395970394734,
      "logps/rejected": -392.4440104166667,
      "loss": 0.5801,
      "loss/base_kto": 3.777139663696289,
      "metrics/advantage_var": 222.96934509277344,
      "regularization/mmd": 0.8638244867324829,
      "regularization/rkhs_norm": 166.44015502929688,
      "rewards/chosen": -0.014353609398791664,
      "rewards/margins": 0.18498335080338002,
      "rewards/rejected": -0.19933696020217168,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 18.46519660949707,
      "kl": 4.6849565505981445,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -34838346.531645566,
      "logits/rejected": -36299061.72839506,
      "logps/chosen": -496.81788963607596,
      "logps/rejected": -388.7009066358025,
      "loss": 0.5854,
      "loss/base_kto": 3.8158562183380127,
      "metrics/advantage_var": 220.43008422851562,
      "regularization/mmd": 0.866993248462677,
      "regularization/rkhs_norm": 167.0507354736328,
      "rewards/chosen": -0.006396480753451963,
      "rewards/margins": 0.17232688983393232,
      "rewards/rejected": -0.17872337058738427,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 17.094303131103516,
      "kl": 15.332864761352539,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36305746.33918129,
      "logits/rejected": -37991987.54362416,
      "logps/chosen": -503.6694078947368,
      "logps/rejected": -353.52385696308727,
      "loss": 0.5782,
      "loss/base_kto": 3.7261035442352295,
      "metrics/advantage_var": 227.9002685546875,
      "regularization/mmd": 0.8994961977005005,
      "regularization/rkhs_norm": 173.31333923339844,
      "rewards/chosen": 0.2943106088024831,
      "rewards/margins": 0.23246875288928248,
      "rewards/rejected": 0.061841855913200634,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 17.33824920654297,
      "kl": 14.3850679397583,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -35676741.530864194,
      "logits/rejected": -37013601.215189874,
      "logps/chosen": -485.3454861111111,
      "logps/rejected": -350.10759493670884,
      "loss": 0.5878,
      "loss/base_kto": 3.7968785762786865,
      "metrics/advantage_var": 234.15122985839844,
      "regularization/mmd": 0.9053724408149719,
      "regularization/rkhs_norm": 174.44554138183594,
      "rewards/chosen": 0.25135205115801024,
      "rewards/margins": 0.17415049762013446,
      "rewards/rejected": 0.07720155353787579,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 18.95519256591797,
      "kl": 7.190995693206787,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -35703291.82381729,
      "logits/rejected": -38211138.782608695,
      "logps/chosen": -489.71044045676996,
      "logps/rejected": -365.9665011244378,
      "loss": 0.5815,
      "loss/base_kto": 3.773439407348633,
      "metrics/advantage_var": 221.27774047851562,
      "regularization/mmd": 0.8783413171768188,
      "regularization/rkhs_norm": 169.2372283935547,
      "rewards/chosen": 0.05311624299838142,
      "rewards/margins": 0.18281556112496267,
      "rewards/rejected": -0.12969931812658125,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 13.247111320495605,
      "kl": 9.16240406036377,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -34967635.2,
      "logits/rejected": -36332473.6,
      "logps/chosen": -496.133203125,
      "logps/rejected": -380.3676025390625,
      "loss": 0.5856,
      "loss/base_kto": 3.7572200298309326,
      "metrics/advantage_var": 239.2593231201172,
      "regularization/mmd": 0.9278042912483215,
      "regularization/rkhs_norm": 178.76768493652344,
      "rewards/chosen": 0.1451313853263855,
      "rewards/margins": 0.22013087272644044,
      "rewards/rejected": -0.07499948740005494,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 16.3098087310791,
      "kl": 6.792578220367432,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -36317901.88770865,
      "logits/rejected": -36482547.632850245,
      "logps/chosen": -458.5691388467375,
      "logps/rejected": -377.06154388083735,
      "loss": 0.5841,
      "loss/base_kto": 3.8088531494140625,
      "metrics/advantage_var": 216.6083526611328,
      "regularization/mmd": 0.8642234802246094,
      "regularization/rkhs_norm": 166.51707458496094,
      "rewards/chosen": 0.04237878593942645,
      "rewards/margins": 0.1907803135284965,
      "rewards/rejected": -0.14840152758907005,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 11.708365440368652,
      "kl": 11.88242244720459,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36764419.98753894,
      "logits/rejected": -37439157.36677116,
      "logps/chosen": -474.22605140186914,
      "logps/rejected": -384.5523608934169,
      "loss": 0.5828,
      "loss/base_kto": 3.7636170387268066,
      "metrics/advantage_var": 235.3195343017578,
      "regularization/mmd": 0.8990959525108337,
      "regularization/rkhs_norm": 173.23623657226562,
      "rewards/chosen": 0.19089648285387462,
      "rewards/margins": 0.21902219198098388,
      "rewards/rejected": -0.028125709127109252,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 14.318680763244629,
      "kl": 8.728950500488281,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -37032318.20839813,
      "logits/rejected": -37374051.667189956,
      "logps/chosen": -493.59958203732504,
      "logps/rejected": -381.1712127158556,
      "loss": 0.5837,
      "loss/base_kto": 3.7761952877044678,
      "metrics/advantage_var": 229.3874053955078,
      "regularization/mmd": 0.8933205008506775,
      "regularization/rkhs_norm": 172.1234130859375,
      "rewards/chosen": 0.1257154070200112,
      "rewards/margins": 0.2105761084088149,
      "rewards/rejected": -0.08486070138880372,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 16.535734176635742,
      "kl": 11.216611862182617,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -36508873.6,
      "logits/rejected": -37794348.8,
      "logps/chosen": -500.35439453125,
      "logps/rejected": -369.51689453125,
      "loss": 0.5748,
      "loss/base_kto": 3.7446205615997314,
      "metrics/advantage_var": 202.92088317871094,
      "regularization/mmd": 0.8536295294761658,
      "regularization/rkhs_norm": 164.47581481933594,
      "rewards/chosen": 0.18334336280822755,
      "rewards/margins": 0.22333242297172548,
      "rewards/rejected": -0.03998906016349792,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 16.610736846923828,
      "kl": 9.801573753356934,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -35763556.67415731,
      "logits/rejected": -37100420.87062405,
      "logps/chosen": -495.1868980738363,
      "logps/rejected": -375.4642551369863,
      "loss": 0.5756,
      "loss/base_kto": 3.720391035079956,
      "metrics/advantage_var": 217.6693115234375,
      "regularization/mmd": 0.884712815284729,
      "regularization/rkhs_norm": 170.4649200439453,
      "rewards/chosen": 0.12955010721809793,
      "rewards/margins": 0.2572847167900455,
      "rewards/rejected": -0.12773460957194754,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 14.878124237060547,
      "kl": 10.152867317199707,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -35227657.95786062,
      "logits/rejected": -35691506.09954751,
      "logps/chosen": -483.7574959481361,
      "logps/rejected": -374.82847850678735,
      "loss": 0.5803,
      "loss/base_kto": 3.7252635955810547,
      "metrics/advantage_var": 236.1919403076172,
      "regularization/mmd": 0.9174157381057739,
      "regularization/rkhs_norm": 176.7660369873047,
      "rewards/chosen": 0.06529909749085165,
      "rewards/margins": 0.23877645446053253,
      "rewards/rejected": -0.1734773569696809,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 17.19070053100586,
      "kl": 4.2611212730407715,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -35364382.68769716,
      "logits/rejected": -36261510.7368421,
      "logps/chosen": -497.1975059148265,
      "logps/rejected": -363.3968895123839,
      "loss": 0.5845,
      "loss/base_kto": 3.769343137741089,
      "metrics/advantage_var": 237.5447235107422,
      "regularization/mmd": 0.9070194363594055,
      "regularization/rkhs_norm": 174.7628936767578,
      "rewards/chosen": -0.05222674950440223,
      "rewards/margins": 0.21744712057045748,
      "rewards/rejected": -0.2696738700748597,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 16.37181854248047,
      "kl": 14.163302421569824,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35703692.33222591,
      "logits/rejected": -36923225.864306785,
      "logps/chosen": -476.10356104651163,
      "logps/rejected": -371.15532817109147,
      "loss": 0.5842,
      "loss/base_kto": 3.721531629562378,
      "metrics/advantage_var": 264.5733642578125,
      "regularization/mmd": 0.9521066546440125,
      "regularization/rkhs_norm": 183.45022583007812,
      "rewards/chosen": 0.20643810893214026,
      "rewards/margins": 0.2370058295085138,
      "rewards/rejected": -0.030567720576373525,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 17.20627212524414,
      "kl": 5.4861884117126465,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -34640203.43206107,
      "logits/rejected": -34950722.3552,
      "logps/chosen": -456.92643129770994,
      "logps/rejected": -369.90875,
      "loss": 0.5838,
      "loss/base_kto": 3.8004844188690186,
      "metrics/advantage_var": 219.798828125,
      "regularization/mmd": 0.8696132898330688,
      "regularization/rkhs_norm": 167.55552673339844,
      "rewards/chosen": 0.05286537927525644,
      "rewards/margins": 0.19684445642369394,
      "rewards/rejected": -0.1439790771484375,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 14.005735397338867,
      "kl": 18.02949333190918,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35412104.53333333,
      "logits/rejected": -34605636.53543307,
      "logps/chosen": -474.1112403100775,
      "logps/rejected": -372.8711614173228,
      "loss": 0.5748,
      "loss/base_kto": 3.701120138168335,
      "metrics/advantage_var": 223.4566192626953,
      "regularization/mmd": 0.8973054885864258,
      "regularization/rkhs_norm": 172.89122009277344,
      "rewards/chosen": 0.3634170413941376,
      "rewards/margins": 0.25110125222398505,
      "rewards/rejected": 0.11231578917015256,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 12.172685623168945,
      "kl": 9.777069091796875,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -34705865.63549921,
      "logits/rejected": -35008127.01386749,
      "logps/chosen": -450.7083003169572,
      "logps/rejected": -334.51926040061636,
      "loss": 0.571,
      "loss/base_kto": 3.7229297161102295,
      "metrics/advantage_var": 203.33544921875,
      "regularization/mmd": 0.8448325991630554,
      "regularization/rkhs_norm": 162.7808380126953,
      "rewards/chosen": 0.11691013342227119,
      "rewards/margins": 0.26104623015913797,
      "rewards/rejected": -0.1441360967368668,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 17.458974838256836,
      "kl": 7.39281702041626,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -34642952.16586922,
      "logits/rejected": -35806457.33537519,
      "logps/chosen": -468.87250797448166,
      "logps/rejected": -387.4709753062787,
      "loss": 0.585,
      "loss/base_kto": 3.754244327545166,
      "metrics/advantage_var": 255.39453125,
      "regularization/mmd": 0.9255840182304382,
      "regularization/rkhs_norm": 178.33990478515625,
      "rewards/chosen": -0.005259002795059716,
      "rewards/margins": 0.23410685969058345,
      "rewards/rejected": -0.23936586248564318,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 18.057920455932617,
      "kl": 8.799531936645508,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -34666563.957767725,
      "logits/rejected": -35867158.405186385,
      "logps/chosen": -494.5712669683258,
      "logps/rejected": -373.538239465154,
      "loss": 0.5895,
      "loss/base_kto": 3.738917589187622,
      "metrics/advantage_var": 280.51007080078125,
      "regularization/mmd": 0.977146565914154,
      "regularization/rkhs_norm": 188.27488708496094,
      "rewards/chosen": 0.09878811613287142,
      "rewards/margins": 0.237137998533854,
      "rewards/rejected": -0.1383498824009826,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 15.648366928100586,
      "kl": 17.0057430267334,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -37764959.37254902,
      "logits/rejected": -37616206.83306321,
      "logps/chosen": -507.4282145550528,
      "logps/rejected": -382.2111780794165,
      "loss": 0.5864,
      "loss/base_kto": 3.7137954235076904,
      "metrics/advantage_var": 272.029296875,
      "regularization/mmd": 0.9773384928703308,
      "regularization/rkhs_norm": 188.3118438720703,
      "rewards/chosen": 0.3289657753876791,
      "rewards/margins": 0.25797809870227784,
      "rewards/rejected": 0.07098767668540126,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 13.922004699707031,
      "kl": 11.66692066192627,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -35887049.25786164,
      "logits/rejected": -37288495.701863356,
      "logps/chosen": -494.5318887578616,
      "logps/rejected": -356.20792895962734,
      "loss": 0.5759,
      "loss/base_kto": 3.684947967529297,
      "metrics/advantage_var": 247.2151641845703,
      "regularization/mmd": 0.9218670725822449,
      "regularization/rkhs_norm": 177.62371826171875,
      "rewards/chosen": 0.21588211539406446,
      "rewards/margins": 0.2977201545316263,
      "rewards/rejected": -0.08183803913756187,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 12.778936386108398,
      "kl": 10.353636741638184,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -35359593.025641024,
      "logits/rejected": -36776095.219512194,
      "logps/chosen": -466.09204727564105,
      "logps/rejected": -383.8243378429878,
      "loss": 0.5803,
      "loss/base_kto": 3.743032932281494,
      "metrics/advantage_var": 236.04234313964844,
      "regularization/mmd": 0.8997423052787781,
      "regularization/rkhs_norm": 173.3607635498047,
      "rewards/chosen": 0.08536517314421825,
      "rewards/margins": 0.2294842497567373,
      "rewards/rejected": -0.14411907661251905,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 12.796786308288574,
      "kl": 10.059364318847656,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -37100506.91023622,
      "logits/rejected": -38145832.086821705,
      "logps/chosen": -483.1859744094488,
      "logps/rejected": -385.253003875969,
      "loss": 0.5757,
      "loss/base_kto": 3.691363573074341,
      "metrics/advantage_var": 237.73448181152344,
      "regularization/mmd": 0.9140893816947937,
      "regularization/rkhs_norm": 176.12513732910156,
      "rewards/chosen": 0.1902135894054503,
      "rewards/margins": 0.2664576242834935,
      "rewards/rejected": -0.07624403487804324,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 18.834814071655273,
      "kl": 11.848525047302246,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -38043121.28290469,
      "logits/rejected": -38057669.686591275,
      "logps/chosen": -480.3556637670197,
      "logps/rejected": -382.6019789983845,
      "loss": 0.5912,
      "loss/base_kto": 3.799571990966797,
      "metrics/advantage_var": 250.9722137451172,
      "regularization/mmd": 0.9297453165054321,
      "regularization/rkhs_norm": 179.14169311523438,
      "rewards/chosen": 0.17881142389756288,
      "rewards/margins": 0.1670404575325683,
      "rewards/rejected": 0.011770966364994574,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 18.071857452392578,
      "kl": 10.782426834106445,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -36713761.42809917,
      "logits/rejected": -37365374.672592595,
      "logps/chosen": -463.689979338843,
      "logps/rejected": -371.9513425925926,
      "loss": 0.5846,
      "loss/base_kto": 3.789031982421875,
      "metrics/advantage_var": 230.16294860839844,
      "regularization/mmd": 0.8880990147590637,
      "regularization/rkhs_norm": 171.11734008789062,
      "rewards/chosen": 0.1387394802629455,
      "rewards/margins": 0.16395804263635405,
      "rewards/rejected": -0.025218562373408564,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 13.347097396850586,
      "kl": 8.719230651855469,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -36403756.82949309,
      "logits/rejected": -37686954.12400636,
      "logps/chosen": -487.795410906298,
      "logps/rejected": -384.7078696343402,
      "loss": 0.578,
      "loss/base_kto": 3.6972596645355225,
      "metrics/advantage_var": 266.856689453125,
      "regularization/mmd": 0.9265327453613281,
      "regularization/rkhs_norm": 178.522705078125,
      "rewards/chosen": 0.11646345756753432,
      "rewards/margins": 0.28006160762979215,
      "rewards/rejected": -0.1635981500622578,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 21.081863403320312,
      "kl": 10.20801067352295,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36710791.90123457,
      "logits/rejected": -37301760.0,
      "logps/chosen": -505.49508101851853,
      "logps/rejected": -377.8860512262658,
      "loss": 0.5778,
      "loss/base_kto": 3.713108777999878,
      "metrics/advantage_var": 232.65188598632812,
      "regularization/mmd": 0.9090795516967773,
      "regularization/rkhs_norm": 175.1598358154297,
      "rewards/chosen": 0.13450986367684822,
      "rewards/margins": 0.23079654078387452,
      "rewards/rejected": -0.0962866771070263,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 15.837052345275879,
      "kl": 14.241604804992676,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -35378290.14012739,
      "logits/rejected": -37924980.220858894,
      "logps/chosen": -472.6911325636943,
      "logps/rejected": -360.00575153374234,
      "loss": 0.5734,
      "loss/base_kto": 3.7171380519866943,
      "metrics/advantage_var": 216.7089385986328,
      "regularization/mmd": 0.8703489303588867,
      "regularization/rkhs_norm": 167.69729614257812,
      "rewards/chosen": 0.19872902305262863,
      "rewards/margins": 0.24355476709645862,
      "rewards/rejected": -0.04482574404382998,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 16.90869140625,
      "kl": 15.4988431930542,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -36117483.10204082,
      "logits/rejected": -36762427.47474747,
      "logps/chosen": -481.2785167638484,
      "logps/rejected": -401.38862584175087,
      "loss": 0.5882,
      "loss/base_kto": 3.7433693408966064,
      "metrics/advantage_var": 260.5050354003906,
      "regularization/mmd": 0.962075412273407,
      "regularization/rkhs_norm": 185.37098693847656,
      "rewards/chosen": 0.23253413609095983,
      "rewards/margins": 0.20535857956130785,
      "rewards/rejected": 0.02717555652965199,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 16.020484924316406,
      "kl": 14.999961853027344,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -36411713.37846154,
      "logits/rejected": -37450761.78343949,
      "logps/chosen": -479.30384615384617,
      "logps/rejected": -370.94436703821657,
      "loss": 0.5736,
      "loss/base_kto": 3.504671812057495,
      "metrics/advantage_var": 369.0525817871094,
      "regularization/mmd": 1.0838483572006226,
      "regularization/rkhs_norm": 208.833984375,
      "rewards/chosen": 0.3749387066180889,
      "rewards/margins": 0.5237920815123465,
      "rewards/rejected": -0.14885337489425757,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 13.673979759216309,
      "kl": 9.484530448913574,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -35978683.51592357,
      "logits/rejected": -36763145.42331288,
      "logps/chosen": -466.19342157643314,
      "logps/rejected": -360.1636790644172,
      "loss": 0.5727,
      "loss/base_kto": 3.2092158794403076,
      "metrics/advantage_var": 611.8053588867188,
      "regularization/mmd": 1.3727378845214844,
      "regularization/rkhs_norm": 264.4967041015625,
      "rewards/chosen": 0.4779043015401075,
      "rewards/margins": 0.9047891457993579,
      "rewards/rejected": -0.4268848442592504,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 11.30276870727539,
      "kl": 14.168227195739746,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -36663465.04126984,
      "logits/rejected": -38344281.79692308,
      "logps/chosen": -479.315376984127,
      "logps/rejected": -402.0303125,
      "loss": 0.5832,
      "loss/base_kto": 3.178797960281372,
      "metrics/advantage_var": 1516.0076904296875,
      "regularization/mmd": 1.486470103263855,
      "regularization/rkhs_norm": 286.41046142578125,
      "rewards/chosen": 0.5310749356708829,
      "rewards/margins": 0.9633089162898734,
      "rewards/rejected": -0.4322339806189904,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 10.916857719421387,
      "kl": 9.80307388305664,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36472250.03726708,
      "logits/rejected": -37288348.17610063,
      "logps/chosen": -484.9952445652174,
      "logps/rejected": -401.23388364779873,
      "loss": 0.5754,
      "loss/base_kto": 3.1596202850341797,
      "metrics/advantage_var": 647.99560546875,
      "regularization/mmd": 1.4439016580581665,
      "regularization/rkhs_norm": 278.20843505859375,
      "rewards/chosen": 0.4981846779769992,
      "rewards/margins": 0.9537182634354309,
      "rewards/rejected": -0.4555335854584316,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 16.762218475341797,
      "kl": 9.19613265991211,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -37068884.3866878,
      "logits/rejected": -37087137.33127889,
      "logps/chosen": -506.1333696513471,
      "logps/rejected": -366.7082530816641,
      "loss": 0.5686,
      "loss/base_kto": 3.1624512672424316,
      "metrics/advantage_var": 606.7392578125,
      "regularization/mmd": 1.3861886262893677,
      "regularization/rkhs_norm": 267.0883483886719,
      "rewards/chosen": 0.4608260697305864,
      "rewards/margins": 0.9256741928195935,
      "rewards/rejected": -0.46484812308900714,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 14.93901252746582,
      "kl": 14.44323444366455,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -34925424.41910632,
      "logits/rejected": -36761541.57844691,
      "logps/chosen": -485.94655238828966,
      "logps/rejected": -382.1991135103011,
      "loss": 0.5686,
      "loss/base_kto": 3.1689491271972656,
      "metrics/advantage_var": 631.2672729492188,
      "regularization/mmd": 1.3795952796936035,
      "regularization/rkhs_norm": 265.8179931640625,
      "rewards/chosen": 0.5743349425046947,
      "rewards/margins": 0.9278340512559948,
      "rewards/rejected": -0.35349910875130003,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 14.874568939208984,
      "kl": 12.614941596984863,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -36720434.8844376,
      "logits/rejected": -37408209.7496038,
      "logps/chosen": -480.2295839753467,
      "logps/rejected": -381.60831022187006,
      "loss": 0.5672,
      "loss/base_kto": 3.21881365776062,
      "metrics/advantage_var": 558.1520385742188,
      "regularization/mmd": 1.319035291671753,
      "regularization/rkhs_norm": 254.1493682861328,
      "rewards/chosen": 0.5033026226496292,
      "rewards/margins": 0.84659389170111,
      "rewards/rejected": -0.3432912690514808,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 49.197689056396484,
      "kl": 17.997711181640625,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -35001093.972006224,
      "logits/rejected": -37126870.605965465,
      "logps/chosen": -467.9345839813375,
      "logps/rejected": -369.26157770800626,
      "loss": 0.5643,
      "loss/base_kto": 3.2193355560302734,
      "metrics/advantage_var": 537.2255859375,
      "regularization/mmd": 1.295191764831543,
      "regularization/rkhs_norm": 249.55528259277344,
      "rewards/chosen": 0.5455483633845256,
      "rewards/margins": 0.8122700080031604,
      "rewards/rejected": -0.26672164461863473,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 15.033616065979004,
      "kl": 16.166486740112305,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -36191353.49152543,
      "logits/rejected": -37762445.5911252,
      "logps/chosen": -514.6885111710324,
      "logps/rejected": -379.8275059429477,
      "loss": 0.5708,
      "loss/base_kto": 3.178814172744751,
      "metrics/advantage_var": 581.7435913085938,
      "regularization/mmd": 1.3872652053833008,
      "regularization/rkhs_norm": 267.29583740234375,
      "rewards/chosen": 0.579269150556144,
      "rewards/margins": 0.9180882160430943,
      "rewards/rejected": -0.3388190654869503,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 13.95463752746582,
      "kl": 16.31585121154785,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -35213970.62174579,
      "logits/rejected": -36111237.51196172,
      "logps/chosen": -506.10681470137825,
      "logps/rejected": -374.78401614832535,
      "loss": 0.5613,
      "loss/base_kto": 3.1615407466888428,
      "metrics/advantage_var": 577.4911499023438,
      "regularization/mmd": 1.3287709951400757,
      "regularization/rkhs_norm": 256.0252380371094,
      "rewards/chosen": 0.5943864293704537,
      "rewards/margins": 0.9013198917050456,
      "rewards/rejected": -0.3069334623345918,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 14.795431137084961,
      "kl": 15.70853328704834,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35944115.2,
      "logits/rejected": -37309516.8,
      "logps/chosen": -460.71923828125,
      "logps/rejected": -373.9236328125,
      "loss": 0.5632,
      "loss/base_kto": 3.1640241146087646,
      "metrics/advantage_var": 581.5676879882812,
      "regularization/mmd": 1.34121572971344,
      "regularization/rkhs_norm": 258.4230651855469,
      "rewards/chosen": 0.5741420269012452,
      "rewards/margins": 0.8985859870910645,
      "rewards/rejected": -0.32444396018981936,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 11.119890213012695,
      "kl": 15.3007230758667,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -36073831.064935066,
      "logits/rejected": -37801724.915662654,
      "logps/chosen": -469.5836546266234,
      "logps/rejected": -368.4045557228916,
      "loss": 0.5594,
      "loss/base_kto": 3.218740940093994,
      "metrics/advantage_var": 471.51123046875,
      "regularization/mmd": 1.2567896842956543,
      "regularization/rkhs_norm": 242.156005859375,
      "rewards/chosen": 0.47081776408406045,
      "rewards/margins": 0.7942033474330815,
      "rewards/rejected": -0.32338558334902107,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 14.737159729003906,
      "kl": 14.101458549499512,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -34087149.06752411,
      "logits/rejected": -36154109.6656535,
      "logps/chosen": -475.9912580385852,
      "logps/rejected": -368.0076462765957,
      "loss": 0.5604,
      "loss/base_kto": 3.1185109615325928,
      "metrics/advantage_var": 601.0817260742188,
      "regularization/mmd": 1.3650661706924438,
      "regularization/rkhs_norm": 263.0185546875,
      "rewards/chosen": 0.5136829633804763,
      "rewards/margins": 0.9522213031054952,
      "rewards/rejected": -0.438538339725019,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 13.771965980529785,
      "kl": 16.108701705932617,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -36075593.25660964,
      "logits/rejected": -36407508.19466248,
      "logps/chosen": -491.42029548989115,
      "logps/rejected": -398.88383045525904,
      "loss": 0.5698,
      "loss/base_kto": 3.122931718826294,
      "metrics/advantage_var": 642.3717651367188,
      "regularization/mmd": 1.435685396194458,
      "regularization/rkhs_norm": 276.6253356933594,
      "rewards/chosen": 0.5806941719114502,
      "rewards/margins": 0.9448497720834601,
      "rewards/rejected": -0.36415560017200993,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 11.923416137695312,
      "kl": 12.88659954071045,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -34624768.79256966,
      "logits/rejected": -36532085.0977918,
      "logps/chosen": -486.4784249226006,
      "logps/rejected": -378.5344785094637,
      "loss": 0.5644,
      "loss/base_kto": 3.1559431552886963,
      "metrics/advantage_var": 554.8573608398438,
      "regularization/mmd": 1.3591049909591675,
      "regularization/rkhs_norm": 261.86993408203125,
      "rewards/chosen": 0.5736723625253967,
      "rewards/margins": 0.9181868904329036,
      "rewards/rejected": -0.3445145279075069,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 11.900761604309082,
      "kl": 14.61228084564209,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35385788.10608424,
      "logits/rejected": -35233836.87010954,
      "logps/chosen": -473.9101989079563,
      "logps/rejected": -377.85785895931144,
      "loss": 0.5651,
      "loss/base_kto": 3.23760986328125,
      "metrics/advantage_var": 529.1963500976562,
      "regularization/mmd": 1.2834538221359253,
      "regularization/rkhs_norm": 247.2936553955078,
      "rewards/chosen": 0.49898316000999415,
      "rewards/margins": 0.8162707185044068,
      "rewards/rejected": -0.31728755849441265,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 11.39002513885498,
      "kl": 11.355319023132324,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -33878128.90658499,
      "logits/rejected": -36059887.2599681,
      "logps/chosen": -459.093989280245,
      "logps/rejected": -377.0118620414673,
      "loss": 0.5616,
      "loss/base_kto": 3.210296630859375,
      "metrics/advantage_var": 507.0902404785156,
      "regularization/mmd": 1.2828924655914307,
      "regularization/rkhs_norm": 247.18545532226562,
      "rewards/chosen": 0.4646820021624952,
      "rewards/margins": 0.8663505203262463,
      "rewards/rejected": -0.401668518163751,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 12.310845375061035,
      "kl": 19.80482292175293,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35533879.26349206,
      "logits/rejected": -36472617.74769231,
      "logps/chosen": -470.63665674603175,
      "logps/rejected": -366.924375,
      "loss": 0.5716,
      "loss/base_kto": 3.2418675422668457,
      "metrics/advantage_var": 535.7623901367188,
      "regularization/mmd": 1.3306423425674438,
      "regularization/rkhs_norm": 256.38580322265625,
      "rewards/chosen": 0.5851631285652282,
      "rewards/margins": 0.7435311658624038,
      "rewards/rejected": -0.15836803729717547,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 9.959711074829102,
      "kl": 12.635095596313477,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -33910771.45482389,
      "logits/rejected": -34576929.480063796,
      "logps/chosen": -470.28493491577336,
      "logps/rejected": -374.89493620414675,
      "loss": 0.5597,
      "loss/base_kto": 3.207439422607422,
      "metrics/advantage_var": 512.9755249023438,
      "regularization/mmd": 1.2700233459472656,
      "regularization/rkhs_norm": 244.7058563232422,
      "rewards/chosen": 0.4744691717314318,
      "rewards/margins": 0.8727347852447517,
      "rewards/rejected": -0.39826561351331985,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 13.300355911254883,
      "kl": 16.797433853149414,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -35690637.68553459,
      "logits/rejected": -36165212.22360248,
      "logps/chosen": -485.21658805031444,
      "logps/rejected": -377.45455648291926,
      "loss": 0.5691,
      "loss/base_kto": 3.1995484828948975,
      "metrics/advantage_var": 560.8448486328125,
      "regularization/mmd": 1.352962851524353,
      "regularization/rkhs_norm": 260.6864929199219,
      "rewards/chosen": 0.5934259666586822,
      "rewards/margins": 0.8719599969806445,
      "rewards/rejected": -0.27853403032196233,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 11.905561447143555,
      "kl": 6.885768890380859,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -35484328.464516126,
      "logits/rejected": -36705503.41818182,
      "logps/chosen": -499.04002016129033,
      "logps/rejected": -385.2001420454545,
      "loss": 0.5667,
      "loss/base_kto": 3.2374038696289062,
      "metrics/advantage_var": 552.7958374023438,
      "regularization/mmd": 1.296007752418518,
      "regularization/rkhs_norm": 249.7124786376953,
      "rewards/chosen": 0.3102018540905368,
      "rewards/margins": 0.8404505076179989,
      "rewards/rejected": -0.5302486535274621,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 10.156408309936523,
      "kl": 11.827449798583984,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34034260.936434105,
      "logits/rejected": -36051140.73700787,
      "logps/chosen": -480.66143410852715,
      "logps/rejected": -356.6687992125984,
      "loss": 0.5631,
      "loss/base_kto": 3.159776449203491,
      "metrics/advantage_var": 559.4918212890625,
      "regularization/mmd": 1.3446849584579468,
      "regularization/rkhs_norm": 259.0915222167969,
      "rewards/chosen": 0.5264409028282461,
      "rewards/margins": 0.9246205893670625,
      "rewards/rejected": -0.3981796865388164,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 11.013405799865723,
      "kl": 13.303488731384277,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34680534.913580246,
      "logits/rejected": -36481876.25316456,
      "logps/chosen": -499.7567515432099,
      "logps/rejected": -371.2193433544304,
      "loss": 0.5589,
      "loss/base_kto": 3.1792984008789062,
      "metrics/advantage_var": 509.5606384277344,
      "regularization/mmd": 1.2922959327697754,
      "regularization/rkhs_norm": 248.99729919433594,
      "rewards/chosen": 0.5382480385862751,
      "rewards/margins": 0.8853367678652259,
      "rewards/rejected": -0.34708872927895074,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 12.741703987121582,
      "kl": 11.428122520446777,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -33385719.716486905,
      "logits/rejected": -34775533.33755943,
      "logps/chosen": -460.9606606317411,
      "logps/rejected": -354.9784815768621,
      "loss": 0.5625,
      "loss/base_kto": 3.305621385574341,
      "metrics/advantage_var": 438.2381896972656,
      "regularization/mmd": 1.1939852237701416,
      "regularization/rkhs_norm": 230.05496215820312,
      "rewards/chosen": 0.483033825326223,
      "rewards/margins": 0.7381768110616009,
      "rewards/rejected": -0.25514298573537786,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 11.366232872009277,
      "kl": 5.900028228759766,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -35032320.0,
      "logits/rejected": -36894805.333333336,
      "logps/chosen": -504.27163856907896,
      "logps/rejected": -380.8592587425595,
      "loss": 0.5669,
      "loss/base_kto": 3.2237019538879395,
      "metrics/advantage_var": 545.4990234375,
      "regularization/mmd": 1.3115177154541016,
      "regularization/rkhs_norm": 252.7008819580078,
      "rewards/chosen": 0.33717579590646846,
      "rewards/margins": 0.8409306387554734,
      "rewards/rejected": -0.5037548428490048,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 12.256927490234375,
      "kl": 13.522027969360352,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -33599860.36363637,
      "logits/rejected": -34542833.80031696,
      "logps/chosen": -489.6143586286595,
      "logps/rejected": -366.5445721077655,
      "loss": 0.5548,
      "loss/base_kto": 3.1496124267578125,
      "metrics/advantage_var": 518.9310913085938,
      "regularization/mmd": 1.2890504598617554,
      "regularization/rkhs_norm": 248.3719482421875,
      "rewards/chosen": 0.5277198356546129,
      "rewards/margins": 0.8914266148323193,
      "rewards/rejected": -0.3637067791777065,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 12.442972183227539,
      "kl": 14.00787353515625,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33877180.34003091,
      "logits/rejected": -34560682.666666664,
      "logps/chosen": -470.5526468315301,
      "logps/rejected": -361.9406101895735,
      "loss": 0.5586,
      "loss/base_kto": 3.203158140182495,
      "metrics/advantage_var": 506.0343933105469,
      "regularization/mmd": 1.2659547328948975,
      "regularization/rkhs_norm": 243.921875,
      "rewards/chosen": 0.5248600813852395,
      "rewards/margins": 0.8424741621668598,
      "rewards/rejected": -0.31761408078162023,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 13.786334037780762,
      "kl": 15.736841201782227,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -35004769.82723279,
      "logits/rejected": -34845584.50921273,
      "logps/chosen": -471.00155563689606,
      "logps/rejected": -361.7496335845896,
      "loss": 0.5575,
      "loss/base_kto": 3.2126832008361816,
      "metrics/advantage_var": 481.12945556640625,
      "regularization/mmd": 1.2475121021270752,
      "regularization/rkhs_norm": 240.36843872070312,
      "rewards/chosen": 0.6033123234180545,
      "rewards/margins": 0.8328046898318291,
      "rewards/rejected": -0.2294923664137746,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 11.664563179016113,
      "kl": 20.466232299804688,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -35302931.482496195,
      "logits/rejected": -37887283.36436597,
      "logps/chosen": -465.6245719178082,
      "logps/rejected": -364.2195274879615,
      "loss": 0.5596,
      "loss/base_kto": 3.2070748805999756,
      "metrics/advantage_var": 491.7352600097656,
      "regularization/mmd": 1.269645094871521,
      "regularization/rkhs_norm": 244.6329803466797,
      "rewards/chosen": 0.6266446048266268,
      "rewards/margins": 0.8284691175838561,
      "rewards/rejected": -0.20182451275722937,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 12.147101402282715,
      "kl": 20.791614532470703,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -35249375.14029851,
      "logits/rejected": -36237849.18032787,
      "logps/chosen": -464.20289179104475,
      "logps/rejected": -395.3352971311475,
      "loss": 0.564,
      "loss/base_kto": 3.1634483337402344,
      "metrics/advantage_var": 613.6397094726562,
      "regularization/mmd": 1.3485749959945679,
      "regularization/rkhs_norm": 259.841064453125,
      "rewards/chosen": 0.7005729561421409,
      "rewards/margins": 0.8940029593239736,
      "rewards/rejected": -0.19343000318183273,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 12.986946105957031,
      "kl": 17.93374252319336,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34984960.0,
      "logits/rejected": -35616488.44171779,
      "logps/chosen": -454.2381568471338,
      "logps/rejected": -392.0549750766871,
      "loss": 0.5702,
      "loss/base_kto": 3.215028762817383,
      "metrics/advantage_var": 649.096923828125,
      "regularization/mmd": 1.3462086915969849,
      "regularization/rkhs_norm": 259.3851013183594,
      "rewards/chosen": 0.6024145624440187,
      "rewards/margins": 0.8665838224807446,
      "rewards/rejected": -0.2641692600367259,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 17.55329132080078,
      "kl": 14.653973579406738,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -33923944.81582953,
      "logits/rejected": -34592531.3130016,
      "logps/chosen": -499.34103881278537,
      "logps/rejected": -377.31270064205455,
      "loss": 0.5655,
      "loss/base_kto": 3.2681057453155518,
      "metrics/advantage_var": 524.8186645507812,
      "regularization/mmd": 1.2558765411376953,
      "regularization/rkhs_norm": 241.98008728027344,
      "rewards/chosen": 0.5251274399198536,
      "rewards/margins": 0.776356764383136,
      "rewards/rejected": -0.2512293244632825,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 14.489201545715332,
      "kl": 12.35971450805664,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -32918209.752012882,
      "logits/rejected": -34491322.07587253,
      "logps/chosen": -479.5342190016103,
      "logps/rejected": -367.57833839150226,
      "loss": 0.5548,
      "loss/base_kto": 3.199578285217285,
      "metrics/advantage_var": 461.0147399902344,
      "regularization/mmd": 1.238900065422058,
      "regularization/rkhs_norm": 238.7090606689453,
      "rewards/chosen": 0.48035454250956117,
      "rewards/margins": 0.8472090582219924,
      "rewards/rejected": -0.3668545157124312,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 18.35354232788086,
      "kl": 11.523287773132324,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34819714.97674418,
      "logits/rejected": -36969760.65511811,
      "logps/chosen": -503.99040697674417,
      "logps/rejected": -388.50851377952756,
      "loss": 0.5693,
      "loss/base_kto": 3.199770450592041,
      "metrics/advantage_var": 548.7518920898438,
      "regularization/mmd": 1.3547416925430298,
      "regularization/rkhs_norm": 261.02923583984375,
      "rewards/chosen": 0.4939948503361192,
      "rewards/margins": 0.8702494351817147,
      "rewards/rejected": -0.3762545848455955,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 10.568586349487305,
      "kl": 12.54379940032959,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -32372905.28363047,
      "logits/rejected": -34948523.82503771,
      "logps/chosen": -468.6657718800648,
      "logps/rejected": -366.2825697586727,
      "loss": 0.5583,
      "loss/base_kto": 3.1970772743225098,
      "metrics/advantage_var": 500.9302673339844,
      "regularization/mmd": 1.2695538997650146,
      "regularization/rkhs_norm": 244.61538696289062,
      "rewards/chosen": 0.524188816064374,
      "rewards/margins": 0.8467278104406701,
      "rewards/rejected": -0.3225389943762962,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 13.829594612121582,
      "kl": 7.175731658935547,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -33422147.32598425,
      "logits/rejected": -35442037.08527132,
      "logps/chosen": -455.57632874015746,
      "logps/rejected": -364.9403343023256,
      "loss": 0.5672,
      "loss/base_kto": 3.2825868129730225,
      "metrics/advantage_var": 501.10504150390625,
      "regularization/mmd": 1.255201816558838,
      "regularization/rkhs_norm": 241.8500518798828,
      "rewards/chosen": 0.37917466050996557,
      "rewards/margins": 0.7758713810768261,
      "rewards/rejected": -0.39669672056686045,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 12.887574195861816,
      "kl": 8.26279067993164,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35577927.066246055,
      "logits/rejected": -36394488.0743034,
      "logps/chosen": -510.4785094637224,
      "logps/rejected": -375.7524671052632,
      "loss": 0.5703,
      "loss/base_kto": 3.154165029525757,
      "metrics/advantage_var": 658.8799438476562,
      "regularization/mmd": 1.4083423614501953,
      "regularization/rkhs_norm": 271.3569030761719,
      "rewards/chosen": 0.40889289928159506,
      "rewards/margins": 0.9694168152367517,
      "rewards/rejected": -0.5605239159551567,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 13.918904304504395,
      "kl": 17.184967041015625,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -35319389.23961661,
      "logits/rejected": -36476683.74311927,
      "logps/chosen": -493.04537739616615,
      "logps/rejected": -365.0766915137615,
      "loss": 0.562,
      "loss/base_kto": 3.172517776489258,
      "metrics/advantage_var": 538.2606811523438,
      "regularization/mmd": 1.3230832815170288,
      "regularization/rkhs_norm": 254.92935180664062,
      "rewards/chosen": 0.5746032422343001,
      "rewards/margins": 0.862835328135963,
      "rewards/rejected": -0.2882320859016628,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 10.085333824157715,
      "kl": 12.689724922180176,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -35791337.66720517,
      "logits/rejected": -35233637.083207265,
      "logps/chosen": -485.1252524232633,
      "logps/rejected": -404.0833018154312,
      "loss": 0.5569,
      "loss/base_kto": 3.140934705734253,
      "metrics/advantage_var": 536.6046142578125,
      "regularization/mmd": 1.3145030736923218,
      "regularization/rkhs_norm": 253.276123046875,
      "rewards/chosen": 0.4152452410326636,
      "rewards/margins": 0.882829386444445,
      "rewards/rejected": -0.4675841454117814,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 12.90168285369873,
      "kl": 9.561026573181152,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35614466.702865764,
      "logits/rejected": -35578731.811382115,
      "logps/chosen": -473.2989253393665,
      "logps/rejected": -404.78363821138214,
      "loss": 0.5397,
      "loss/base_kto": 3.083563804626465,
      "metrics/advantage_var": 452.0966796875,
      "regularization/mmd": 1.2337735891342163,
      "regularization/rkhs_norm": 237.7213134765625,
      "rewards/chosen": 0.5207481787035728,
      "rewards/margins": 0.9779926051517436,
      "rewards/rejected": -0.4572444264481707,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 11.978687286376953,
      "kl": 10.999704360961914,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -35162362.709062,
      "logits/rejected": -35614141.14900154,
      "logps/chosen": -474.64109697933225,
      "logps/rejected": -371.28619431643625,
      "loss": 0.5278,
      "loss/base_kto": 3.097466230392456,
      "metrics/advantage_var": 362.7237243652344,
      "regularization/mmd": 1.124959945678711,
      "regularization/rkhs_norm": 216.75527954101562,
      "rewards/chosen": 0.5506757727109003,
      "rewards/margins": 0.9320460241569928,
      "rewards/rejected": -0.38137025144609255,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 8.585254669189453,
      "kl": 8.297131538391113,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33853078.78317152,
      "logits/rejected": -35635599.081571,
      "logps/chosen": -481.5484425566343,
      "logps/rejected": -367.30579682779455,
      "loss": 0.5317,
      "loss/base_kto": 3.094708204269409,
      "metrics/advantage_var": 429.740966796875,
      "regularization/mmd": 1.1589012145996094,
      "regularization/rkhs_norm": 223.29502868652344,
      "rewards/chosen": 0.49697243897274473,
      "rewards/margins": 0.9603473346044715,
      "rewards/rejected": -0.4633748956317268,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 14.268289566040039,
      "kl": 9.894978523254395,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -33966447.58974359,
      "logits/rejected": -35747574.63414634,
      "logps/chosen": -469.7408854166667,
      "logps/rejected": -369.0608565167683,
      "loss": 0.5221,
      "loss/base_kto": 3.08331561088562,
      "metrics/advantage_var": 355.4896545410156,
      "regularization/mmd": 1.0933945178985596,
      "regularization/rkhs_norm": 210.6732940673828,
      "rewards/chosen": 0.4529416010929988,
      "rewards/margins": 0.9207873317582522,
      "rewards/rejected": -0.4678457306652534,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 12.006796836853027,
      "kl": 6.364843368530273,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34064046.9068323,
      "logits/rejected": -34977965.88679245,
      "logps/chosen": -511.1766304347826,
      "logps/rejected": -377.6381682389937,
      "loss": 0.5302,
      "loss/base_kto": 3.134618043899536,
      "metrics/advantage_var": 341.3658752441406,
      "regularization/mmd": 1.106918215751648,
      "regularization/rkhs_norm": 213.27903747558594,
      "rewards/chosen": 0.4341693191054445,
      "rewards/margins": 0.9212265071275062,
      "rewards/rejected": -0.4870571880220617,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 14.187989234924316,
      "kl": 13.8912992477417,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -33409737.556561086,
      "logits/rejected": -34950779.64343598,
      "logps/chosen": -468.34931184012066,
      "logps/rejected": -370.4262560777958,
      "loss": 0.5263,
      "loss/base_kto": 3.0540947914123535,
      "metrics/advantage_var": 389.7168884277344,
      "regularization/mmd": 1.1566543579101562,
      "regularization/rkhs_norm": 222.8621063232422,
      "rewards/chosen": 0.618285902664852,
      "rewards/margins": 0.9646606616897455,
      "rewards/rejected": -0.3463747590248936,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 14.874004364013672,
      "kl": 15.622693061828613,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -33923773.896713614,
      "logits/rejected": -35869502.70202808,
      "logps/chosen": -493.13478090766824,
      "logps/rejected": -407.02525351014043,
      "loss": 0.529,
      "loss/base_kto": 2.9775712490081787,
      "metrics/advantage_var": 467.7118225097656,
      "regularization/mmd": 1.2547889947891235,
      "regularization/rkhs_norm": 241.7705078125,
      "rewards/chosen": 0.6382726152924492,
      "rewards/margins": 1.075445539221125,
      "rewards/rejected": -0.4371729239286759,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 14.977208137512207,
      "kl": 9.718948364257812,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34752083.34883721,
      "logits/rejected": -35523466.28031496,
      "logps/chosen": -456.39171511627904,
      "logps/rejected": -402.3578494094488,
      "loss": 0.5288,
      "loss/base_kto": 3.1442782878875732,
      "metrics/advantage_var": 351.5884704589844,
      "regularization/mmd": 1.0862476825714111,
      "regularization/rkhs_norm": 209.2962646484375,
      "rewards/chosen": 0.49977743459302326,
      "rewards/margins": 0.8968435659445242,
      "rewards/rejected": -0.397066131351501,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 10.490389823913574,
      "kl": 14.246256828308105,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34071342.6918239,
      "logits/rejected": -35946155.72670808,
      "logps/chosen": -485.077731918239,
      "logps/rejected": -368.78188082298135,
      "loss": 0.5234,
      "loss/base_kto": 3.106316089630127,
      "metrics/advantage_var": 342.7874450683594,
      "regularization/mmd": 1.0810221433639526,
      "regularization/rkhs_norm": 208.28945922851562,
      "rewards/chosen": 0.5692803724756781,
      "rewards/margins": 0.8992850941705296,
      "rewards/rejected": -0.3300047216948515,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 11.920390129089355,
      "kl": 9.458094596862793,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -34456400.22677165,
      "logits/rejected": -34348697.20310078,
      "logps/chosen": -477.24379921259845,
      "logps/rejected": -378.56332364341085,
      "loss": 0.5301,
      "loss/base_kto": 3.113023519515991,
      "metrics/advantage_var": 366.575439453125,
      "regularization/mmd": 1.1278514862060547,
      "regularization/rkhs_norm": 217.31240844726562,
      "rewards/chosen": 0.5088973878875492,
      "rewards/margins": 0.9098979386233776,
      "rewards/rejected": -0.40100055073582846,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 10.451973915100098,
      "kl": 12.80419921875,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -35743516.44444445,
      "logits/rejected": -35723599.47581904,
      "logps/chosen": -478.9630281690141,
      "logps/rejected": -372.9942228939158,
      "loss": 0.5261,
      "loss/base_kto": 3.093827486038208,
      "metrics/advantage_var": 354.83294677734375,
      "regularization/mmd": 1.1146318912506104,
      "regularization/rkhs_norm": 214.7652587890625,
      "rewards/chosen": 0.5280838132091158,
      "rewards/margins": 0.9237132675376796,
      "rewards/rejected": -0.3956294543285638,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 9.721475601196289,
      "kl": 14.141409873962402,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34409271.550834596,
      "logits/rejected": -35842974.71175523,
      "logps/chosen": -467.9213770864947,
      "logps/rejected": -361.8383152173913,
      "loss": 0.5254,
      "loss/base_kto": 3.0692737102508545,
      "metrics/advantage_var": 386.54248046875,
      "regularization/mmd": 1.1342560052871704,
      "regularization/rkhs_norm": 218.5464324951172,
      "rewards/chosen": 0.6341614325238287,
      "rewards/margins": 0.9522332248069407,
      "rewards/rejected": -0.31807179228311194,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 12.84123706817627,
      "kl": 11.918646812438965,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -35401582.8921095,
      "logits/rejected": -37565171.18057663,
      "logps/chosen": -495.46643518518516,
      "logps/rejected": -387.23937784522,
      "loss": 0.5271,
      "loss/base_kto": 3.0971181392669678,
      "metrics/advantage_var": 363.2221374511719,
      "regularization/mmd": 1.1196831464767456,
      "regularization/rkhs_norm": 215.73855590820312,
      "rewards/chosen": 0.5170429616734602,
      "rewards/margins": 0.9170634857866999,
      "rewards/rejected": -0.40002052411323974,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 12.777566909790039,
      "kl": 9.683692932128906,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -34964963.018867925,
      "logits/rejected": -36498492.42236025,
      "logps/chosen": -475.05139544025155,
      "logps/rejected": -372.8525815217391,
      "loss": 0.53,
      "loss/base_kto": 3.1250438690185547,
      "metrics/advantage_var": 349.35504150390625,
      "regularization/mmd": 1.115165114402771,
      "regularization/rkhs_norm": 214.8680419921875,
      "rewards/chosen": 0.47042045353343653,
      "rewards/margins": 0.8982806898494303,
      "rewards/rejected": -0.4278602363159938,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 13.83116626739502,
      "kl": 12.920928001403809,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -36302362.52433281,
      "logits/rejected": -36534147.7822706,
      "logps/chosen": -489.30170722135006,
      "logps/rejected": -389.0027702177294,
      "loss": 0.5268,
      "loss/base_kto": 3.1076176166534424,
      "metrics/advantage_var": 348.1242370605469,
      "regularization/mmd": 1.1069954633712769,
      "regularization/rkhs_norm": 213.2938995361328,
      "rewards/chosen": 0.5731726246688579,
      "rewards/margins": 0.9055391883383028,
      "rewards/rejected": -0.332366563669445,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 10.138191223144531,
      "kl": 10.839698791503906,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -34733122.37037037,
      "logits/rejected": -36802955.341772154,
      "logps/chosen": -471.93161651234567,
      "logps/rejected": -346.95475672468353,
      "loss": 0.5226,
      "loss/base_kto": 3.072765588760376,
      "metrics/advantage_var": 362.6602478027344,
      "regularization/mmd": 1.1081881523132324,
      "regularization/rkhs_norm": 213.5237274169922,
      "rewards/chosen": 0.547323203381197,
      "rewards/margins": 0.9454841861167463,
      "rewards/rejected": -0.39816098273554934,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 10.297775268554688,
      "kl": 12.261195182800293,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -34236499.27710843,
      "logits/rejected": -35840422.233766235,
      "logps/chosen": -483.4094503012048,
      "logps/rejected": -389.18633319805195,
      "loss": 0.5246,
      "loss/base_kto": 3.023761510848999,
      "metrics/advantage_var": 406.6839599609375,
      "regularization/mmd": 1.173167109489441,
      "regularization/rkhs_norm": 226.0437774658203,
      "rewards/chosen": 0.6181016944977175,
      "rewards/margins": 1.0205599101864475,
      "rewards/rejected": -0.4024582156887302,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 13.980240821838379,
      "kl": 14.290509223937988,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -33901226.666666664,
      "logits/rejected": -34959430.62068965,
      "logps/chosen": -476.6505062305296,
      "logps/rejected": -362.0511363636364,
      "loss": 0.5244,
      "loss/base_kto": 3.062887668609619,
      "metrics/advantage_var": 357.87786865234375,
      "regularization/mmd": 1.1319570541381836,
      "regularization/rkhs_norm": 218.1034698486328,
      "rewards/chosen": 0.5935802043784073,
      "rewards/margins": 0.9548575783589863,
      "rewards/rejected": -0.36127737398057896,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 12.550543785095215,
      "kl": 13.003260612487793,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34526914.89032258,
      "logits/rejected": -36702536.92121212,
      "logps/chosen": -493.12318548387094,
      "logps/rejected": -375.2939393939394,
      "loss": 0.527,
      "loss/base_kto": 3.0775673389434814,
      "metrics/advantage_var": 368.90826416015625,
      "regularization/mmd": 1.1387715339660645,
      "regularization/rkhs_norm": 219.41650390625,
      "rewards/chosen": 0.5661862281060988,
      "rewards/margins": 0.9272098709061698,
      "rewards/rejected": -0.36102364280007104,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 12.30187702178955,
      "kl": 7.758034706115723,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34263442.67895545,
      "logits/rejected": -35832615.47853736,
      "logps/chosen": -491.775153609831,
      "logps/rejected": -387.5150536565978,
      "loss": 0.5301,
      "loss/base_kto": 3.1168315410614014,
      "metrics/advantage_var": 368.3890075683594,
      "regularization/mmd": 1.1242722272872925,
      "regularization/rkhs_norm": 216.6227569580078,
      "rewards/chosen": 0.47406005859375,
      "rewards/margins": 0.9331607788280007,
      "rewards/rejected": -0.4591007202342508,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 8.568299293518066,
      "kl": 13.640970230102539,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -33591270.59844961,
      "logits/rejected": -37399105.31023622,
      "logps/chosen": -493.2200581395349,
      "logps/rejected": -351.5467765748032,
      "loss": 0.5273,
      "loss/base_kto": 3.0856311321258545,
      "metrics/advantage_var": 387.4796447753906,
      "regularization/mmd": 1.1325229406356812,
      "regularization/rkhs_norm": 218.21255493164062,
      "rewards/chosen": 0.5919730726138566,
      "rewards/margins": 0.9574057397829019,
      "rewards/rejected": -0.36543266716904527,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 7.520174503326416,
      "kl": 14.801548957824707,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -33570673.73182552,
      "logits/rejected": -35778814.06354009,
      "logps/chosen": -461.5297859450727,
      "logps/rejected": -372.69095121028744,
      "loss": 0.523,
      "loss/base_kto": 3.0906670093536377,
      "metrics/advantage_var": 337.0397644042969,
      "regularization/mmd": 1.0931990146636963,
      "regularization/rkhs_norm": 210.63565063476562,
      "rewards/chosen": 0.5426389235094407,
      "rewards/margins": 0.91538383145993,
      "rewards/rejected": -0.3727449079504893,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.153304100036621,
      "kl": 12.090093612670898,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -34274220.972972974,
      "logits/rejected": -35502780.456026055,
      "logps/chosen": -475.06465840840843,
      "logps/rejected": -389.267457247557,
      "loss": 0.5308,
      "loss/base_kto": 3.1084365844726562,
      "metrics/advantage_var": 365.2424011230469,
      "regularization/mmd": 1.1381527185440063,
      "regularization/rkhs_norm": 219.29727172851562,
      "rewards/chosen": 0.5549318239137575,
      "rewards/margins": 0.9265550091616199,
      "rewards/rejected": -0.37162318524786236,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 10.674633026123047,
      "kl": 13.322845458984375,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35482065.9116203,
      "logits/rejected": -35838371.396113604,
      "logps/chosen": -488.06106792144027,
      "logps/rejected": -369.5948477204783,
      "loss": 0.5287,
      "loss/base_kto": 3.0646259784698486,
      "metrics/advantage_var": 399.35321044921875,
      "regularization/mmd": 1.165199637413025,
      "regularization/rkhs_norm": 224.5085906982422,
      "rewards/chosen": 0.581476339536876,
      "rewards/margins": 0.955891533331252,
      "rewards/rejected": -0.37441519379437593,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 8.666868209838867,
      "kl": 11.906843185424805,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35078330.46433991,
      "logits/rejected": -35586740.56038647,
      "logps/chosen": -483.5044575113809,
      "logps/rejected": -387.401519726248,
      "loss": 0.5216,
      "loss/base_kto": 3.0373425483703613,
      "metrics/advantage_var": 371.4295959472656,
      "regularization/mmd": 1.1354891061782837,
      "regularization/rkhs_norm": 218.7840118408203,
      "rewards/chosen": 0.5717829934382113,
      "rewards/margins": 0.9905606312577564,
      "rewards/rejected": -0.4187776378195451,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 11.692424774169922,
      "kl": 13.390151977539062,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34022912.79379845,
      "logits/rejected": -35373110.82834645,
      "logps/chosen": -482.69447674418603,
      "logps/rejected": -376.36097440944883,
      "loss": 0.5248,
      "loss/base_kto": 3.034217596054077,
      "metrics/advantage_var": 392.9123229980469,
      "regularization/mmd": 1.1644567251205444,
      "regularization/rkhs_norm": 224.365478515625,
      "rewards/chosen": 0.5863914312318314,
      "rewards/margins": 0.9843042691839721,
      "rewards/rejected": -0.3979128379521408,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 9.955215454101562,
      "kl": 14.835789680480957,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -36104574.593406595,
      "logits/rejected": -37224244.155521,
      "logps/chosen": -485.3425235478807,
      "logps/rejected": -370.96359836702953,
      "loss": 0.5249,
      "loss/base_kto": 3.0935723781585693,
      "metrics/advantage_var": 345.5436706542969,
      "regularization/mmd": 1.1055536270141602,
      "regularization/rkhs_norm": 213.01611328125,
      "rewards/chosen": 0.6026077629819957,
      "rewards/margins": 0.9071907533287589,
      "rewards/rejected": -0.3045829903467632,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 8.96081829071045,
      "kl": 13.66186809539795,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -33680170.267912775,
      "logits/rejected": -36078116.9153605,
      "logps/chosen": -460.74591121495325,
      "logps/rejected": -376.73474235893417,
      "loss": 0.5207,
      "loss/base_kto": 3.078190803527832,
      "metrics/advantage_var": 344.8306884765625,
      "regularization/mmd": 1.0873230695724487,
      "regularization/rkhs_norm": 209.5034637451172,
      "rewards/chosen": 0.5968351275007301,
      "rewards/margins": 0.9276716956335671,
      "rewards/rejected": -0.330836568132837,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 12.003533363342285,
      "kl": 13.155600547790527,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34270780.32628399,
      "logits/rejected": -35165478.93851133,
      "logps/chosen": -477.3462990936556,
      "logps/rejected": -352.68648867313914,
      "loss": 0.5251,
      "loss/base_kto": 3.098294973373413,
      "metrics/advantage_var": 356.3811950683594,
      "regularization/mmd": 1.1022199392318726,
      "regularization/rkhs_norm": 212.373779296875,
      "rewards/chosen": 0.5785075300049566,
      "rewards/margins": 0.917265535684189,
      "rewards/rejected": -0.3387580056792324,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 10.422096252441406,
      "kl": 13.961255073547363,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -34598256.64,
      "logits/rejected": -35445864.74503817,
      "logps/chosen": -478.49345,
      "logps/rejected": -386.29017175572517,
      "loss": 0.5233,
      "loss/base_kto": 3.076866626739502,
      "metrics/advantage_var": 351.24884033203125,
      "regularization/mmd": 1.1095365285873413,
      "regularization/rkhs_norm": 213.78355407714844,
      "rewards/chosen": 0.551825390625,
      "rewards/margins": 0.9326069108569895,
      "rewards/rejected": -0.3807815202319895,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 10.507048606872559,
      "kl": 12.8363037109375,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -36340272.3328,
      "logits/rejected": -36478532.00610687,
      "logps/chosen": -504.8625,
      "logps/rejected": -385.6226860687023,
      "loss": 0.5249,
      "loss/base_kto": 3.0856473445892334,
      "metrics/advantage_var": 352.6623229980469,
      "regularization/mmd": 1.1136070489883423,
      "regularization/rkhs_norm": 214.5678253173828,
      "rewards/chosen": 0.567965234375,
      "rewards/margins": 0.9208772403163765,
      "rewards/rejected": -0.35291200594137645,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 11.083954811096191,
      "kl": 14.43707275390625,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35108926.86268175,
      "logits/rejected": -36068589.02269289,
      "logps/chosen": -479.93245153473345,
      "logps/rejected": -380.5613180786687,
      "loss": 0.5238,
      "loss/base_kto": 3.059920072555542,
      "metrics/advantage_var": 361.0622253417969,
      "regularization/mmd": 1.1302847862243652,
      "regularization/rkhs_norm": 217.78125,
      "rewards/chosen": 0.5592205682363186,
      "rewards/margins": 0.9417497072140157,
      "rewards/rejected": -0.38252913897769714,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 9.955196380615234,
      "kl": 12.403254508972168,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35138235.39356815,
      "logits/rejected": -35390361.11004785,
      "logps/chosen": -489.0685777182236,
      "logps/rejected": -396.19659589314193,
      "loss": 0.5287,
      "loss/base_kto": 3.0336174964904785,
      "metrics/advantage_var": 407.16680908203125,
      "regularization/mmd": 1.1960941553115845,
      "regularization/rkhs_norm": 230.46128845214844,
      "rewards/chosen": 0.5870492184508997,
      "rewards/margins": 0.9914722279065791,
      "rewards/rejected": -0.4044230094556793,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 11.344730377197266,
      "kl": 14.39224910736084,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -34632143.82137161,
      "logits/rejected": -34399562.87901991,
      "logps/chosen": -467.9655602073365,
      "logps/rejected": -368.33580589586524,
      "loss": 0.5236,
      "loss/base_kto": 3.100672483444214,
      "metrics/advantage_var": 348.1331481933594,
      "regularization/mmd": 1.08786141872406,
      "regularization/rkhs_norm": 209.6072235107422,
      "rewards/chosen": 0.5287956468986742,
      "rewards/margins": 0.8986455134999376,
      "rewards/rejected": -0.3698498666012634,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 12.729537010192871,
      "kl": 13.068961143493652,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35895873.163636364,
      "logits/rejected": -35758020.54193548,
      "logps/chosen": -483.4882102272727,
      "logps/rejected": -372.08689516129033,
      "loss": 0.5269,
      "loss/base_kto": 3.126361846923828,
      "metrics/advantage_var": 338.41412353515625,
      "regularization/mmd": 1.0887268781661987,
      "regularization/rkhs_norm": 209.7739715576172,
      "rewards/chosen": 0.5410943233605587,
      "rewards/margins": 0.8966187925515869,
      "rewards/rejected": -0.3555244691910282,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 14.17967414855957,
      "kl": 11.574604988098145,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -34513269.5345912,
      "logits/rejected": -34806147.97515528,
      "logps/chosen": -478.07881289308176,
      "logps/rejected": -377.5923913043478,
      "loss": 0.531,
      "loss/base_kto": 3.090505361557007,
      "metrics/advantage_var": 388.46661376953125,
      "regularization/mmd": 1.1578208208084106,
      "regularization/rkhs_norm": 223.0868682861328,
      "rewards/chosen": 0.5807230247641509,
      "rewards/margins": 0.9314504614695567,
      "rewards/rejected": -0.3507274367054057,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 8.735228538513184,
      "kl": 12.658534049987793,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34755086.44514106,
      "logits/rejected": -35124801.39563863,
      "logps/chosen": -485.3395865987461,
      "logps/rejected": -354.87670366043614,
      "loss": 0.5251,
      "loss/base_kto": 3.1163642406463623,
      "metrics/advantage_var": 325.663330078125,
      "regularization/mmd": 1.0844676494598389,
      "regularization/rkhs_norm": 208.9532928466797,
      "rewards/chosen": 0.5551200555783454,
      "rewards/margins": 0.8920120142250305,
      "rewards/rejected": -0.33689195864668514,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 10.3009672164917,
      "kl": 13.03227424621582,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34693307.06141733,
      "logits/rejected": -35954256.17364341,
      "logps/chosen": -485.29675196850394,
      "logps/rejected": -381.8583091085271,
      "loss": 0.5325,
      "loss/base_kto": 3.098832368850708,
      "metrics/advantage_var": 422.102294921875,
      "regularization/mmd": 1.160827398300171,
      "regularization/rkhs_norm": 223.66616821289062,
      "rewards/chosen": 0.5834943636195866,
      "rewards/margins": 0.93468535350573,
      "rewards/rejected": -0.3511909898861434,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.977114557203087e+17,
      "train_loss": 0.5577637589245469,
      "train_runtime": 11071.1673,
      "train_samples_per_second": 13.388,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.977114557203087e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}