{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 23.90775489807129,
      "kl": 12.681447982788086,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35304941.26829268,
      "logits/rejected": -36776152.615384616,
      "logps/chosen": -498.387480945122,
      "logps/rejected": -364.2709334935897,
      "loss": 0.589,
      "loss/base_kto": 3.8656768798828125,
      "metrics/advantage_var": 206.61790466308594,
      "regularization/mmd": 0.8460672497749329,
      "regularization/rkhs_norm": 163.01873779296875,
      "rewards/chosen": 0.2146739727113305,
      "rewards/margins": 0.12779606424919734,
      "rewards/rejected": 0.08687790846213317,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 16.661895751953125,
      "kl": 9.479524612426758,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36529332.33690658,
      "logits/rejected": -36652876.350877196,
      "logps/chosen": -483.9310394333844,
      "logps/rejected": -404.54291267942585,
      "loss": 0.5875,
      "loss/base_kto": 3.8932931423187256,
      "metrics/advantage_var": 187.1262969970703,
      "regularization/mmd": 0.8067044615745544,
      "regularization/rkhs_norm": 155.4343719482422,
      "rewards/chosen": 0.12601447580054198,
      "rewards/margins": 0.10988777010155218,
      "rewards/rejected": 0.016126705698989795,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 17.62555503845215,
      "kl": 10.208549499511719,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35283677.89280245,
      "logits/rejected": -36556519.09409888,
      "logps/chosen": -493.9445348392037,
      "logps/rejected": -388.88800837320576,
      "loss": 0.5882,
      "loss/base_kto": 3.854313611984253,
      "metrics/advantage_var": 210.4778289794922,
      "regularization/mmd": 0.8510956764221191,
      "regularization/rkhs_norm": 163.9875946044922,
      "rewards/chosen": 0.18612776050888927,
      "rewards/margins": 0.1310663958406713,
      "rewards/rejected": 0.055061364668217955,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 14.005806922912598,
      "kl": 2.2349817752838135,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37167114.413559325,
      "logits/rejected": -37620780.52173913,
      "logps/chosen": -490.4091101694915,
      "logps/rejected": -394.4764492753623,
      "loss": 0.5817,
      "loss/base_kto": 3.8326363563537598,
      "metrics/advantage_var": 186.0223846435547,
      "regularization/mmd": 0.8209611773490906,
      "regularization/rkhs_norm": 158.18136596679688,
      "rewards/chosen": -0.11016082763671875,
      "rewards/margins": 0.14244287463201993,
      "rewards/rejected": -0.2526037022687387,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 17.238176345825195,
      "kl": 10.287137031555176,
      "learning_rate": 9.9e-07,
      "logits/chosen": -34916587.32919255,
      "logits/rejected": -36685221.83647799,
      "logps/chosen": -481.85025232919253,
      "logps/rejected": -353.9864878144654,
      "loss": 0.5817,
      "loss/base_kto": 3.8582093715667725,
      "metrics/advantage_var": 175.45828247070312,
      "regularization/mmd": 0.7956187129020691,
      "regularization/rkhs_norm": 153.2984161376953,
      "rewards/chosen": 0.07971524896088594,
      "rewards/margins": 0.12481347873502516,
      "rewards/rejected": -0.045098229774139215,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 15.216425895690918,
      "kl": 7.181277751922607,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -36097753.55910543,
      "logits/rejected": -36191335.33944954,
      "logps/chosen": -470.92831469648564,
      "logps/rejected": -376.39475821865443,
      "loss": 0.5829,
      "loss/base_kto": 3.8616745471954346,
      "metrics/advantage_var": 181.6775360107422,
      "regularization/mmd": 0.8012104034423828,
      "regularization/rkhs_norm": 154.37579345703125,
      "rewards/chosen": 0.07693188335187139,
      "rewards/margins": 0.11716781689485209,
      "rewards/rejected": -0.0402359335429807,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 17.482717514038086,
      "kl": 10.225573539733887,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35949917.47242921,
      "logits/rejected": -37559647.422003284,
      "logps/chosen": -496.74236214605065,
      "logps/rejected": -368.60129310344826,
      "loss": 0.5811,
      "loss/base_kto": 3.8098678588867188,
      "metrics/advantage_var": 199.4989471435547,
      "regularization/mmd": 0.8390861749649048,
      "regularization/rkhs_norm": 161.67364501953125,
      "rewards/chosen": 0.16720601521021564,
      "rewards/margins": 0.18618105854617878,
      "rewards/rejected": -0.018975043335963157,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 13.397759437561035,
      "kl": 7.023360729217529,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -35450873.43589743,
      "logits/rejected": -35668174.048780486,
      "logps/chosen": -477.0907451923077,
      "logps/rejected": -349.8881002286585,
      "loss": 0.5793,
      "loss/base_kto": 3.7969701290130615,
      "metrics/advantage_var": 199.3280029296875,
      "regularization/mmd": 0.8371647000312805,
      "regularization/rkhs_norm": 161.3034210205078,
      "rewards/chosen": 0.10001216790614983,
      "rewards/margins": 0.19661575917380536,
      "rewards/rejected": -0.09660359126765554,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 14.562145233154297,
      "kl": 5.361706733703613,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -34593796.777604975,
      "logits/rejected": -35693000.540031396,
      "logps/chosen": -454.53003499222393,
      "logps/rejected": -360.58442896389323,
      "loss": 0.579,
      "loss/base_kto": 3.832873582839966,
      "metrics/advantage_var": 180.25230407714844,
      "regularization/mmd": 0.7988579273223877,
      "regularization/rkhs_norm": 153.9225311279297,
      "rewards/chosen": 0.04733409555295763,
      "rewards/margins": 0.15071037930338688,
      "rewards/rejected": -0.10337628375042926,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 16.442813873291016,
      "kl": 10.120930671691895,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -34595783.20126783,
      "logits/rejected": -37669536.14791988,
      "logps/chosen": -497.20973652931855,
      "logps/rejected": -373.6237962249615,
      "loss": 0.5775,
      "loss/base_kto": 3.753746509552002,
      "metrics/advantage_var": 217.498291015625,
      "regularization/mmd": 0.8664171099662781,
      "regularization/rkhs_norm": 166.93972778320312,
      "rewards/chosen": 0.15937588747632106,
      "rewards/margins": 0.22018825717707763,
      "rewards/rejected": -0.060812369700756576,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 17.87065887451172,
      "kl": 5.803158283233643,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -36220817.99684044,
      "logits/rejected": -36433166.63987635,
      "logps/chosen": -481.3564375987362,
      "logps/rejected": -376.2250772797527,
      "loss": 0.5888,
      "loss/base_kto": 3.793182373046875,
      "metrics/advantage_var": 248.9573974609375,
      "regularization/mmd": 0.9168861508369446,
      "regularization/rkhs_norm": 176.6640167236328,
      "rewards/chosen": 0.030898312633448114,
      "rewards/margins": 0.2135540126566924,
      "rewards/rejected": -0.1826557000232443,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 15.884252548217773,
      "kl": 12.579646110534668,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36456635.33643411,
      "logits/rejected": -36561599.09291339,
      "logps/chosen": -473.1209302325581,
      "logps/rejected": -384.91274606299214,
      "loss": 0.5879,
      "loss/base_kto": 3.786097288131714,
      "metrics/advantage_var": 239.2100067138672,
      "regularization/mmd": 0.9171223044395447,
      "regularization/rkhs_norm": 176.70948791503906,
      "rewards/chosen": 0.18350818249606346,
      "rewards/margins": 0.18441684338473532,
      "rewards/rejected": -0.000908660888671875,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 18.382980346679688,
      "kl": 11.036842346191406,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -35736208.33234421,
      "logits/rejected": -37681736.66006601,
      "logps/chosen": -479.64938798219583,
      "logps/rejected": -371.23999587458746,
      "loss": 0.5838,
      "loss/base_kto": 3.7316291332244873,
      "metrics/advantage_var": 280.1012878417969,
      "regularization/mmd": 0.9384465217590332,
      "regularization/rkhs_norm": 180.8182373046875,
      "rewards/chosen": 0.2972555188821402,
      "rewards/margins": 0.26465956967769216,
      "rewards/rejected": 0.032595949204448034,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 18.595134735107422,
      "kl": 12.837692260742188,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -37466318.96229261,
      "logits/rejected": -39280646.63857374,
      "logps/chosen": -491.5295060331825,
      "logps/rejected": -355.3635028363047,
      "loss": 0.5853,
      "loss/base_kto": 3.7730443477630615,
      "metrics/advantage_var": 241.9972686767578,
      "regularization/mmd": 0.9096607565879822,
      "regularization/rkhs_norm": 175.27183532714844,
      "rewards/chosen": 0.2359304068494886,
      "rewards/margins": 0.20733151634398694,
      "rewards/rejected": 0.02859889050550167,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 19.06772232055664,
      "kl": 11.678929328918457,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36305840.73301738,
      "logits/rejected": -37630303.357032456,
      "logps/chosen": -492.65501579778834,
      "logps/rejected": -383.62688369397216,
      "loss": 0.5858,
      "loss/base_kto": 3.7701878547668457,
      "metrics/advantage_var": 244.46861267089844,
      "regularization/mmd": 0.9164788126945496,
      "regularization/rkhs_norm": 176.5855255126953,
      "rewards/chosen": 0.1869154292825274,
      "rewards/margins": 0.2168836055096076,
      "rewards/rejected": -0.029968176227080215,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 23.510053634643555,
      "kl": 10.49164867401123,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -37824399.95136778,
      "logits/rejected": -38689067.627009645,
      "logps/chosen": -495.4928761398176,
      "logps/rejected": -359.66881028938906,
      "loss": 0.5875,
      "loss/base_kto": 3.80584979057312,
      "metrics/advantage_var": 232.99180603027344,
      "regularization/mmd": 0.8941518664360046,
      "regularization/rkhs_norm": 172.28359985351562,
      "rewards/chosen": 0.14421909413439163,
      "rewards/margins": 0.1627761336221752,
      "rewards/rejected": -0.018557039487783547,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.82607078552246,
      "kl": 7.366345405578613,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -35904137.444270015,
      "logits/rejected": -37476915.75738725,
      "logps/chosen": -476.3798567503925,
      "logps/rejected": -363.42952954898914,
      "loss": 0.5814,
      "loss/base_kto": 3.760831594467163,
      "metrics/advantage_var": 233.8565216064453,
      "regularization/mmd": 0.8902547955513,
      "regularization/rkhs_norm": 171.53271484375,
      "rewards/chosen": 0.09115862284949286,
      "rewards/margins": 0.22523825493374758,
      "rewards/rejected": -0.13407963208425472,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 16.150705337524414,
      "kl": 11.253143310546875,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -36481020.84437596,
      "logits/rejected": -37468559.21394612,
      "logps/chosen": -486.3712442218798,
      "logps/rejected": -386.98638074484944,
      "loss": 0.5781,
      "loss/base_kto": 3.739018201828003,
      "metrics/advantage_var": 224.9400634765625,
      "regularization/mmd": 0.885532557964325,
      "regularization/rkhs_norm": 170.6228485107422,
      "rewards/chosen": 0.19712246511309467,
      "rewards/margins": 0.22026693318783885,
      "rewards/rejected": -0.023144468074744174,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 16.826560974121094,
      "kl": 11.830695152282715,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -34938873.80332829,
      "logits/rejected": -38308968.21970921,
      "logps/chosen": -502.84880862329805,
      "logps/rejected": -372.40619951534734,
      "loss": 0.5859,
      "loss/base_kto": 3.7713570594787598,
      "metrics/advantage_var": 268.22540283203125,
      "regularization/mmd": 0.9160144925117493,
      "regularization/rkhs_norm": 176.49607849121094,
      "rewards/chosen": 0.21172615547584153,
      "rewards/margins": 0.1934679605447373,
      "rewards/rejected": 0.018258194931104227,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 16.222583770751953,
      "kl": 10.096343994140625,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -35715616.646967344,
      "logits/rejected": -37105972.64678179,
      "logps/chosen": -461.8372861586314,
      "logps/rejected": -363.0498920722135,
      "loss": 0.5796,
      "loss/base_kto": 3.748002290725708,
      "metrics/advantage_var": 225.37364196777344,
      "regularization/mmd": 0.8887432217597961,
      "regularization/rkhs_norm": 171.24147033691406,
      "rewards/chosen": 0.1531903042949006,
      "rewards/margins": 0.23799627046705157,
      "rewards/rejected": -0.08480596617215096,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 18.901710510253906,
      "kl": 7.529710292816162,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -35298238.42364532,
      "logits/rejected": -36855258.61102831,
      "logps/chosen": -473.2080767651888,
      "logps/rejected": -352.67206128912073,
      "loss": 0.5767,
      "loss/base_kto": 3.7175674438476562,
      "metrics/advantage_var": 235.353515625,
      "regularization/mmd": 0.8964071273803711,
      "regularization/rkhs_norm": 172.71815490722656,
      "rewards/chosen": 0.14702863019871204,
      "rewards/margins": 0.2649733845590418,
      "rewards/rejected": -0.11794475436032974,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 14.329428672790527,
      "kl": 5.690273284912109,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -36533533.47878788,
      "logits/rejected": -37200050.37419355,
      "logps/chosen": -509.5935606060606,
      "logps/rejected": -372.37386592741933,
      "loss": 0.5879,
      "loss/base_kto": 3.7858269214630127,
      "metrics/advantage_var": 239.9020538330078,
      "regularization/mmd": 0.9174237251281738,
      "regularization/rkhs_norm": 176.767578125,
      "rewards/chosen": 0.057962741273822206,
      "rewards/margins": 0.21008462355866342,
      "rewards/rejected": -0.15212188228484122,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 17.99163055419922,
      "kl": 6.039793491363525,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -36726399.78559464,
      "logits/rejected": -37461209.39385066,
      "logps/chosen": -472.76350502512565,
      "logps/rejected": -402.56927159590043,
      "loss": 0.5755,
      "loss/base_kto": 3.721526861190796,
      "metrics/advantage_var": 254.74917602539062,
      "regularization/mmd": 0.8826885223388672,
      "regularization/rkhs_norm": 170.07485961914062,
      "rewards/chosen": -0.007207811377955042,
      "rewards/margins": 0.2573227558861829,
      "rewards/rejected": -0.264530567264138,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 20.017250061035156,
      "kl": 7.532881259918213,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36146347.98151001,
      "logits/rejected": -36922759.09984152,
      "logps/chosen": -496.83840523882895,
      "logps/rejected": -372.50168383518223,
      "loss": 0.5828,
      "loss/base_kto": 3.754721164703369,
      "metrics/advantage_var": 261.9855651855469,
      "regularization/mmd": 0.9074088335037231,
      "regularization/rkhs_norm": 174.8379364013672,
      "rewards/chosen": 0.08191461078191208,
      "rewards/margins": 0.22445522963003262,
      "rewards/rejected": -0.14254061884812055,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 15.541690826416016,
      "kl": 9.11847972869873,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -34632853.19205298,
      "logits/rejected": -36152507.834319524,
      "logps/chosen": -489.0806084437086,
      "logps/rejected": -374.3659393491124,
      "loss": 0.581,
      "loss/base_kto": 3.6857070922851562,
      "metrics/advantage_var": 268.4297790527344,
      "regularization/mmd": 0.962521493434906,
      "regularization/rkhs_norm": 185.45693969726562,
      "rewards/chosen": 0.1476656174817622,
      "rewards/margins": 0.3030050108717631,
      "rewards/rejected": -0.1553393933900009,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 14.271484375,
      "kl": 10.363009452819824,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36159265.927710846,
      "logits/rejected": -36064482.077922076,
      "logps/chosen": -508.9699736445783,
      "logps/rejected": -377.15163352272725,
      "loss": 0.5943,
      "loss/base_kto": 3.725666046142578,
      "metrics/advantage_var": 329.1117858886719,
      "regularization/mmd": 1.0289491415023804,
      "regularization/rkhs_norm": 198.25608825683594,
      "rewards/chosen": 0.18816217169704208,
      "rewards/margins": 0.26723369964876564,
      "rewards/rejected": -0.07907152795172356,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 14.527031898498535,
      "kl": 7.910184383392334,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36705883.980922095,
      "logits/rejected": -36531236.17818741,
      "logps/chosen": -497.47406597774244,
      "logps/rejected": -370.1050787250384,
      "loss": 0.5809,
      "loss/base_kto": 3.6962249279022217,
      "metrics/advantage_var": 271.1266784667969,
      "regularization/mmd": 0.9510257840156555,
      "regularization/rkhs_norm": 183.24197387695312,
      "rewards/chosen": 0.09800809365956317,
      "rewards/margins": 0.27047829190850115,
      "rewards/rejected": -0.17247019824893794,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 16.18922233581543,
      "kl": 7.231024265289307,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35402629.51196172,
      "logits/rejected": -36064514.74425727,
      "logps/chosen": -480.5859250398724,
      "logps/rejected": -362.80015313935684,
      "loss": 0.5838,
      "loss/base_kto": 3.7060494422912598,
      "metrics/advantage_var": 267.0411071777344,
      "regularization/mmd": 0.964124858379364,
      "regularization/rkhs_norm": 185.76588439941406,
      "rewards/chosen": 0.03829733378579172,
      "rewards/margins": 0.282168721775718,
      "rewards/rejected": -0.2438713879899263,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 19.428863525390625,
      "kl": 6.699652194976807,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -34408579.405320816,
      "logits/rejected": -36810844.65522621,
      "logps/chosen": -472.9701682316119,
      "logps/rejected": -392.33014820592825,
      "loss": 0.5821,
      "loss/base_kto": 3.7921574115753174,
      "metrics/advantage_var": 216.35459899902344,
      "regularization/mmd": 0.8642802238464355,
      "regularization/rkhs_norm": 166.52798461914062,
      "rewards/chosen": 0.05617255746665322,
      "rewards/margins": 0.18439785127332736,
      "rewards/rejected": -0.12822529380667413,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 13.99908447265625,
      "kl": 11.094145774841309,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37669820.8256,
      "logits/rejected": -37664030.09465649,
      "logps/chosen": -468.0696,
      "logps/rejected": -397.8416030534351,
      "loss": 0.5725,
      "loss/base_kto": 3.6761891841888428,
      "metrics/advantage_var": 231.89492797851562,
      "regularization/mmd": 0.9035652279853821,
      "regularization/rkhs_norm": 174.0973358154297,
      "rewards/chosen": 0.1345001220703125,
      "rewards/margins": 0.2822148912735568,
      "rewards/rejected": -0.14771476920324428,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 12.809364318847656,
      "kl": 8.228848457336426,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -35409770.63209076,
      "logits/rejected": -36674515.209653094,
      "logps/chosen": -493.70765802269045,
      "logps/rejected": -365.242411387632,
      "loss": 0.5781,
      "loss/base_kto": 3.7145614624023438,
      "metrics/advantage_var": 234.45376586914062,
      "regularization/mmd": 0.9103992581367493,
      "regularization/rkhs_norm": 175.41412353515625,
      "rewards/chosen": 0.11652515232079999,
      "rewards/margins": 0.25433267330658316,
      "rewards/rejected": -0.13780752098578314,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 18.693523406982422,
      "kl": 9.805913925170898,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35212472.75840978,
      "logits/rejected": -36863155.936102234,
      "logps/chosen": -486.26748853211006,
      "logps/rejected": -368.1652605830671,
      "loss": 0.578,
      "loss/base_kto": 3.7344930171966553,
      "metrics/advantage_var": 215.81729125976562,
      "regularization/mmd": 0.889321506023407,
      "regularization/rkhs_norm": 171.35292053222656,
      "rewards/chosen": 0.14162699381510416,
      "rewards/margins": 0.2428915208647934,
      "rewards/rejected": -0.10126452704968925,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 24.230606079101562,
      "kl": 7.2070631980896,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36908811.27044025,
      "logits/rejected": -35498960.298136644,
      "logps/chosen": -476.51994889937106,
      "logps/rejected": -384.64834045031057,
      "loss": 0.5796,
      "loss/base_kto": 3.7236766815185547,
      "metrics/advantage_var": 243.0139617919922,
      "regularization/mmd": 0.9129219055175781,
      "regularization/rkhs_norm": 175.9001922607422,
      "rewards/chosen": 0.07593924444426531,
      "rewards/margins": 0.25153042300659545,
      "rewards/rejected": -0.17559117856233017,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 19.190467834472656,
      "kl": 7.582091808319092,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -36191556.37267081,
      "logits/rejected": -36882010.16352201,
      "logps/chosen": -489.1235442546584,
      "logps/rejected": -365.31318789308176,
      "loss": 0.5826,
      "loss/base_kto": 3.775066375732422,
      "metrics/advantage_var": 229.5860137939453,
      "regularization/mmd": 0.8857846260070801,
      "regularization/rkhs_norm": 170.6714324951172,
      "rewards/chosen": 0.13538101148901519,
      "rewards/margins": 0.2081826254466422,
      "rewards/rejected": -0.07280161395762702,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 20.92852210998535,
      "kl": 13.803125381469727,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -36878574.13953488,
      "logits/rejected": -37196543.596850395,
      "logps/chosen": -489.7798449612403,
      "logps/rejected": -366.0732529527559,
      "loss": 0.5838,
      "loss/base_kto": 3.735741376876831,
      "metrics/advantage_var": 250.020263671875,
      "regularization/mmd": 0.9350200891494751,
      "regularization/rkhs_norm": 180.15802001953125,
      "rewards/chosen": 0.23140116846838663,
      "rewards/margins": 0.24454379777641136,
      "rewards/rejected": -0.01314262930802473,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 19.827178955078125,
      "kl": 9.982133865356445,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -37341917.5623987,
      "logits/rejected": -38152565.76772247,
      "logps/chosen": -480.5630571312804,
      "logps/rejected": -385.2698906485671,
      "loss": 0.5793,
      "loss/base_kto": 3.7122788429260254,
      "metrics/advantage_var": 239.6793975830078,
      "regularization/mmd": 0.9221993684768677,
      "regularization/rkhs_norm": 177.687744140625,
      "rewards/chosen": 0.12878181791382826,
      "rewards/margins": 0.270680121283779,
      "rewards/rejected": -0.14189830336995074,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 17.058692932128906,
      "kl": 4.240655422210693,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36959863.14465409,
      "logits/rejected": -36906018.98136646,
      "logps/chosen": -471.84994103773585,
      "logps/rejected": -367.23163334627327,
      "loss": 0.5794,
      "loss/base_kto": 3.724996566772461,
      "metrics/advantage_var": 232.15652465820312,
      "regularization/mmd": 0.9098402261734009,
      "regularization/rkhs_norm": 175.30642700195312,
      "rewards/chosen": -0.040978053830704596,
      "rewards/margins": 0.2626346966788063,
      "rewards/rejected": -0.30361275050951086,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 13.596495628356934,
      "kl": 14.249995231628418,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -36507934.41203008,
      "logits/rejected": -37932126.90731707,
      "logps/chosen": -503.81954887218046,
      "logps/rejected": -360.6779217479675,
      "loss": 0.5784,
      "loss/base_kto": 3.680901050567627,
      "metrics/advantage_var": 252.53746032714844,
      "regularization/mmd": 0.9465328454971313,
      "regularization/rkhs_norm": 182.37628173828125,
      "rewards/chosen": 0.24967366842398966,
      "rewards/margins": 0.28866869480308643,
      "rewards/rejected": -0.0389950263790968,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 16.117652893066406,
      "kl": 11.919455528259277,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -37659727.25696594,
      "logits/rejected": -38779605.87341772,
      "logps/chosen": -470.2393575851393,
      "logps/rejected": -362.3188538370253,
      "loss": 0.5694,
      "loss/base_kto": 3.4670631885528564,
      "metrics/advantage_var": 372.93902587890625,
      "regularization/mmd": 1.0879830121994019,
      "regularization/rkhs_norm": 209.63064575195312,
      "rewards/chosen": 0.40930671809996616,
      "rewards/margins": 0.5463803677357639,
      "rewards/rejected": -0.13707364963579782,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 12.704465866088867,
      "kl": 9.122496604919434,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36618425.87458746,
      "logits/rejected": -38060457.400593475,
      "logps/chosen": -471.3839727722772,
      "logps/rejected": -366.02346068249255,
      "loss": 0.5713,
      "loss/base_kto": 3.233105182647705,
      "metrics/advantage_var": 561.404541015625,
      "regularization/mmd": 1.3372970819473267,
      "regularization/rkhs_norm": 257.6680603027344,
      "rewards/chosen": 0.3727112823587046,
      "rewards/margins": 0.8635088289265366,
      "rewards/rejected": -0.490797546567832,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 15.845054626464844,
      "kl": 7.558243751525879,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -34239751.684044234,
      "logits/rejected": -36231050.88098918,
      "logps/chosen": -464.8693226698262,
      "logps/rejected": -381.7858867851623,
      "loss": 0.5691,
      "loss/base_kto": 3.2288079261779785,
      "metrics/advantage_var": 574.4853515625,
      "regularization/mmd": 1.3237676620483398,
      "regularization/rkhs_norm": 255.0612335205078,
      "rewards/chosen": 0.41367517263403436,
      "rewards/margins": 0.8618098175094208,
      "rewards/rejected": -0.4481346448753864,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 12.698341369628906,
      "kl": 8.256165504455566,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36301010.05128205,
      "logits/rejected": -38198771.512195125,
      "logps/chosen": -487.06590544871796,
      "logps/rejected": -379.7520960365854,
      "loss": 0.5684,
      "loss/base_kto": 3.219708204269409,
      "metrics/advantage_var": 547.5316772460938,
      "regularization/mmd": 1.327531099319458,
      "regularization/rkhs_norm": 255.7863311767578,
      "rewards/chosen": 0.41121551318046373,
      "rewards/margins": 0.8806920734474701,
      "rewards/rejected": -0.46947656026700646,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 11.37092399597168,
      "kl": 8.794673919677734,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -35675913.331220284,
      "logits/rejected": -37025309.18952234,
      "logps/chosen": -473.1105388272583,
      "logps/rejected": -376.31399268104775,
      "loss": 0.5706,
      "loss/base_kto": 3.216496706008911,
      "metrics/advantage_var": 592.7921752929688,
      "regularization/mmd": 1.3485082387924194,
      "regularization/rkhs_norm": 259.82818603515625,
      "rewards/chosen": 0.37435173119304677,
      "rewards/margins": 0.8877941365763288,
      "rewards/rejected": -0.513442405383282,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 10.955135345458984,
      "kl": 11.502577781677246,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -34308980.658227846,
      "logits/rejected": -35746401.97530864,
      "logps/chosen": -456.9647943037975,
      "logps/rejected": -369.28120177469134,
      "loss": 0.5608,
      "loss/base_kto": 3.21818470954895,
      "metrics/advantage_var": 499.4261169433594,
      "regularization/mmd": 1.2681556940078735,
      "regularization/rkhs_norm": 244.3459930419922,
      "rewards/chosen": 0.46269433709639535,
      "rewards/margins": 0.8524536833872662,
      "rewards/rejected": -0.38975934629087094,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 11.038398742675781,
      "kl": 18.00704574584961,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -34835183.12893983,
      "logits/rejected": -34586880.87972508,
      "logps/chosen": -463.0404727793696,
      "logps/rejected": -374.49151632302403,
      "loss": 0.5675,
      "loss/base_kto": 3.201265811920166,
      "metrics/advantage_var": 578.2403564453125,
      "regularization/mmd": 1.3390430212020874,
      "regularization/rkhs_norm": 258.00445556640625,
      "rewards/chosen": 0.5857707463567783,
      "rewards/margins": 0.8486113581849302,
      "rewards/rejected": -0.2628406118281518,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 11.733174324035645,
      "kl": 20.5081844329834,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34788497.35247209,
      "logits/rejected": -36518286.3093415,
      "logps/chosen": -468.3646331738437,
      "logps/rejected": -350.9655436447167,
      "loss": 0.5542,
      "loss/base_kto": 3.2452073097229004,
      "metrics/advantage_var": 411.1564025878906,
      "regularization/mmd": 1.1881401538848877,
      "regularization/rkhs_norm": 228.92874145507812,
      "rewards/chosen": 0.5807296047180273,
      "rewards/margins": 0.7610419089007955,
      "rewards/rejected": -0.18031230418276822,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 15.177841186523438,
      "kl": 12.759466171264648,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35313090.81761006,
      "logits/rejected": -36495560.347826086,
      "logps/chosen": -474.4255110062893,
      "logps/rejected": -369.9761257763975,
      "loss": 0.5647,
      "loss/base_kto": 3.229013204574585,
      "metrics/advantage_var": 521.7021484375,
      "regularization/mmd": 1.28860342502594,
      "regularization/rkhs_norm": 248.2858123779297,
      "rewards/chosen": 0.4632223357194625,
      "rewards/margins": 0.8284195114976797,
      "rewards/rejected": -0.3651971757782172,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 9.456308364868164,
      "kl": 14.029640197753906,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -34754284.612055644,
      "logits/rejected": -35375521.36492891,
      "logps/chosen": -488.0619204018547,
      "logps/rejected": -349.91869075829385,
      "loss": 0.5709,
      "loss/base_kto": 3.2024781703948975,
      "metrics/advantage_var": 556.343017578125,
      "regularization/mmd": 1.3645777702331543,
      "regularization/rkhs_norm": 262.9244079589844,
      "rewards/chosen": 0.5272589422635723,
      "rewards/margins": 0.8624981055824654,
      "rewards/rejected": -0.33523916331889314,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 12.763233184814453,
      "kl": 13.030585289001465,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35205675.754122935,
      "logits/rejected": -35327893.08972268,
      "logps/chosen": -498.0701836581709,
      "logps/rejected": -373.61964722675367,
      "loss": 0.5664,
      "loss/base_kto": 3.2134625911712646,
      "metrics/advantage_var": 646.94775390625,
      "regularization/mmd": 1.317788004875183,
      "regularization/rkhs_norm": 253.9090576171875,
      "rewards/chosen": 0.5833422705151331,
      "rewards/margins": 0.887209255892185,
      "rewards/rejected": -0.3038669853770519,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 18.62638282775879,
      "kl": 13.3413724899292,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -35348151.57639939,
      "logits/rejected": -37456331.8901454,
      "logps/chosen": -500.2219648260212,
      "logps/rejected": -376.20100464458807,
      "loss": 0.5657,
      "loss/base_kto": 3.220020294189453,
      "metrics/advantage_var": 549.8190307617188,
      "regularization/mmd": 1.3052831888198853,
      "regularization/rkhs_norm": 251.4996337890625,
      "rewards/chosen": 0.528188731414287,
      "rewards/margins": 0.8517421834207364,
      "rewards/rejected": -0.3235534520064494,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 15.56907844543457,
      "kl": 17.59682273864746,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35049228.34726688,
      "logits/rejected": -36515973.835866265,
      "logps/chosen": -457.3328476688103,
      "logps/rejected": -377.69125189969606,
      "loss": 0.5639,
      "loss/base_kto": 3.2209317684173584,
      "metrics/advantage_var": 563.7723999023438,
      "regularization/mmd": 1.2898712158203125,
      "regularization/rkhs_norm": 248.5301055908203,
      "rewards/chosen": 0.49676317417353294,
      "rewards/margins": 0.8339053764407014,
      "rewards/rejected": -0.3371422022671685,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 14.420731544494629,
      "kl": 15.833075523376465,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35450782.32114468,
      "logits/rejected": -36681284.42396314,
      "logps/chosen": -489.96363275039744,
      "logps/rejected": -383.83885368663596,
      "loss": 0.5722,
      "loss/base_kto": 3.184199094772339,
      "metrics/advantage_var": 710.5199584960938,
      "regularization/mmd": 1.3931556940078735,
      "regularization/rkhs_norm": 268.4307556152344,
      "rewards/chosen": 0.5141282801787311,
      "rewards/margins": 0.8975747582533279,
      "rewards/rejected": -0.38344647807459675,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 13.51924991607666,
      "kl": 7.321361064910889,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -33592987.5443038,
      "logits/rejected": -35911743.209876545,
      "logps/chosen": -478.610957278481,
      "logps/rejected": -397.3753858024691,
      "loss": 0.5611,
      "loss/base_kto": 3.250958204269409,
      "metrics/advantage_var": 468.6973571777344,
      "regularization/mmd": 1.237536907196045,
      "regularization/rkhs_norm": 238.4464111328125,
      "rewards/chosen": 0.3320492611655706,
      "rewards/margins": 0.8196819184403584,
      "rewards/rejected": -0.48763265727478783,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 10.645336151123047,
      "kl": 13.960247993469238,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35695479.36431784,
      "logits/rejected": -36218365.494290374,
      "logps/chosen": -482.2657421289355,
      "logps/rejected": -375.55189641109297,
      "loss": 0.5679,
      "loss/base_kto": 3.1963846683502197,
      "metrics/advantage_var": 570.2843017578125,
      "regularization/mmd": 1.3467546701431274,
      "regularization/rkhs_norm": 259.4903259277344,
      "rewards/chosen": 0.5581533269010026,
      "rewards/margins": 0.8575564568395732,
      "rewards/rejected": -0.29940312993857054,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 13.598370552062988,
      "kl": 20.37125587463379,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -37020278.15384615,
      "logits/rejected": -37173443.83792545,
      "logps/chosen": -462.88353129713425,
      "logps/rejected": -382.0808093598055,
      "loss": 0.5577,
      "loss/base_kto": 3.15634822845459,
      "metrics/advantage_var": 540.4674072265625,
      "regularization/mmd": 1.3054908514022827,
      "regularization/rkhs_norm": 251.53965759277344,
      "rewards/chosen": 0.6769872236755279,
      "rewards/margins": 0.891834312308467,
      "rewards/rejected": -0.2148470886329391,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 14.927032470703125,
      "kl": 12.652097702026367,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35232756.8174727,
      "logits/rejected": -36155372.76995305,
      "logps/chosen": -504.7025643525741,
      "logps/rejected": -390.96909233176837,
      "loss": 0.5765,
      "loss/base_kto": 3.1824538707733154,
      "metrics/advantage_var": 801.2633056640625,
      "regularization/mmd": 1.4298820495605469,
      "regularization/rkhs_norm": 275.5071716308594,
      "rewards/chosen": 0.5310748928980596,
      "rewards/margins": 0.9131931776936386,
      "rewards/rejected": -0.382118284795579,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 12.155492782592773,
      "kl": 12.634875297546387,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -33998885.40639269,
      "logits/rejected": -35240964.93097913,
      "logps/chosen": -510.9780726788432,
      "logps/rejected": -367.4885132423756,
      "loss": 0.5634,
      "loss/base_kto": 3.177760124206543,
      "metrics/advantage_var": 548.15625,
      "regularization/mmd": 1.3298225402832031,
      "regularization/rkhs_norm": 256.22784423828125,
      "rewards/chosen": 0.5586960791089944,
      "rewards/margins": 0.9076369330055258,
      "rewards/rejected": -0.3489408538965314,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 11.107390403747559,
      "kl": 14.30541706085205,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -33680413.703816794,
      "logits/rejected": -36054204.416,
      "logps/chosen": -457.80753816793896,
      "logps/rejected": -373.3871,
      "loss": 0.5567,
      "loss/base_kto": 3.231868028640747,
      "metrics/advantage_var": 455.01080322265625,
      "regularization/mmd": 1.221943736076355,
      "regularization/rkhs_norm": 235.4419403076172,
      "rewards/chosen": 0.4908312149630248,
      "rewards/margins": 0.8198550919161498,
      "rewards/rejected": -0.329023876953125,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 15.35888671875,
      "kl": 15.176299095153809,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34026542.40483384,
      "logits/rejected": -35535394.79611651,
      "logps/chosen": -481.6341106495468,
      "logps/rejected": -390.0205805016181,
      "loss": 0.569,
      "loss/base_kto": 3.25787615776062,
      "metrics/advantage_var": 572.3829956054688,
      "regularization/mmd": 1.2940419912338257,
      "regularization/rkhs_norm": 249.33372497558594,
      "rewards/chosen": 0.5241825530176076,
      "rewards/margins": 0.7955340926047334,
      "rewards/rejected": -0.2713515395871258,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 15.133277893066406,
      "kl": 16.57997703552246,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34742376.8576,
      "logits/rejected": -35383405.4351145,
      "logps/chosen": -466.6202,
      "logps/rejected": -386.9606393129771,
      "loss": 0.5684,
      "loss/base_kto": 3.2192013263702393,
      "metrics/advantage_var": 629.2428588867188,
      "regularization/mmd": 1.3281208276748657,
      "regularization/rkhs_norm": 255.8999481201172,
      "rewards/chosen": 0.472026220703125,
      "rewards/margins": 0.7915639702483899,
      "rewards/rejected": -0.3195377495452648,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 10.430379867553711,
      "kl": 13.760481834411621,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -33074653.866666667,
      "logits/rejected": -36026586.65864661,
      "logps/chosen": -500.7354674796748,
      "logps/rejected": -401.15211466165414,
      "loss": 0.5577,
      "loss/base_kto": 3.153414487838745,
      "metrics/advantage_var": 536.83984375,
      "regularization/mmd": 1.3082083463668823,
      "regularization/rkhs_norm": 252.063232421875,
      "rewards/chosen": 0.5060722537157012,
      "rewards/margins": 0.8814182955169818,
      "rewards/rejected": -0.37534604180128056,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 11.11693000793457,
      "kl": 10.771425247192383,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -33730726.303759396,
      "logits/rejected": -34242689.87317073,
      "logps/chosen": -491.69915413533835,
      "logps/rejected": -374.57708333333335,
      "loss": 0.5635,
      "loss/base_kto": 3.1932923793792725,
      "metrics/advantage_var": 531.4647827148438,
      "regularization/mmd": 1.3149811029434204,
      "regularization/rkhs_norm": 253.3682098388672,
      "rewards/chosen": 0.489731454490719,
      "rewards/margins": 0.8919199866299465,
      "rewards/rejected": -0.4021885321392276,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 12.586292266845703,
      "kl": 20.481595993041992,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -35130701.80787402,
      "logits/rejected": -36678428.97364341,
      "logps/chosen": -492.09655511811025,
      "logps/rejected": -384.52284399224806,
      "loss": 0.5616,
      "loss/base_kto": 3.1790807247161865,
      "metrics/advantage_var": 545.5796508789062,
      "regularization/mmd": 1.3135855197906494,
      "regularization/rkhs_norm": 253.0993194580078,
      "rewards/chosen": 0.6542080616387795,
      "rewards/margins": 0.8926301656389006,
      "rewards/rejected": -0.23842210400012112,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 13.15192699432373,
      "kl": 15.327397346496582,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34864882.27871363,
      "logits/rejected": -37420546.449760765,
      "logps/chosen": -516.3475306278714,
      "logps/rejected": -353.7198963317384,
      "loss": 0.5722,
      "loss/base_kto": 3.1635806560516357,
      "metrics/advantage_var": 609.1104736328125,
      "regularization/mmd": 1.4136465787887573,
      "regularization/rkhs_norm": 272.37890625,
      "rewards/chosen": 0.6116748944175919,
      "rewards/margins": 0.9000959412084861,
      "rewards/rejected": -0.28842104679089414,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 16.29294204711914,
      "kl": 13.822619438171387,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -34549783.085346214,
      "logits/rejected": -36527761.28679818,
      "logps/chosen": -473.2524154589372,
      "logps/rejected": -366.9097591047041,
      "loss": 0.5574,
      "loss/base_kto": 3.207738161087036,
      "metrics/advantage_var": 483.0428161621094,
      "regularization/mmd": 1.2514506578445435,
      "regularization/rkhs_norm": 241.1272735595703,
      "rewards/chosen": 0.5131281608544686,
      "rewards/margins": 0.8448688686918642,
      "rewards/rejected": -0.3317407078373957,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 11.576709747314453,
      "kl": 12.038655281066895,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34267869.70716511,
      "logits/rejected": -34709391.64890282,
      "logps/chosen": -483.4979556074766,
      "logps/rejected": -383.8982415752351,
      "loss": 0.5645,
      "loss/base_kto": 3.2420220375061035,
      "metrics/advantage_var": 526.3206176757812,
      "regularization/mmd": 1.2743748426437378,
      "regularization/rkhs_norm": 245.5442657470703,
      "rewards/chosen": 0.4424971345800477,
      "rewards/margins": 0.8194643703724409,
      "rewards/rejected": -0.3769672357923932,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 11.38870620727539,
      "kl": 13.090853691101074,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -33939830.185970634,
      "logits/rejected": -36001134.92053973,
      "logps/chosen": -470.4476447797716,
      "logps/rejected": -368.1530640929535,
      "loss": 0.558,
      "loss/base_kto": 3.1668918132781982,
      "metrics/advantage_var": 513.3076171875,
      "regularization/mmd": 1.297245740890503,
      "regularization/rkhs_norm": 249.9510040283203,
      "rewards/chosen": 0.517286988300367,
      "rewards/margins": 0.893335452410597,
      "rewards/rejected": -0.37604846411023,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 13.650389671325684,
      "kl": 8.224946975708008,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -33210067.37614679,
      "logits/rejected": -35512692.958466455,
      "logps/chosen": -473.4014239296636,
      "logps/rejected": -364.13248801916933,
      "loss": 0.5638,
      "loss/base_kto": 3.248142957687378,
      "metrics/advantage_var": 484.4810485839844,
      "regularization/mmd": 1.2620840072631836,
      "regularization/rkhs_norm": 243.17613220214844,
      "rewards/chosen": 0.4389438454164277,
      "rewards/margins": 0.8400392753457284,
      "rewards/rejected": -0.40109542992930064,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 35.175785064697266,
      "kl": 17.141820907592773,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35519068.86677367,
      "logits/rejected": -35112560.99847793,
      "logps/chosen": -500.48475120385234,
      "logps/rejected": -396.3122621765601,
      "loss": 0.5787,
      "loss/base_kto": 3.1796255111694336,
      "metrics/advantage_var": 821.1322631835938,
      "regularization/mmd": 1.4498852491378784,
      "regularization/rkhs_norm": 279.361328125,
      "rewards/chosen": 0.5144086565290179,
      "rewards/margins": 0.8857092014419745,
      "rewards/rejected": -0.3713005449129566,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 16.471954345703125,
      "kl": 6.620677471160889,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -33879606.4,
      "logits/rejected": -36046924.8,
      "logps/chosen": -471.56845703125,
      "logps/rejected": -379.3955810546875,
      "loss": 0.5753,
      "loss/base_kto": 3.2660164833068848,
      "metrics/advantage_var": 629.7620239257812,
      "regularization/mmd": 1.336260437965393,
      "regularization/rkhs_norm": 257.4682922363281,
      "rewards/chosen": 0.41168975830078125,
      "rewards/margins": 0.8532877922058105,
      "rewards/rejected": -0.4415980339050293,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 17.38668441772461,
      "kl": 12.52214241027832,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -35436497.94302849,
      "logits/rejected": -35837878.49918434,
      "logps/chosen": -469.97985382308843,
      "logps/rejected": -359.86421798531813,
      "loss": 0.5749,
      "loss/base_kto": 3.249713659286499,
      "metrics/advantage_var": 892.2927856445312,
      "regularization/mmd": 1.34954833984375,
      "regularization/rkhs_norm": 260.028564453125,
      "rewards/chosen": 0.516921745426115,
      "rewards/margins": 0.7756842730661793,
      "rewards/rejected": -0.25876252764006424,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 11.343464851379395,
      "kl": 12.849252700805664,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34218797.99046105,
      "logits/rejected": -35599304.94623656,
      "logps/chosen": -496.7043918918919,
      "logps/rejected": -386.3777361751152,
      "loss": 0.5582,
      "loss/base_kto": 3.1670498847961426,
      "metrics/advantage_var": 505.1261901855469,
      "regularization/mmd": 1.2983702421188354,
      "regularization/rkhs_norm": 250.1676788330078,
      "rewards/chosen": 0.5697533830360691,
      "rewards/margins": 0.883675239659893,
      "rewards/rejected": -0.31392185662382394,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 12.956347465515137,
      "kl": 9.079574584960938,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -35288816.42433697,
      "logits/rejected": -35891449.99061033,
      "logps/chosen": -480.0551872074883,
      "logps/rejected": -364.0055751173709,
      "loss": 0.5746,
      "loss/base_kto": 3.2379226684570312,
      "metrics/advantage_var": 586.6058959960938,
      "regularization/mmd": 1.3591790199279785,
      "regularization/rkhs_norm": 261.88421630859375,
      "rewards/chosen": 0.43446138236154447,
      "rewards/margins": 0.8490052864108402,
      "rewards/rejected": -0.41454390404929575,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 9.659762382507324,
      "kl": 8.767071723937988,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -33956107.41401274,
      "logits/rejected": -35763363.33742331,
      "logps/chosen": -466.2066580414013,
      "logps/rejected": -391.6272526840491,
      "loss": 0.5647,
      "loss/base_kto": 3.2862496376037598,
      "metrics/advantage_var": 492.1122131347656,
      "regularization/mmd": 1.231597900390625,
      "regularization/rkhs_norm": 237.30210876464844,
      "rewards/chosen": 0.3718239365109972,
      "rewards/margins": 0.7595649317659818,
      "rewards/rejected": -0.3877409952549847,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 12.61690616607666,
      "kl": 9.680636405944824,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -34631756.68018721,
      "logits/rejected": -37399765.13302034,
      "logps/chosen": -488.1589313572543,
      "logps/rejected": -391.39862089201876,
      "loss": 0.5599,
      "loss/base_kto": 3.236057996749878,
      "metrics/advantage_var": 480.9098205566406,
      "regularization/mmd": 1.2427895069122314,
      "regularization/rkhs_norm": 239.45849609375,
      "rewards/chosen": 0.43861253473576445,
      "rewards/margins": 0.8102007506905644,
      "rewards/rejected": -0.37158821595479996,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 13.531383514404297,
      "kl": 9.788293838500977,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33087113.32713178,
      "logits/rejected": -34771073.00787402,
      "logps/chosen": -505.1265503875969,
      "logps/rejected": -365.8290846456693,
      "loss": 0.5659,
      "loss/base_kto": 3.218250036239624,
      "metrics/advantage_var": 558.4009399414062,
      "regularization/mmd": 1.3091548681259155,
      "regularization/rkhs_norm": 252.2456512451172,
      "rewards/chosen": 0.48601793392684106,
      "rewards/margins": 0.8748068330256353,
      "rewards/rejected": -0.38878889909879427,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 10.86054515838623,
      "kl": 11.343132972717285,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -32781315.36842105,
      "logits/rejected": -34524117.333333336,
      "logps/chosen": -491.22974917763156,
      "logps/rejected": -369.2662760416667,
      "loss": 0.5564,
      "loss/base_kto": 3.237612247467041,
      "metrics/advantage_var": 437.27813720703125,
      "regularization/mmd": 1.2136201858520508,
      "regularization/rkhs_norm": 233.8381805419922,
      "rewards/chosen": 0.4579402522036904,
      "rewards/margins": 0.795162646692797,
      "rewards/rejected": -0.3372223944891067,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 8.429838180541992,
      "kl": 4.938611030578613,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35576375.98732171,
      "logits/rejected": -35768968.90262751,
      "logps/chosen": -500.66333201267827,
      "logps/rejected": -373.9417986862442,
      "loss": 0.5488,
      "loss/base_kto": 3.1686315536499023,
      "metrics/advantage_var": 463.1637878417969,
      "regularization/mmd": 1.2215324640274048,
      "regularization/rkhs_norm": 235.36270141601562,
      "rewards/chosen": 0.37699362970947653,
      "rewards/margins": 0.9123040241380893,
      "rewards/rejected": -0.5353103944286128,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 15.727373123168945,
      "kl": 10.224392890930176,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34169435.11181103,
      "logits/rejected": -35353080.85581395,
      "logps/chosen": -483.05826771653545,
      "logps/rejected": -390.73168604651164,
      "loss": 0.5299,
      "loss/base_kto": 3.1048107147216797,
      "metrics/advantage_var": 380.64837646484375,
      "regularization/mmd": 1.1341379880905151,
      "regularization/rkhs_norm": 218.52371215820312,
      "rewards/chosen": 0.4996672863096703,
      "rewards/margins": 0.9463607895194571,
      "rewards/rejected": -0.44669350320978685,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 13.826533317565918,
      "kl": 9.03232479095459,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -34003890.42424242,
      "logits/rejected": -34565113.393548384,
      "logps/chosen": -479.08357007575756,
      "logps/rejected": -382.40057963709677,
      "loss": 0.5293,
      "loss/base_kto": 3.0912117958068848,
      "metrics/advantage_var": 367.74786376953125,
      "regularization/mmd": 1.143200159072876,
      "regularization/rkhs_norm": 220.269775390625,
      "rewards/chosen": 0.5297567656545928,
      "rewards/margins": 0.9286658114585242,
      "rewards/rejected": -0.39890904580393144,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 10.95798110961914,
      "kl": 9.553214073181152,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34640685.2733119,
      "logits/rejected": -36745863.392097265,
      "logps/chosen": -479.6584103697749,
      "logps/rejected": -407.0407959726444,
      "loss": 0.5306,
      "loss/base_kto": 3.100395917892456,
      "metrics/advantage_var": 376.327392578125,
      "regularization/mmd": 1.144347071647644,
      "regularization/rkhs_norm": 220.4907684326172,
      "rewards/chosen": 0.5258300388740956,
      "rewards/margins": 0.9229663212876595,
      "rewards/rejected": -0.39713628241356386,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 10.353028297424316,
      "kl": 10.044342041015625,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34038157.42097027,
      "logits/rejected": -35445748.8174727,
      "logps/chosen": -455.44600938967136,
      "logps/rejected": -377.6342628705148,
      "loss": 0.5329,
      "loss/base_kto": 3.1051576137542725,
      "metrics/advantage_var": 379.6630554199219,
      "regularization/mmd": 1.1578160524368286,
      "regularization/rkhs_norm": 223.0859375,
      "rewards/chosen": 0.5091337821852993,
      "rewards/margins": 0.9376146238115424,
      "rewards/rejected": -0.42848084162624317,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 9.661334037780762,
      "kl": 9.330436706542969,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -35977493.590361446,
      "logits/rejected": -36188432.62337662,
      "logps/chosen": -475.7725903614458,
      "logps/rejected": -377.3262987012987,
      "loss": 0.5294,
      "loss/base_kto": 3.0793023109436035,
      "metrics/advantage_var": 376.308837890625,
      "regularization/mmd": 1.1555709838867188,
      "regularization/rkhs_norm": 222.6533660888672,
      "rewards/chosen": 0.5504086965537933,
      "rewards/margins": 0.9590159565270964,
      "rewards/rejected": -0.40860725997330305,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 12.885091781616211,
      "kl": 13.551025390625,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34820204.29237947,
      "logits/rejected": -36722845.538461536,
      "logps/chosen": -498.05234253499225,
      "logps/rejected": -364.3190737833595,
      "loss": 0.5244,
      "loss/base_kto": 3.0458972454071045,
      "metrics/advantage_var": 389.7593688964844,
      "regularization/mmd": 1.1493853330612183,
      "regularization/rkhs_norm": 221.4615478515625,
      "rewards/chosen": 0.6288376685063909,
      "rewards/margins": 0.9763314136005824,
      "rewards/rejected": -0.3474937450941915,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 13.358975410461426,
      "kl": 12.604578018188477,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34915930.16352201,
      "logits/rejected": -36279900.22360248,
      "logps/chosen": -478.5619103773585,
      "logps/rejected": -371.0585209627329,
      "loss": 0.5284,
      "loss/base_kto": 3.1058311462402344,
      "metrics/advantage_var": 371.24053955078125,
      "regularization/mmd": 1.1212542057037354,
      "regularization/rkhs_norm": 216.04129028320312,
      "rewards/chosen": 0.5444417509642787,
      "rewards/margins": 0.9048593717001581,
      "rewards/rejected": -0.3604176207358793,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 12.47192096710205,
      "kl": 13.442543029785156,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34275425.52380952,
      "logits/rejected": -34517924.627692305,
      "logps/chosen": -466.6615079365079,
      "logps/rejected": -383.03625,
      "loss": 0.5273,
      "loss/base_kto": 3.1435506343841553,
      "metrics/advantage_var": 328.599853515625,
      "regularization/mmd": 1.075141191482544,
      "regularization/rkhs_norm": 207.15628051757812,
      "rewards/chosen": 0.49446861630394345,
      "rewards/margins": 0.857264636811756,
      "rewards/rejected": -0.3627960205078125,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 12.349054336547852,
      "kl": 9.524312019348145,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -35792114.87179487,
      "logits/rejected": -37086373.46341463,
      "logps/chosen": -477.56275040064105,
      "logps/rejected": -368.961318597561,
      "loss": 0.5267,
      "loss/base_kto": 3.122302293777466,
      "metrics/advantage_var": 341.88995361328125,
      "regularization/mmd": 1.0910618305206299,
      "regularization/rkhs_norm": 210.2238311767578,
      "rewards/chosen": 0.4709402720133464,
      "rewards/margins": 0.9085386167696821,
      "rewards/rejected": -0.43759834475633574,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 14.046501159667969,
      "kl": 9.571139335632324,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34408404.688821755,
      "logits/rejected": -35771739.96116505,
      "logps/chosen": -487.6857533987915,
      "logps/rejected": -368.93782868122975,
      "loss": 0.5244,
      "loss/base_kto": 3.106714963912964,
      "metrics/advantage_var": 336.606201171875,
      "regularization/mmd": 1.0885037183761597,
      "regularization/rkhs_norm": 209.73095703125,
      "rewards/chosen": 0.5020798974166824,
      "rewards/margins": 0.9009448904164042,
      "rewards/rejected": -0.3988649929997219,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 8.706116676330566,
      "kl": 15.946802139282227,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34518271.19242902,
      "logits/rejected": -35398031.45510836,
      "logps/chosen": -485.34611593059935,
      "logps/rejected": -389.8077109133127,
      "loss": 0.5297,
      "loss/base_kto": 3.0599441528320312,
      "metrics/advantage_var": 391.05816650390625,
      "regularization/mmd": 1.17766273021698,
      "regularization/rkhs_norm": 226.9099578857422,
      "rewards/chosen": 0.6544270801243346,
      "rewards/margins": 0.959031412637819,
      "rewards/rejected": -0.3046043325134844,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 13.162550926208496,
      "kl": 10.673392295837402,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34212820.02453988,
      "logits/rejected": -35669468.12738854,
      "logps/chosen": -465.9872028374233,
      "logps/rejected": -371.60225915605093,
      "loss": 0.5275,
      "loss/base_kto": 3.1158010959625244,
      "metrics/advantage_var": 348.5242614746094,
      "regularization/mmd": 1.1044648885726929,
      "regularization/rkhs_norm": 212.8063507080078,
      "rewards/chosen": 0.5190921151564897,
      "rewards/margins": 0.9168897665081268,
      "rewards/rejected": -0.39779765135163714,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 10.201983451843262,
      "kl": 13.881949424743652,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -35078533.94902549,
      "logits/rejected": -36514550.39477977,
      "logps/chosen": -525.0691529235382,
      "logps/rejected": -358.8348796900489,
      "loss": 0.5295,
      "loss/base_kto": 3.0585391521453857,
      "metrics/advantage_var": 389.27783203125,
      "regularization/mmd": 1.1772745847702026,
      "regularization/rkhs_norm": 226.83518981933594,
      "rewards/chosen": 0.6114501953125,
      "rewards/margins": 0.9704890538973288,
      "rewards/rejected": -0.3590388585848287,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 8.7860746383667,
      "kl": 15.788287162780762,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -33913246.059097975,
      "logits/rejected": -35752505.06750392,
      "logps/chosen": -490.89900855365477,
      "logps/rejected": -366.88947213500785,
      "loss": 0.5299,
      "loss/base_kto": 3.11326003074646,
      "metrics/advantage_var": 357.09552001953125,
      "regularization/mmd": 1.1259396076202393,
      "regularization/rkhs_norm": 216.9440460205078,
      "rewards/chosen": 0.6084082145156979,
      "rewards/margins": 0.9001660346810236,
      "rewards/rejected": -0.29175782016532575,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 31.27370262145996,
      "kl": 15.632532119750977,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -34557501.244019136,
      "logits/rejected": -35704850.81776416,
      "logps/chosen": -450.49601275917064,
      "logps/rejected": -386.5971238514548,
      "loss": 0.5288,
      "loss/base_kto": 3.090442657470703,
      "metrics/advantage_var": 366.620361328125,
      "regularization/mmd": 1.1398146152496338,
      "regularization/rkhs_norm": 219.61746215820312,
      "rewards/chosen": 0.5583569589033842,
      "rewards/margins": 0.9100697334400778,
      "rewards/rejected": -0.35171277453669364,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 12.42688274383545,
      "kl": 13.420425415039062,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -33294260.381538462,
      "logits/rejected": -35762162.9968254,
      "logps/chosen": -475.5403846153846,
      "logps/rejected": -371.23055555555555,
      "loss": 0.5364,
      "loss/base_kto": 3.106178045272827,
      "metrics/advantage_var": 407.9663391113281,
      "regularization/mmd": 1.1849819421768188,
      "regularization/rkhs_norm": 228.32022094726562,
      "rewards/chosen": 0.5992143366887019,
      "rewards/margins": 0.9340711753388755,
      "rewards/rejected": -0.3348568386501736,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 8.732342720031738,
      "kl": 12.329475402832031,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -33890418.526315786,
      "logits/rejected": -36418870.85714286,
      "logps/chosen": -475.4996402138158,
      "logps/rejected": -377.1099330357143,
      "loss": 0.5301,
      "loss/base_kto": 3.1279115676879883,
      "metrics/advantage_var": 363.8894958496094,
      "regularization/mmd": 1.1131722927093506,
      "regularization/rkhs_norm": 214.48403930664062,
      "rewards/chosen": 0.5284870549252159,
      "rewards/margins": 0.8872456180123159,
      "rewards/rejected": -0.3587585630871001,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 13.289918899536133,
      "kl": 9.346504211425781,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34114033.82153846,
      "logits/rejected": -36181222.8063492,
      "logps/chosen": -478.6663461538462,
      "logps/rejected": -362.11063988095236,
      "loss": 0.5295,
      "loss/base_kto": 3.1132397651672363,
      "metrics/advantage_var": 358.97418212890625,
      "regularization/mmd": 1.1230419874191284,
      "regularization/rkhs_norm": 216.3857421875,
      "rewards/chosen": 0.5090151742788461,
      "rewards/margins": 0.9224266326121795,
      "rewards/rejected": -0.4134114583333333,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 16.736160278320312,
      "kl": 10.813815116882324,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -33832985.801574804,
      "logits/rejected": -35365714.95193798,
      "logps/chosen": -509.0462598425197,
      "logps/rejected": -384.3090843023256,
      "loss": 0.532,
      "loss/base_kto": 3.09063982963562,
      "metrics/advantage_var": 386.1016845703125,
      "regularization/mmd": 1.1649986505508423,
      "regularization/rkhs_norm": 224.46987915039062,
      "rewards/chosen": 0.5305650125338337,
      "rewards/margins": 0.9422878315442504,
      "rewards/rejected": -0.4117228190104167,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 10.82371997833252,
      "kl": 12.002352714538574,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34679065.8048,
      "logits/rejected": -36309939.39541985,
      "logps/chosen": -460.6847,
      "logps/rejected": -379.85696564885495,
      "loss": 0.5252,
      "loss/base_kto": 3.0716331005096436,
      "metrics/advantage_var": 372.6279602050781,
      "regularization/mmd": 1.1301687955856323,
      "regularization/rkhs_norm": 217.75889587402344,
      "rewards/chosen": 0.5291451171875,
      "rewards/margins": 0.939521233525167,
      "rewards/rejected": -0.41037611633766696,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 12.417250633239746,
      "kl": 9.33301830291748,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35359239.87692308,
      "logits/rejected": -35752865.72698413,
      "logps/chosen": -484.42115384615386,
      "logps/rejected": -395.8510912698413,
      "loss": 0.5325,
      "loss/base_kto": 3.1038146018981934,
      "metrics/advantage_var": 398.2544860839844,
      "regularization/mmd": 1.156279444694519,
      "regularization/rkhs_norm": 222.7898406982422,
      "rewards/chosen": 0.5300423490084135,
      "rewards/margins": 0.9482267565604969,
      "rewards/rejected": -0.41818440755208336,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 11.20324420928955,
      "kl": 13.021185874938965,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -34471727.84262295,
      "logits/rejected": -35850325.5880597,
      "logps/chosen": -475.0947745901639,
      "logps/rejected": -402.4258395522388,
      "loss": 0.5245,
      "loss/base_kto": 3.1126742362976074,
      "metrics/advantage_var": 339.88214111328125,
      "regularization/mmd": 1.0831588506698608,
      "regularization/rkhs_norm": 208.7011260986328,
      "rewards/chosen": 0.5231860551677766,
      "rewards/margins": 0.9091018284740966,
      "rewards/rejected": -0.38591577330632,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 13.183117866516113,
      "kl": 8.468612670898438,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34805089.27186009,
      "logits/rejected": -35219817.78187404,
      "logps/chosen": -507.5408386327504,
      "logps/rejected": -360.90430587557603,
      "loss": 0.5339,
      "loss/base_kto": 3.110563039779663,
      "metrics/advantage_var": 389.3725280761719,
      "regularization/mmd": 1.1603578329086304,
      "regularization/rkhs_norm": 223.57568359375,
      "rewards/chosen": 0.5026676923936805,
      "rewards/margins": 0.9296209861544812,
      "rewards/rejected": -0.4269532937608007,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 10.095125198364258,
      "kl": 9.794113159179688,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -34838252.8,
      "logits/rejected": -35133248.0,
      "logps/chosen": -499.425,
      "logps/rejected": -408.65634765625,
      "loss": 0.5297,
      "loss/base_kto": 3.0677144527435303,
      "metrics/advantage_var": 388.46734619140625,
      "regularization/mmd": 1.1697847843170166,
      "regularization/rkhs_norm": 225.3920440673828,
      "rewards/chosen": 0.4837846279144287,
      "rewards/margins": 0.9642529010772705,
      "rewards/rejected": -0.4804682731628418,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 10.045802116394043,
      "kl": 10.972128868103027,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34800860.8,
      "logits/rejected": -35979273.6,
      "logps/chosen": -498.69697265625,
      "logps/rejected": -378.28642578125,
      "loss": 0.5287,
      "loss/base_kto": 3.090186357498169,
      "metrics/advantage_var": 368.100341796875,
      "regularization/mmd": 1.1397637128829956,
      "regularization/rkhs_norm": 219.607666015625,
      "rewards/chosen": 0.5003337860107422,
      "rewards/margins": 0.9343658924102783,
      "rewards/rejected": -0.43403210639953616,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 11.280572891235352,
      "kl": 8.247645378112793,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -33784674.11453745,
      "logits/rejected": -36023906.29716194,
      "logps/chosen": -503.7002569750367,
      "logps/rejected": -352.47876669449084,
      "loss": 0.5397,
      "loss/base_kto": 3.07843017578125,
      "metrics/advantage_var": 763.4011840820312,
      "regularization/mmd": 1.2387841939926147,
      "regularization/rkhs_norm": 238.68675231933594,
      "rewards/chosen": 0.557862269195691,
      "rewards/margins": 0.9298415893842866,
      "rewards/rejected": -0.3719793201885956,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 8.601116180419922,
      "kl": 11.35707950592041,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -32804689.56466877,
      "logits/rejected": -35856808.81733746,
      "logps/chosen": -475.0592468454259,
      "logps/rejected": -353.30098684210526,
      "loss": 0.5302,
      "loss/base_kto": 3.1280622482299805,
      "metrics/advantage_var": 365.3536682128906,
      "regularization/mmd": 1.113821268081665,
      "regularization/rkhs_norm": 214.6090850830078,
      "rewards/chosen": 0.501492665769174,
      "rewards/margins": 0.8903800770697858,
      "rewards/rejected": -0.38888741130061194,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 12.126202583312988,
      "kl": 13.453925132751465,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34088487.384615384,
      "logits/rejected": -36759580.665629864,
      "logps/chosen": -492.50539638932497,
      "logps/rejected": -376.3147113141524,
      "loss": 0.5308,
      "loss/base_kto": 3.071422815322876,
      "metrics/advantage_var": 398.5673828125,
      "regularization/mmd": 1.1750009059906006,
      "regularization/rkhs_norm": 226.39707946777344,
      "rewards/chosen": 0.5706526009216788,
      "rewards/margins": 0.9671484140808205,
      "rewards/rejected": -0.39649581315914173,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 10.9151029586792,
      "kl": 9.662836074829102,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35570628.13538461,
      "logits/rejected": -37069053.56190476,
      "logps/chosen": -485.08846153846156,
      "logps/rejected": -382.0393353174603,
      "loss": 0.5296,
      "loss/base_kto": 3.0812783241271973,
      "metrics/advantage_var": 394.0767517089844,
      "regularization/mmd": 1.1554718017578125,
      "regularization/rkhs_norm": 222.63426208496094,
      "rewards/chosen": 0.5557008713942307,
      "rewards/margins": 0.972709480257698,
      "rewards/rejected": -0.41700860886346724,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 10.44419002532959,
      "kl": 11.893241882324219,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34611472.20253164,
      "logits/rejected": -34804666.469135806,
      "logps/chosen": -483.117929193038,
      "logps/rejected": -354.1086516203704,
      "loss": 0.5247,
      "loss/base_kto": 3.085773229598999,
      "metrics/advantage_var": 369.8644104003906,
      "regularization/mmd": 1.1118484735488892,
      "regularization/rkhs_norm": 214.22900390625,
      "rewards/chosen": 0.5326901206487342,
      "rewards/margins": 0.9465245865233002,
      "rewards/rejected": -0.41383446587456596,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 9.923108100891113,
      "kl": 11.843527793884277,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35572256.820512824,
      "logits/rejected": -34997360.3902439,
      "logps/chosen": -460.6005608974359,
      "logps/rejected": -404.8897198932927,
      "loss": 0.5256,
      "loss/base_kto": 3.0892350673675537,
      "metrics/advantage_var": 350.0757751464844,
      "regularization/mmd": 1.1153289079666138,
      "regularization/rkhs_norm": 214.8996124267578,
      "rewards/chosen": 0.48085985428247696,
      "rewards/margins": 0.9150912634949151,
      "rewards/rejected": -0.4342314092124381,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 14.040332794189453,
      "kl": 11.11498737335205,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -33898979.95618153,
      "logits/rejected": -34968705.39781591,
      "logps/chosen": -464.94874804381845,
      "logps/rejected": -375.38672484399376,
      "loss": 0.5275,
      "loss/base_kto": 3.1421051025390625,
      "metrics/advantage_var": 339.7447204589844,
      "regularization/mmd": 1.078227162361145,
      "regularization/rkhs_norm": 207.75088500976562,
      "rewards/chosen": 0.5223304617199727,
      "rewards/margins": 0.8760799913398294,
      "rewards/rejected": -0.3537495296198567,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 9.759392738342285,
      "kl": 10.804986953735352,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35574749.01863354,
      "logits/rejected": -36234394.56603774,
      "logps/chosen": -496.5819099378882,
      "logps/rejected": -370.5633352987421,
      "loss": 0.5302,
      "loss/base_kto": 3.097810745239258,
      "metrics/advantage_var": 378.4516296386719,
      "regularization/mmd": 1.1435234546661377,
      "regularization/rkhs_norm": 220.3320770263672,
      "rewards/chosen": 0.5362494332449776,
      "rewards/margins": 0.9271788892720265,
      "rewards/rejected": -0.3909294560270489,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 9.869982719421387,
      "kl": 11.468153953552246,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -34745472.80503145,
      "logits/rejected": -36531956.86956522,
      "logps/chosen": -470.8713639937107,
      "logps/rejected": -372.5007278726708,
      "loss": 0.5236,
      "loss/base_kto": 3.0981380939483643,
      "metrics/advantage_var": 346.8184509277344,
      "regularization/mmd": 1.0902889966964722,
      "regularization/rkhs_norm": 210.074951171875,
      "rewards/chosen": 0.5356377175768966,
      "rewards/margins": 0.915488519791146,
      "rewards/rejected": -0.37985080221424933,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 14.449504852294922,
      "kl": 11.762713432312012,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -35228356.682926826,
      "logits/rejected": -35463424.0,
      "logps/chosen": -469.6153296493902,
      "logps/rejected": -390.17052283653845,
      "loss": 0.5292,
      "loss/base_kto": 3.1139495372772217,
      "metrics/advantage_var": 372.5120544433594,
      "regularization/mmd": 1.1196266412734985,
      "regularization/rkhs_norm": 215.72769165039062,
      "rewards/chosen": 0.5367694017363758,
      "rewards/margins": 0.9177948138801808,
      "rewards/rejected": -0.3810254121438051,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 11.837974548339844,
      "kl": 11.047762870788574,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34728321.01105845,
      "logits/rejected": -36686320.96445131,
      "logps/chosen": -478.1500789889416,
      "logps/rejected": -363.3257341576507,
      "loss": 0.5278,
      "loss/base_kto": 3.11044979095459,
      "metrics/advantage_var": 355.1884765625,
      "regularization/mmd": 1.1120823621749878,
      "regularization/rkhs_norm": 214.2740478515625,
      "rewards/chosen": 0.5352003148758393,
      "rewards/margins": 0.9138535802849994,
      "rewards/rejected": -0.37865326540916006,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 12.430468559265137,
      "kl": 12.598543167114258,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -33505824.851330202,
      "logits/rejected": -35347122.121684864,
      "logps/chosen": -475.14074726134584,
      "logps/rejected": -352.0942618954758,
      "loss": 0.5259,
      "loss/base_kto": 3.089165210723877,
      "metrics/advantage_var": 367.65313720703125,
      "regularization/mmd": 1.1177597045898438,
      "regularization/rkhs_norm": 215.3679656982422,
      "rewards/chosen": 0.5344488534942292,
      "rewards/margins": 0.9333023637869846,
      "rewards/rejected": -0.39885351029275545,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.972016217523487e+17,
      "train_loss": 0.5590571298912194,
      "train_runtime": 11070.9945,
      "train_samples_per_second": 13.388,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.972016217523487e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}