{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 20.67009162902832,
      "kl": 1.9479252099990845,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36745789.44,
      "logits/rejected": -38806148.51764706,
      "logps/chosen": -486.13395833333334,
      "logps/rejected": -357.45815716911767,
      "loss": 0.6003,
      "loss/base_kto": 3.873600721359253,
      "metrics/advantage_var": 199.37672424316406,
      "regularization/lipschitz_norm": 943.3322143554688,
      "regularization/mmd": 0.1362994909286499,
      "regularization/rkhs_norm": 162.26129150390625,
      "regularization/w1": 0.7923991084098816,
      "rewards/chosen": -0.2502688344319661,
      "rewards/margins": 0.08248990077598423,
      "rewards/rejected": -0.33275873520795035,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 20.44187355041504,
      "kl": 3.2897727489471436,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35149222.78178964,
      "logits/rejected": -36379245.08864697,
      "logps/chosen": -465.46178375196234,
      "logps/rejected": -379.6661158631415,
      "loss": 0.5941,
      "loss/base_kto": 3.940912961959839,
      "metrics/advantage_var": 152.79051208496094,
      "regularization/lipschitz_norm": 825.8759765625,
      "regularization/mmd": 0.11786098778247833,
      "regularization/rkhs_norm": 140.31069946289062,
      "regularization/w1": 0.6937357783317566,
      "rewards/chosen": -0.023100373883449488,
      "rewards/margins": 0.05192807082496384,
      "rewards/rejected": -0.07502844470841333,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 23.028261184692383,
      "kl": 4.990315914154053,
      "learning_rate": 5.9e-07,
      "logits/chosen": -34687452.12738854,
      "logits/rejected": -37365656.34355828,
      "logps/chosen": -481.92112858280257,
      "logps/rejected": -378.5552386886503,
      "loss": 0.589,
      "loss/base_kto": 3.824028491973877,
      "metrics/advantage_var": 176.4379425048828,
      "regularization/lipschitz_norm": 904.6116333007812,
      "regularization/mmd": 0.12806551158428192,
      "regularization/rkhs_norm": 152.4589385986328,
      "regularization/w1": 0.7598738074302673,
      "rewards/chosen": 0.024187677225489524,
      "rewards/margins": 0.1632834282799374,
      "rewards/rejected": -0.13909575105444785,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 28.615266799926758,
      "kl": 3.786818027496338,
      "learning_rate": 7.9e-07,
      "logits/chosen": -36571955.2,
      "logits/rejected": -37887108.885496184,
      "logps/chosen": -479.1769,
      "logps/rejected": -371.84217557251907,
      "loss": 0.5936,
      "loss/base_kto": 3.852421522140503,
      "metrics/advantage_var": 198.41836547851562,
      "regularization/lipschitz_norm": 910.3705444335938,
      "regularization/mmd": 0.131294384598732,
      "regularization/rkhs_norm": 156.30284118652344,
      "regularization/w1": 0.7647112607955933,
      "rewards/chosen": 0.05660130615234375,
      "rewards/margins": 0.15614959940437143,
      "rewards/rejected": -0.09954829325202767,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 24.30677032470703,
      "kl": 4.0709686279296875,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35430678.820097245,
      "logits/rejected": -36955505.13423831,
      "logps/chosen": -448.11841572123177,
      "logps/rejected": -361.8265460030166,
      "loss": 0.5904,
      "loss/base_kto": 3.8806111812591553,
      "metrics/advantage_var": 162.788818359375,
      "regularization/lipschitz_norm": 857.3562622070312,
      "regularization/mmd": 0.12275510281324387,
      "regularization/rkhs_norm": 146.13702392578125,
      "regularization/w1": 0.7201792597770691,
      "rewards/chosen": -0.018179986233456208,
      "rewards/margins": 0.09848384731566356,
      "rewards/rejected": -0.11666383354911977,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 27.274593353271484,
      "kl": 16.874574661254883,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35657038.30588235,
      "logits/rejected": -36616512.85333333,
      "logps/chosen": -481.6936580882353,
      "logps/rejected": -367.100703125,
      "loss": 0.5979,
      "loss/base_kto": 3.8359532356262207,
      "metrics/advantage_var": 202.48439025878906,
      "regularization/lipschitz_norm": 964.8781127929688,
      "regularization/mmd": 0.13669872283935547,
      "regularization/rkhs_norm": 162.736572265625,
      "regularization/w1": 0.8104975819587708,
      "rewards/chosen": 0.23281494589412913,
      "rewards/margins": 0.15118915370866365,
      "rewards/rejected": 0.08162579218546549,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 24.10140609741211,
      "kl": 21.570547103881836,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35325692.17910448,
      "logits/rejected": -36794008.76065574,
      "logps/chosen": -473.5366604477612,
      "logps/rejected": -354.75676229508196,
      "loss": 0.5916,
      "loss/base_kto": 3.7875802516937256,
      "metrics/advantage_var": 201.50961303710938,
      "regularization/lipschitz_norm": 960.6087036132812,
      "regularization/mmd": 0.13858595490455627,
      "regularization/rkhs_norm": 164.9832763671875,
      "regularization/w1": 0.8069114089012146,
      "rewards/chosen": 0.3974429002448694,
      "rewards/margins": 0.18101103549077102,
      "rewards/rejected": 0.21643186475409837,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 23.85628890991211,
      "kl": 17.967758178710938,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -37210617.559748426,
      "logits/rejected": -37033786.83229814,
      "logps/chosen": -473.3713639937107,
      "logps/rejected": -363.1606414984472,
      "loss": 0.5902,
      "loss/base_kto": 3.7869834899902344,
      "metrics/advantage_var": 201.8460693359375,
      "regularization/lipschitz_norm": 949.5863647460938,
      "regularization/mmd": 0.13735289871692657,
      "regularization/rkhs_norm": 163.51536560058594,
      "regularization/w1": 0.7976524233818054,
      "rewards/chosen": 0.28570477467662886,
      "rewards/margins": 0.18095441983870184,
      "rewards/rejected": 0.10475035483792702,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 20.733516693115234,
      "kl": 10.516373634338379,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36012218.474097334,
      "logits/rejected": -37267675.76982893,
      "logps/chosen": -471.9190541601256,
      "logps/rejected": -348.91874027993777,
      "loss": 0.5935,
      "loss/base_kto": 3.753783941268921,
      "metrics/advantage_var": 225.03857421875,
      "regularization/lipschitz_norm": 1012.8804931640625,
      "regularization/mmd": 0.14312028884887695,
      "regularization/rkhs_norm": 170.3813018798828,
      "regularization/w1": 0.8508195877075195,
      "rewards/chosen": 0.1454004141000601,
      "rewards/margins": 0.2274882675629082,
      "rewards/rejected": -0.0820878534628481,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 22.378005981445312,
      "kl": 19.50136375427246,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -37822526.62996942,
      "logits/rejected": -37933265.38019169,
      "logps/chosen": -508.591121941896,
      "logps/rejected": -361.4126397763578,
      "loss": 0.5949,
      "loss/base_kto": 3.7431771755218506,
      "metrics/advantage_var": 230.7646484375,
      "regularization/lipschitz_norm": 1031.614501953125,
      "regularization/mmd": 0.1491633653640747,
      "regularization/rkhs_norm": 177.57542419433594,
      "regularization/w1": 0.8665561676025391,
      "rewards/chosen": 0.36003075462597955,
      "rewards/margins": 0.2266431006535854,
      "rewards/rejected": 0.13338765397239416,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 20.391454696655273,
      "kl": 20.642642974853516,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37195620.61305007,
      "logits/rejected": -39158485.53945249,
      "logps/chosen": -496.6452010622155,
      "logps/rejected": -375.17811493558776,
      "loss": 0.5898,
      "loss/base_kto": 3.7457730770111084,
      "metrics/advantage_var": 228.0854949951172,
      "regularization/lipschitz_norm": 986.2400512695312,
      "regularization/mmd": 0.14456598460674286,
      "regularization/rkhs_norm": 172.1023712158203,
      "regularization/w1": 0.8284416198730469,
      "rewards/chosen": 0.4145056763621491,
      "rewards/margins": 0.23657980478798066,
      "rewards/rejected": 0.17792587157416842,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 21.531587600708008,
      "kl": 14.1281099319458,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36354458.874028,
      "logits/rejected": -37320385.70800628,
      "logps/chosen": -474.1392398911353,
      "logps/rejected": -365.2935145211931,
      "loss": 0.5919,
      "loss/base_kto": 3.8021271228790283,
      "metrics/advantage_var": 209.7309112548828,
      "regularization/lipschitz_norm": 947.1066284179688,
      "regularization/mmd": 0.13743311166763306,
      "regularization/rkhs_norm": 163.61083984375,
      "regularization/w1": 0.7955695986747742,
      "rewards/chosen": 0.25722177366075766,
      "rewards/margins": 0.17277734736823042,
      "rewards/rejected": 0.08444442629252723,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 18.736751556396484,
      "kl": 6.519759654998779,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36864680.51653543,
      "logits/rejected": -38052208.32248062,
      "logps/chosen": -488.0936023622047,
      "logps/rejected": -380.5171027131783,
      "loss": 0.5995,
      "loss/base_kto": 3.8403961658477783,
      "metrics/advantage_var": 209.14273071289062,
      "regularization/lipschitz_norm": 970.310546875,
      "regularization/mmd": 0.1402624249458313,
      "regularization/rkhs_norm": 166.9790802001953,
      "regularization/w1": 0.8150609135627747,
      "rewards/chosen": 0.04636677419106791,
      "rewards/margins": 0.1527477277210764,
      "rewards/rejected": -0.10638095353000848,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 24.319786071777344,
      "kl": 9.462249755859375,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -34828143.8208,
      "logits/rejected": -37237870.99847328,
      "logps/chosen": -478.8565,
      "logps/rejected": -384.2642891221374,
      "loss": 0.5974,
      "loss/base_kto": 3.741943359375,
      "metrics/advantage_var": 268.43017578125,
      "regularization/lipschitz_norm": 1053.147705078125,
      "regularization/mmd": 0.15234512090682983,
      "regularization/rkhs_norm": 181.36326599121094,
      "regularization/w1": 0.8846439719200134,
      "rewards/chosen": 0.118431494140625,
      "rewards/margins": 0.21270740985433562,
      "rewards/rejected": -0.09427591571371063,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 23.534250259399414,
      "kl": 11.045422554016113,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36356810.27160494,
      "logits/rejected": -37444384.405063294,
      "logps/chosen": -481.43745177469134,
      "logps/rejected": -390.0705597310127,
      "loss": 0.5972,
      "loss/base_kto": 3.782033681869507,
      "metrics/advantage_var": 228.792724609375,
      "regularization/lipschitz_norm": 1012.0086059570312,
      "regularization/mmd": 0.145858034491539,
      "regularization/rkhs_norm": 173.64051818847656,
      "regularization/w1": 0.8500871658325195,
      "rewards/chosen": 0.21624364970642843,
      "rewards/margins": 0.19835357059443587,
      "rewards/rejected": 0.017890079111992557,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 19.872377395629883,
      "kl": 23.93448829650879,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -38243406.532330826,
      "logits/rejected": -38514947.74634147,
      "logps/chosen": -468.2688439849624,
      "logps/rejected": -386.70543699186993,
      "loss": 0.5965,
      "loss/base_kto": 3.7427849769592285,
      "metrics/advantage_var": 245.9417724609375,
      "regularization/lipschitz_norm": 1043.07763671875,
      "regularization/mmd": 0.15340642631053925,
      "regularization/rkhs_norm": 182.62669372558594,
      "regularization/w1": 0.8761852383613586,
      "rewards/chosen": 0.4155246361753994,
      "rewards/margins": 0.23698629666907955,
      "rewards/rejected": 0.17853833950631987,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 22.400741577148438,
      "kl": 24.455322265625,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -37996048.77379095,
      "logits/rejected": -37879176.61345853,
      "logps/chosen": -507.0338826053042,
      "logps/rejected": -363.8024990219092,
      "loss": 0.5936,
      "loss/base_kto": 3.7253081798553467,
      "metrics/advantage_var": 256.9432067871094,
      "regularization/lipschitz_norm": 1035.1435546875,
      "regularization/mmd": 0.1540052592754364,
      "regularization/rkhs_norm": 183.339599609375,
      "regularization/w1": 0.8695206046104431,
      "rewards/chosen": 0.42766895978573516,
      "rewards/margins": 0.23152582288817794,
      "rewards/rejected": 0.19614313689755722,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 22.976598739624023,
      "kl": 7.938958644866943,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -35306468.324324325,
      "logits/rejected": -36213476.48208469,
      "logps/chosen": -490.83436561561564,
      "logps/rejected": -389.009161237785,
      "loss": 0.6019,
      "loss/base_kto": 3.7460715770721436,
      "metrics/advantage_var": 253.7123565673828,
      "regularization/lipschitz_norm": 1091.4351806640625,
      "regularization/mmd": 0.15219855308532715,
      "regularization/rkhs_norm": 181.18875122070312,
      "regularization/w1": 0.9168054461479187,
      "rewards/chosen": 0.16097977784302858,
      "rewards/margins": 0.25080174554887014,
      "rewards/rejected": -0.08982196770584157,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 20.255399703979492,
      "kl": 13.6774263381958,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -34702110.21118012,
      "logits/rejected": -37870836.72955975,
      "logps/chosen": -495.23325892857144,
      "logps/rejected": -356.2152122641509,
      "loss": 0.5919,
      "loss/base_kto": 3.7645881175994873,
      "metrics/advantage_var": 229.5393829345703,
      "regularization/lipschitz_norm": 985.5276489257812,
      "regularization/mmd": 0.14242756366729736,
      "regularization/rkhs_norm": 169.55662536621094,
      "regularization/w1": 0.8278433084487915,
      "rewards/chosen": 0.24982983133067255,
      "rewards/margins": 0.19894242958191935,
      "rewards/rejected": 0.0508874017487532,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 21.399656295776367,
      "kl": 15.086886405944824,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36604398.45482866,
      "logits/rejected": -38395666.45768025,
      "logps/chosen": -477.8134735202492,
      "logps/rejected": -379.2145131269593,
      "loss": 0.5979,
      "loss/base_kto": 3.763843536376953,
      "metrics/advantage_var": 235.6248016357422,
      "regularization/lipschitz_norm": 1036.3138427734375,
      "regularization/mmd": 0.1487146019935608,
      "regularization/rkhs_norm": 177.0412139892578,
      "regularization/w1": 0.8705036044120789,
      "rewards/chosen": 0.27532802118319216,
      "rewards/margins": 0.23370120455021282,
      "rewards/rejected": 0.04162681663297934,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 23.87718391418457,
      "kl": 25.548213958740234,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -36724739.16049383,
      "logits/rejected": -37107731.44303797,
      "logps/chosen": -483.24546682098764,
      "logps/rejected": -380.68052808544303,
      "loss": 0.6001,
      "loss/base_kto": 3.77905535697937,
      "metrics/advantage_var": 258.1446228027344,
      "regularization/lipschitz_norm": 1038.5205078125,
      "regularization/mmd": 0.14911852777004242,
      "regularization/rkhs_norm": 177.5220489501953,
      "regularization/w1": 0.872357189655304,
      "rewards/chosen": 0.3292643935592086,
      "rewards/margins": 0.15456709703033203,
      "rewards/rejected": 0.17469729652887658,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 21.920976638793945,
      "kl": 12.123173713684082,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -37376329.48895899,
      "logits/rejected": -38672032.099071205,
      "logps/chosen": -485.65605283911674,
      "logps/rejected": -413.11745356037153,
      "loss": 0.6022,
      "loss/base_kto": 3.794027805328369,
      "metrics/advantage_var": 270.994873046875,
      "regularization/lipschitz_norm": 1040.5550537109375,
      "regularization/mmd": 0.14962628483772278,
      "regularization/rkhs_norm": 178.1265411376953,
      "regularization/w1": 0.874066174030304,
      "rewards/chosen": 0.15646629453833547,
      "rewards/margins": 0.1721192389673374,
      "rewards/rejected": -0.01565294442900194,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 19.79609489440918,
      "kl": 8.25532341003418,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -34981651.0681458,
      "logits/rejected": -37423933.929121725,
      "logps/chosen": -502.71374801901743,
      "logps/rejected": -369.4180469953775,
      "loss": 0.5999,
      "loss/base_kto": 3.7118279933929443,
      "metrics/advantage_var": 274.7864685058594,
      "regularization/lipschitz_norm": 1107.482421875,
      "regularization/mmd": 0.15676438808441162,
      "regularization/rkhs_norm": 186.624267578125,
      "regularization/w1": 0.9302851557731628,
      "rewards/chosen": 0.08697658484409049,
      "rewards/margins": 0.281008780747945,
      "rewards/rejected": -0.19403219590385448,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 22.302595138549805,
      "kl": 19.128345489501953,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -38235444.43373494,
      "logits/rejected": -38718304.415584415,
      "logps/chosen": -484.83819653614455,
      "logps/rejected": -376.224609375,
      "loss": 0.6068,
      "loss/base_kto": 3.7533974647521973,
      "metrics/advantage_var": 289.1072692871094,
      "regularization/lipschitz_norm": 1117.0125732421875,
      "regularization/mmd": 0.16270272433757782,
      "regularization/rkhs_norm": 193.69371032714844,
      "regularization/w1": 0.9382905960083008,
      "rewards/chosen": 0.27394630248288077,
      "rewards/margins": 0.20642831593756572,
      "rewards/rejected": 0.06751798654531503,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 19.231454849243164,
      "kl": 14.655878067016602,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -37233006.05238829,
      "logits/rejected": -39298224.88748019,
      "logps/chosen": -507.7286691063174,
      "logps/rejected": -371.590778526149,
      "loss": 0.6054,
      "loss/base_kto": 3.765949249267578,
      "metrics/advantage_var": 252.63059997558594,
      "regularization/lipschitz_norm": 1098.7313232421875,
      "regularization/mmd": 0.154534250497818,
      "regularization/rkhs_norm": 183.96934509277344,
      "regularization/w1": 0.922934353351593,
      "rewards/chosen": 0.2643245938011604,
      "rewards/margins": 0.2025455353114894,
      "rewards/rejected": 0.06177905848967103,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 26.971399307250977,
      "kl": 16.3587703704834,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -37068427.03089431,
      "logits/rejected": -39943499.067669176,
      "logps/chosen": -503.4192581300813,
      "logps/rejected": -344.97246240601504,
      "loss": 0.5875,
      "loss/base_kto": 3.69382905960083,
      "metrics/advantage_var": 247.6055145263672,
      "regularization/lipschitz_norm": 1020.49658203125,
      "regularization/mmd": 0.14904962480068207,
      "regularization/rkhs_norm": 177.4400177001953,
      "regularization/w1": 0.857217013835907,
      "rewards/chosen": 0.28179301440231197,
      "rewards/margins": 0.25828919603375877,
      "rewards/rejected": 0.02350381836855322,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 19.392044067382812,
      "kl": 10.862930297851562,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36843257.35572519,
      "logits/rejected": -37337851.4944,
      "logps/chosen": -469.96450381679387,
      "logps/rejected": -394.654925,
      "loss": 0.5931,
      "loss/base_kto": 3.7576234340667725,
      "metrics/advantage_var": 233.93408203125,
      "regularization/lipschitz_norm": 1001.2546997070312,
      "regularization/mmd": 0.1461522877216339,
      "regularization/rkhs_norm": 173.99082946777344,
      "regularization/w1": 0.8410539031028748,
      "rewards/chosen": 0.11657686888716604,
      "rewards/margins": 0.22866212279341602,
      "rewards/rejected": -0.11208525390625,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 19.202499389648438,
      "kl": 17.498945236206055,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36128863.948303714,
      "logits/rejected": -38197582.620272316,
      "logps/chosen": -482.4040791599354,
      "logps/rejected": -361.5468986384266,
      "loss": 0.5942,
      "loss/base_kto": 3.7312629222869873,
      "metrics/advantage_var": 259.32977294921875,
      "regularization/lipschitz_norm": 1032.8582763671875,
      "regularization/mmd": 0.15501855313777924,
      "regularization/rkhs_norm": 184.54588317871094,
      "regularization/w1": 0.8676008582115173,
      "rewards/chosen": 0.2711190514880351,
      "rewards/margins": 0.22723794824942292,
      "rewards/rejected": 0.04388110323861219,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 20.515392303466797,
      "kl": 8.992721557617188,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -35947380.81553398,
      "logits/rejected": -37067819.311178245,
      "logps/chosen": -481.2120246763754,
      "logps/rejected": -357.7751841012085,
      "loss": 0.5935,
      "loss/base_kto": 3.7053017616271973,
      "metrics/advantage_var": 268.7295837402344,
      "regularization/lipschitz_norm": 1055.6337890625,
      "regularization/mmd": 0.15584371984004974,
      "regularization/rkhs_norm": 185.5282440185547,
      "regularization/w1": 0.8867325186729431,
      "rewards/chosen": 0.16149211007028721,
      "rewards/margins": 0.2782246353915792,
      "rewards/rejected": -0.11673252532129201,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 16.0169734954834,
      "kl": 16.430936813354492,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36692030.62996942,
      "logits/rejected": -37772641.32907348,
      "logps/chosen": -486.86066513761466,
      "logps/rejected": -382.26255491214056,
      "loss": 0.6038,
      "loss/base_kto": 3.7231247425079346,
      "metrics/advantage_var": 280.35968017578125,
      "regularization/lipschitz_norm": 1126.2333984375,
      "regularization/mmd": 0.1615525484085083,
      "regularization/rkhs_norm": 192.324462890625,
      "regularization/w1": 0.9460359811782837,
      "rewards/chosen": 0.22247624761832235,
      "rewards/margins": 0.2452185647449042,
      "rewards/rejected": -0.022742317126581844,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 21.753538131713867,
      "kl": 14.277803421020508,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -36299428.03883495,
      "logits/rejected": -37487204.54380665,
      "logps/chosen": -495.65544093851133,
      "logps/rejected": -384.2934762084592,
      "loss": 0.5961,
      "loss/base_kto": 3.7704811096191406,
      "metrics/advantage_var": 226.02835083007812,
      "regularization/lipschitz_norm": 1014.4932861328125,
      "regularization/mmd": 0.14600291848182678,
      "regularization/rkhs_norm": 173.81300354003906,
      "regularization/w1": 0.8521742820739746,
      "rewards/chosen": 0.21826060767312652,
      "rewards/margins": 0.20829108220625367,
      "rewards/rejected": 0.009969525466872846,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 22.93177604675293,
      "kl": 11.957457542419434,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -36371445.08980213,
      "logits/rejected": -37876290.5682183,
      "logps/chosen": -463.1000761035008,
      "logps/rejected": -372.7319923756019,
      "loss": 0.595,
      "loss/base_kto": 3.7308413982391357,
      "metrics/advantage_var": 244.42764282226562,
      "regularization/lipschitz_norm": 1046.0361328125,
      "regularization/mmd": 0.1500902771949768,
      "regularization/rkhs_norm": 178.67889404296875,
      "regularization/w1": 0.878670334815979,
      "rewards/chosen": 0.22403820392022214,
      "rewards/margins": 0.25638787153824044,
      "rewards/rejected": -0.03234966761801828,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 24.102664947509766,
      "kl": 17.31543731689453,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36464313.82542113,
      "logits/rejected": -37198408.67623605,
      "logps/chosen": -495.7117151607963,
      "logps/rejected": -364.50396232057415,
      "loss": 0.5911,
      "loss/base_kto": 3.725515842437744,
      "metrics/advantage_var": 232.67544555664062,
      "regularization/lipschitz_norm": 1020.2478637695312,
      "regularization/mmd": 0.14654068648815155,
      "regularization/rkhs_norm": 174.45323181152344,
      "regularization/w1": 0.857008159160614,
      "rewards/chosen": 0.2674788610858537,
      "rewards/margins": 0.2269748435708081,
      "rewards/rejected": 0.04050401751504561,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 24.527793884277344,
      "kl": 14.906573295593262,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -36681850.364779875,
      "logits/rejected": -36880317.217391305,
      "logps/chosen": -484.5278105345912,
      "logps/rejected": -365.1701766304348,
      "loss": 0.5986,
      "loss/base_kto": 3.710954427719116,
      "metrics/advantage_var": 260.28436279296875,
      "regularization/lipschitz_norm": 1097.6351318359375,
      "regularization/mmd": 0.15554527938365936,
      "regularization/rkhs_norm": 185.17295837402344,
      "regularization/w1": 0.9220134615898132,
      "rewards/chosen": 0.24328354169737618,
      "rewards/margins": 0.26278783707019104,
      "rewards/rejected": -0.019504295372814867,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 20.30431365966797,
      "kl": 12.334071159362793,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -37734572.60291734,
      "logits/rejected": -38451607.746606335,
      "logps/chosen": -489.3760129659643,
      "logps/rejected": -388.30649038461536,
      "loss": 0.5969,
      "loss/base_kto": 3.74849009513855,
      "metrics/advantage_var": 264.3041687011719,
      "regularization/lipschitz_norm": 1038.9027099609375,
      "regularization/mmd": 0.15424896776676178,
      "regularization/rkhs_norm": 183.62974548339844,
      "regularization/w1": 0.8726783990859985,
      "rewards/chosen": 0.18337045728290874,
      "rewards/margins": 0.23592790627850757,
      "rewards/rejected": -0.05255744899559884,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 20.224668502807617,
      "kl": 12.596447944641113,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -34952888.32,
      "logits/rejected": -38083996.85079365,
      "logps/chosen": -461.76548076923075,
      "logps/rejected": -380.7762648809524,
      "loss": 0.596,
      "loss/base_kto": 3.7214317321777344,
      "metrics/advantage_var": 251.2515411376953,
      "regularization/lipschitz_norm": 1064.1939697265625,
      "regularization/mmd": 0.1525294929742813,
      "regularization/rkhs_norm": 181.58274841308594,
      "regularization/w1": 0.8939228057861328,
      "rewards/chosen": 0.2392426476111779,
      "rewards/margins": 0.22293902507777325,
      "rewards/rejected": 0.016303622533404637,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 15.575621604919434,
      "kl": 12.5536470413208,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36143697.668711655,
      "logits/rejected": -37686258.95541401,
      "logps/chosen": -497.0175421779141,
      "logps/rejected": -362.5546875,
      "loss": 0.6063,
      "loss/base_kto": 3.752284288406372,
      "metrics/advantage_var": 297.0749206542969,
      "regularization/lipschitz_norm": 1117.095947265625,
      "regularization/mmd": 0.15982839465141296,
      "regularization/rkhs_norm": 190.2718963623047,
      "regularization/w1": 0.9383605122566223,
      "rewards/chosen": 0.16839938953610287,
      "rewards/margins": 0.2048152246311622,
      "rewards/rejected": -0.03641583509505934,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 17.124082565307617,
      "kl": 15.74393367767334,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35181453.48460292,
      "logits/rejected": -36652404.223227754,
      "logps/chosen": -499.7367807941653,
      "logps/rejected": -373.7562688536953,
      "loss": 0.6016,
      "loss/base_kto": 3.7982919216156006,
      "metrics/advantage_var": 254.0915069580078,
      "regularization/lipschitz_norm": 1026.367431640625,
      "regularization/mmd": 0.15220218896865845,
      "regularization/rkhs_norm": 181.19308471679688,
      "regularization/w1": 0.8621487021446228,
      "rewards/chosen": 0.17538462040683245,
      "rewards/margins": 0.1723997816190459,
      "rewards/rejected": 0.0029848387877865613,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 17.833799362182617,
      "kl": 4.535059452056885,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -35980446.55483871,
      "logits/rejected": -36960458.31003039,
      "logps/chosen": -518.8235383064516,
      "logps/rejected": -378.08826462765956,
      "loss": 0.6037,
      "loss/base_kto": 3.5379321575164795,
      "metrics/advantage_var": 435.5148620605469,
      "regularization/lipschitz_norm": 1313.1517333984375,
      "regularization/mmd": 0.1887841373682022,
      "regularization/rkhs_norm": 224.74302673339844,
      "regularization/w1": 1.1030473709106445,
      "rewards/chosen": 0.04880948220529864,
      "rewards/margins": 0.5075058881436297,
      "rewards/rejected": -0.4586964059383311,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 21.87369155883789,
      "kl": 10.168845176696777,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36591457.96535433,
      "logits/rejected": -37067441.01705427,
      "logps/chosen": -481.8382874015748,
      "logps/rejected": -377.7716812015504,
      "loss": 0.6138,
      "loss/base_kto": 3.3144783973693848,
      "metrics/advantage_var": 589.0369262695312,
      "regularization/lipschitz_norm": 1633.3592529296875,
      "regularization/mmd": 0.2240116149187088,
      "regularization/rkhs_norm": 266.6805114746094,
      "regularization/w1": 1.3720217943191528,
      "rewards/chosen": 0.3213261641855315,
      "rewards/margins": 0.7587058918078669,
      "rewards/rejected": -0.4373797276223353,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 18.56248664855957,
      "kl": 9.8152437210083,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -35298182.981818184,
      "logits/rejected": -36804135.63870968,
      "logps/chosen": -480.85700757575756,
      "logps/rejected": -368.7930191532258,
      "loss": 0.6239,
      "loss/base_kto": 3.3169960975646973,
      "metrics/advantage_var": 661.1056518554688,
      "regularization/lipschitz_norm": 1710.257080078125,
      "regularization/mmd": 0.2378852665424347,
      "regularization/rkhs_norm": 283.1967468261719,
      "regularization/w1": 1.4366158246994019,
      "rewards/chosen": 0.4595697576349432,
      "rewards/margins": 0.7583705644803314,
      "rewards/rejected": -0.2988008068453881,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 17.172359466552734,
      "kl": 13.928451538085938,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35261304.44657097,
      "logits/rejected": -36561234.71975498,
      "logps/chosen": -478.76305821371614,
      "logps/rejected": -420.7439222817764,
      "loss": 0.5974,
      "loss/base_kto": 3.201690435409546,
      "metrics/advantage_var": 621.3167114257812,
      "regularization/lipschitz_norm": 1605.9063720703125,
      "regularization/mmd": 0.2283145934343338,
      "regularization/rkhs_norm": 271.8031311035156,
      "regularization/w1": 1.3489612340927124,
      "rewards/chosen": 0.48136758271967206,
      "rewards/margins": 0.8656706771366658,
      "rewards/rejected": -0.3843030944169937,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 16.447967529296875,
      "kl": 9.176518440246582,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -35890979.74959872,
      "logits/rejected": -36695830.210045666,
      "logps/chosen": -460.6535413322632,
      "logps/rejected": -381.8759512937595,
      "loss": 0.6182,
      "loss/base_kto": 3.335076093673706,
      "metrics/advantage_var": 640.2711791992188,
      "regularization/lipschitz_norm": 1644.6497802734375,
      "regularization/mmd": 0.22922053933143616,
      "regularization/rkhs_norm": 272.8816223144531,
      "regularization/w1": 1.3815058469772339,
      "rewards/chosen": 0.3886109867983798,
      "rewards/margins": 0.7655763048946627,
      "rewards/rejected": -0.3769653180962828,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 23.55685043334961,
      "kl": 12.506660461425781,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -36198894.98489426,
      "logits/rejected": -37032605.41100324,
      "logps/chosen": -475.5877549093656,
      "logps/rejected": -378.5860891990291,
      "loss": 0.6044,
      "loss/base_kto": 3.284658908843994,
      "metrics/advantage_var": 581.3097534179688,
      "regularization/lipschitz_norm": 1581.3773193359375,
      "regularization/mmd": 0.2217867225408554,
      "regularization/rkhs_norm": 264.0318298339844,
      "regularization/w1": 1.3283569812774658,
      "rewards/chosen": 0.45500657925793053,
      "rewards/margins": 0.8361634512634549,
      "rewards/rejected": -0.3811568720055244,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 15.991052627563477,
      "kl": 21.35040283203125,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -35681537.18335901,
      "logits/rejected": -36956020.43740095,
      "logps/chosen": -478.8996533127889,
      "logps/rejected": -365.395750792393,
      "loss": 0.6029,
      "loss/base_kto": 3.2986061573028564,
      "metrics/advantage_var": 582.2473754882812,
      "regularization/lipschitz_norm": 1549.0313720703125,
      "regularization/mmd": 0.22335056960582733,
      "regularization/rkhs_norm": 265.8935241699219,
      "regularization/w1": 1.301186442375183,
      "rewards/chosen": 0.6604523034235362,
      "rewards/margins": 0.7518704835718468,
      "rewards/rejected": -0.09141818014831059,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 21.807615280151367,
      "kl": 18.00168800354004,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -36559937.9089482,
      "logits/rejected": -38227885.1881804,
      "logps/chosen": -484.48910910518055,
      "logps/rejected": -371.1508067651633,
      "loss": 0.6135,
      "loss/base_kto": 3.2756271362304688,
      "metrics/advantage_var": 726.1383056640625,
      "regularization/lipschitz_norm": 1659.885986328125,
      "regularization/mmd": 0.23783467710018158,
      "regularization/rkhs_norm": 283.13653564453125,
      "regularization/w1": 1.3943042755126953,
      "rewards/chosen": 0.5675269206240189,
      "rewards/margins": 0.7931678657026299,
      "rewards/rejected": -0.22564094507861102,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 21.79051399230957,
      "kl": 13.512042045593262,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35249636.54258675,
      "logits/rejected": -35610585.956656344,
      "logps/chosen": -491.02070189274446,
      "logps/rejected": -375.1051905959752,
      "loss": 0.6036,
      "loss/base_kto": 3.2885327339172363,
      "metrics/advantage_var": 611.0409545898438,
      "regularization/lipschitz_norm": 1565.328125,
      "regularization/mmd": 0.22567521035671234,
      "regularization/rkhs_norm": 268.66094970703125,
      "regularization/w1": 1.314875602722168,
      "rewards/chosen": 0.4869826644753056,
      "rewards/margins": 0.7988680376174181,
      "rewards/rejected": -0.3118853731421125,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 13.28946304321289,
      "kl": 14.684597969055176,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36197448.112676054,
      "logits/rejected": -36856261.691107646,
      "logps/chosen": -447.47378716744913,
      "logps/rejected": -371.0314937597504,
      "loss": 0.5862,
      "loss/base_kto": 3.239898681640625,
      "metrics/advantage_var": 551.99609375,
      "regularization/lipschitz_norm": 1472.745849609375,
      "regularization/mmd": 0.21260903775691986,
      "regularization/rkhs_norm": 253.1060028076172,
      "regularization/w1": 1.2371065616607666,
      "rewards/chosen": 0.4805610191094484,
      "rewards/margins": 0.8032315138198618,
      "rewards/rejected": -0.3226704947104134,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 19.637243270874023,
      "kl": 9.189848899841309,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -34937800.20512821,
      "logits/rejected": -36114091.70731708,
      "logps/chosen": -498.53881209935895,
      "logps/rejected": -369.2965415396341,
      "loss": 0.6079,
      "loss/base_kto": 3.259673833847046,
      "metrics/advantage_var": 614.0714721679688,
      "regularization/lipschitz_norm": 1637.0792236328125,
      "regularization/mmd": 0.2280350774526596,
      "regularization/rkhs_norm": 271.4703063964844,
      "regularization/w1": 1.3751466274261475,
      "rewards/chosen": 0.4148457845052083,
      "rewards/margins": 0.8178668758733485,
      "rewards/rejected": -0.40302109136814024,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 15.831979751586914,
      "kl": 15.795271873474121,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -36102406.87761194,
      "logits/rejected": -37408576.6295082,
      "logps/chosen": -496.81539179104476,
      "logps/rejected": -389.46357581967214,
      "loss": 0.6159,
      "loss/base_kto": 3.332146406173706,
      "metrics/advantage_var": 589.1945190429688,
      "regularization/lipschitz_norm": 1625.651123046875,
      "regularization/mmd": 0.2293902486562729,
      "regularization/rkhs_norm": 273.0836181640625,
      "regularization/w1": 1.3655469417572021,
      "rewards/chosen": 0.4604987756529851,
      "rewards/margins": 0.6943137780943913,
      "rewards/rejected": -0.23381500244140624,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 18.846372604370117,
      "kl": 6.967624187469482,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -34727523.863777086,
      "logits/rejected": -36741740.214511044,
      "logps/chosen": -470.32594814241486,
      "logps/rejected": -366.38194992113563,
      "loss": 0.6102,
      "loss/base_kto": 3.3769073486328125,
      "metrics/advantage_var": 544.1055297851562,
      "regularization/lipschitz_norm": 1533.817626953125,
      "regularization/mmd": 0.21603214740753174,
      "regularization/rkhs_norm": 257.18115234375,
      "regularization/w1": 1.2884068489074707,
      "rewards/chosen": 0.34355999736963044,
      "rewards/margins": 0.7039021254133015,
      "rewards/rejected": -0.36034212804367116,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 16.535308837890625,
      "kl": 11.569666862487793,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35818812.50909091,
      "logits/rejected": -35605583.27741936,
      "logps/chosen": -502.7956439393939,
      "logps/rejected": -366.6118699596774,
      "loss": 0.6022,
      "loss/base_kto": 3.2230117321014404,
      "metrics/advantage_var": 632.7213745117188,
      "regularization/lipschitz_norm": 1622.147705078125,
      "regularization/mmd": 0.23166914284229279,
      "regularization/rkhs_norm": 275.7966003417969,
      "regularization/w1": 1.362604022026062,
      "rewards/chosen": 0.5035967740145597,
      "rewards/margins": 0.8731916187096203,
      "rewards/rejected": -0.3695948446950605,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 17.559795379638672,
      "kl": 17.243452072143555,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35458056.03767661,
      "logits/rejected": -37407341.08864697,
      "logps/chosen": -506.42602040816325,
      "logps/rejected": -372.56738433125975,
      "loss": 0.6172,
      "loss/base_kto": 3.2942306995391846,
      "metrics/advantage_var": 618.4035034179688,
      "regularization/lipschitz_norm": 1680.7093505859375,
      "regularization/mmd": 0.2311849147081375,
      "regularization/rkhs_norm": 275.2201843261719,
      "regularization/w1": 1.411795973777771,
      "rewards/chosen": 0.49288413438542483,
      "rewards/margins": 0.7521722043759478,
      "rewards/rejected": -0.25928806999052295,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 21.55283546447754,
      "kl": 18.124265670776367,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -34883935.357032456,
      "logits/rejected": -36409874.60347551,
      "logps/chosen": -470.66190108191654,
      "logps/rejected": -372.9338467614534,
      "loss": 0.604,
      "loss/base_kto": 3.367394208908081,
      "metrics/advantage_var": 511.4122009277344,
      "regularization/lipschitz_norm": 1487.292236328125,
      "regularization/mmd": 0.21556456387043,
      "regularization/rkhs_norm": 256.6244812011719,
      "regularization/w1": 1.2493253946304321,
      "rewards/chosen": 0.5155438241855439,
      "rewards/margins": 0.6709385847271736,
      "rewards/rejected": -0.15539476054162965,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 17.61018180847168,
      "kl": 11.695985794067383,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34725701.2327044,
      "logits/rejected": -35829683.677018635,
      "logps/chosen": -466.3694968553459,
      "logps/rejected": -373.62533967391306,
      "loss": 0.602,
      "loss/base_kto": 3.3153557777404785,
      "metrics/advantage_var": 524.4066772460938,
      "regularization/lipschitz_norm": 1529.5323486328125,
      "regularization/mmd": 0.21546566486358643,
      "regularization/rkhs_norm": 256.5067443847656,
      "regularization/w1": 1.2848070859909058,
      "rewards/chosen": 0.5077054965421088,
      "rewards/margins": 0.7631978550163664,
      "rewards/rejected": -0.25549235847425755,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 17.635663986206055,
      "kl": 9.056909561157227,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34020453.59370079,
      "logits/rejected": -35876978.30697674,
      "logps/chosen": -477.1708661417323,
      "logps/rejected": -380.3998546511628,
      "loss": 0.6159,
      "loss/base_kto": 3.348447561264038,
      "metrics/advantage_var": 632.1333618164062,
      "regularization/lipschitz_norm": 1609.167724609375,
      "regularization/mmd": 0.22681903839111328,
      "regularization/rkhs_norm": 270.0226745605469,
      "regularization/w1": 1.351700782775879,
      "rewards/chosen": 0.30686888206662155,
      "rewards/margins": 0.7083278588713696,
      "rewards/rejected": -0.40145897680474807,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 18.189739227294922,
      "kl": 10.588912010192871,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34492528.855753645,
      "logits/rejected": -37875242.47360483,
      "logps/chosen": -480.3084987844408,
      "logps/rejected": -364.04859539969834,
      "loss": 0.6032,
      "loss/base_kto": 3.29600191116333,
      "metrics/advantage_var": 570.6554565429688,
      "regularization/lipschitz_norm": 1555.6051025390625,
      "regularization/mmd": 0.22321109473705292,
      "regularization/rkhs_norm": 265.7275085449219,
      "regularization/w1": 1.3067082166671753,
      "rewards/chosen": 0.4437416113756078,
      "rewards/margins": 0.7693087964452486,
      "rewards/rejected": -0.32556718506964083,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 18.128437042236328,
      "kl": 10.108545303344727,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34681704.97244733,
      "logits/rejected": -35432420.199095026,
      "logps/chosen": -490.55287682333875,
      "logps/rejected": -372.83887160633486,
      "loss": 0.6072,
      "loss/base_kto": 3.3541133403778076,
      "metrics/advantage_var": 590.4301147460938,
      "regularization/lipschitz_norm": 1529.4583740234375,
      "regularization/mmd": 0.21881555020809174,
      "regularization/rkhs_norm": 260.4947204589844,
      "regularization/w1": 1.2847449779510498,
      "rewards/chosen": 0.3427481628121201,
      "rewards/margins": 0.7054281520817162,
      "rewards/rejected": -0.36267998926959605,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 17.019926071166992,
      "kl": 16.55487060546875,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34213762.35582822,
      "logits/rejected": -36385994.191082805,
      "logps/chosen": -488.32563266871165,
      "logps/rejected": -383.2771198248408,
      "loss": 0.6231,
      "loss/base_kto": 3.2450668811798096,
      "metrics/advantage_var": 761.9019775390625,
      "regularization/lipschitz_norm": 1777.282470703125,
      "regularization/mmd": 0.24660912156105042,
      "regularization/rkhs_norm": 293.582275390625,
      "regularization/w1": 1.4929174184799194,
      "rewards/chosen": 0.5532671220463478,
      "rewards/margins": 0.8092904561364652,
      "rewards/rejected": -0.25602333409011746,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 16.47247886657715,
      "kl": 11.24480152130127,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34316371.330203444,
      "logits/rejected": -35128161.847113885,
      "logps/chosen": -472.56685250391234,
      "logps/rejected": -385.1658053822153,
      "loss": 0.6102,
      "loss/base_kto": 3.301698684692383,
      "metrics/advantage_var": 624.9652709960938,
      "regularization/lipschitz_norm": 1611.0435791015625,
      "regularization/mmd": 0.22677765786647797,
      "regularization/rkhs_norm": 269.973388671875,
      "regularization/w1": 1.3532766103744507,
      "rewards/chosen": 0.4035864219605829,
      "rewards/margins": 0.7785279815126134,
      "rewards/rejected": -0.3749415595520305,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 15.375174522399902,
      "kl": 12.943961143493652,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -35163236.13880126,
      "logits/rejected": -35130776.965944275,
      "logps/chosen": -472.6810922712934,
      "logps/rejected": -411.40097716718265,
      "loss": 0.6017,
      "loss/base_kto": 3.2502992153167725,
      "metrics/advantage_var": 719.5261840820312,
      "regularization/lipschitz_norm": 1584.6982421875,
      "regularization/mmd": 0.23204405605793,
      "regularization/rkhs_norm": 276.242919921875,
      "regularization/w1": 1.3311465978622437,
      "rewards/chosen": 0.4705423060275779,
      "rewards/margins": 0.8415792016850494,
      "rewards/rejected": -0.37103689565747144,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 16.864608764648438,
      "kl": 7.824089050292969,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34898280.013266996,
      "logits/rejected": -36188247.72821271,
      "logps/chosen": -496.92283374792703,
      "logps/rejected": -352.1043436115214,
      "loss": 0.6152,
      "loss/base_kto": 3.3539161682128906,
      "metrics/advantage_var": 675.998046875,
      "regularization/lipschitz_norm": 1597.838623046875,
      "regularization/mmd": 0.22515200078487396,
      "regularization/rkhs_norm": 268.0380859375,
      "regularization/w1": 1.3421844244003296,
      "rewards/chosen": 0.2688575099356732,
      "rewards/margins": 0.7027325445553187,
      "rewards/rejected": -0.4338750346196455,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 18.610519409179688,
      "kl": 6.846913814544678,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -33535580.455235206,
      "logits/rejected": -32836286.45410628,
      "logps/chosen": -507.3798842943854,
      "logps/rejected": -412.5739231078905,
      "loss": 0.6081,
      "loss/base_kto": 3.2501351833343506,
      "metrics/advantage_var": 642.9050903320312,
      "regularization/lipschitz_norm": 1646.7279052734375,
      "regularization/mmd": 0.23111307621002197,
      "regularization/rkhs_norm": 275.1346130371094,
      "regularization/w1": 1.383251428604126,
      "rewards/chosen": 0.3652233071681762,
      "rewards/margins": 0.8627621455777474,
      "rewards/rejected": -0.49753883840957125,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 17.609745025634766,
      "kl": 11.589434623718262,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34552121.83254345,
      "logits/rejected": -35316326.083462134,
      "logps/chosen": -495.454285150079,
      "logps/rejected": -384.5259611669243,
      "loss": 0.6162,
      "loss/base_kto": 3.3512799739837646,
      "metrics/advantage_var": 615.2280883789062,
      "regularization/lipschitz_norm": 1615.4227294921875,
      "regularization/mmd": 0.22117102146148682,
      "regularization/rkhs_norm": 263.298828125,
      "regularization/w1": 1.3569550514221191,
      "rewards/chosen": 0.3657852100535027,
      "rewards/margins": 0.6919076531804347,
      "rewards/rejected": -0.326122443126932,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 13.600857734680176,
      "kl": 10.581927299499512,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33810810.56969697,
      "logits/rejected": -36186072.36129032,
      "logps/chosen": -499.5164772727273,
      "logps/rejected": -359.4179435483871,
      "loss": 0.5974,
      "loss/base_kto": 3.3308751583099365,
      "metrics/advantage_var": 489.8287658691406,
      "regularization/lipschitz_norm": 1476.373291015625,
      "regularization/mmd": 0.2084968090057373,
      "regularization/rkhs_norm": 248.2104949951172,
      "regularization/w1": 1.2401536703109741,
      "rewards/chosen": 0.42881303267045456,
      "rewards/margins": 0.7129002322199758,
      "rewards/rejected": -0.28408719954952116,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 17.946428298950195,
      "kl": 16.50902557373047,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -31569818.55108359,
      "logits/rejected": -35038372.74447949,
      "logps/chosen": -502.62393575851394,
      "logps/rejected": -358.6391462933754,
      "loss": 0.6113,
      "loss/base_kto": 3.308062791824341,
      "metrics/advantage_var": 860.5252075195312,
      "regularization/lipschitz_norm": 1611.06982421875,
      "regularization/mmd": 0.22933049499988556,
      "regularization/rkhs_norm": 273.01251220703125,
      "regularization/w1": 1.353298544883728,
      "rewards/chosen": 0.5563234028063322,
      "rewards/margins": 0.7589931662209739,
      "rewards/rejected": -0.20266976341464166,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 14.805122375488281,
      "kl": 13.904166221618652,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -32885823.015384614,
      "logits/rejected": -35195933.25714286,
      "logps/chosen": -487.61908653846154,
      "logps/rejected": -365.45689484126984,
      "loss": 0.6026,
      "loss/base_kto": 3.298060655593872,
      "metrics/advantage_var": 564.461181640625,
      "regularization/lipschitz_norm": 1547.9945068359375,
      "regularization/mmd": 0.2226027101278305,
      "regularization/rkhs_norm": 265.00323486328125,
      "regularization/w1": 1.300315260887146,
      "rewards/chosen": 0.534131845327524,
      "rewards/margins": 0.7471982128250438,
      "rewards/rejected": -0.21306636749751984,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 14.087535858154297,
      "kl": 15.207712173461914,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -34511047.962204725,
      "logits/rejected": -35748240.074418604,
      "logps/chosen": -475.345374015748,
      "logps/rejected": -374.0406492248062,
      "loss": 0.5866,
      "loss/base_kto": 3.3016624450683594,
      "metrics/advantage_var": 463.8214416503906,
      "regularization/lipschitz_norm": 1413.26171875,
      "regularization/mmd": 0.2043001651763916,
      "regularization/rkhs_norm": 243.2144775390625,
      "regularization/w1": 1.187139868736267,
      "rewards/chosen": 0.4673630121186024,
      "rewards/margins": 0.7458367168484958,
      "rewards/rejected": -0.2784737047298934,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 22.939298629760742,
      "kl": 10.584595680236816,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34324027.015576325,
      "logits/rejected": -34701870.54545455,
      "logps/chosen": -486.6362441588785,
      "logps/rejected": -360.1907327586207,
      "loss": 0.595,
      "loss/base_kto": 3.322956085205078,
      "metrics/advantage_var": 530.6657104492188,
      "regularization/lipschitz_norm": 1459.2164306640625,
      "regularization/mmd": 0.21153569221496582,
      "regularization/rkhs_norm": 251.82823181152344,
      "regularization/w1": 1.2257417440414429,
      "rewards/chosen": 0.36884842631972836,
      "rewards/margins": 0.7581468205019875,
      "rewards/rejected": -0.38929839418225903,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 14.420248031616211,
      "kl": 11.56763744354248,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34694611.26213592,
      "logits/rejected": -35103759.468277946,
      "logps/chosen": -484.4076152912621,
      "logps/rejected": -388.3403984138973,
      "loss": 0.5901,
      "loss/base_kto": 3.2739503383636475,
      "metrics/advantage_var": 488.2797546386719,
      "regularization/lipschitz_norm": 1472.9439697265625,
      "regularization/mmd": 0.2099769562482834,
      "regularization/rkhs_norm": 249.97256469726562,
      "regularization/w1": 1.23727285861969,
      "rewards/chosen": 0.4295048883431938,
      "rewards/margins": 0.7899368245553704,
      "rewards/rejected": -0.3604319362121766,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 15.317728042602539,
      "kl": 11.60511302947998,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -33141669.78670788,
      "logits/rejected": -32971547.90521327,
      "logps/chosen": -456.81196870170015,
      "logps/rejected": -382.5088368878357,
      "loss": 0.5879,
      "loss/base_kto": 3.2631123065948486,
      "metrics/advantage_var": 560.9888916015625,
      "regularization/lipschitz_norm": 1459.6904296875,
      "regularization/mmd": 0.21361985802650452,
      "regularization/rkhs_norm": 254.3093719482422,
      "regularization/w1": 1.2261399030685425,
      "rewards/chosen": 0.3788637046283327,
      "rewards/margins": 0.8087955380974283,
      "rewards/rejected": -0.4299318334690956,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 25.00137710571289,
      "kl": 15.943984985351562,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34763131.37833595,
      "logits/rejected": -36049622.19595645,
      "logps/chosen": -491.8460557299843,
      "logps/rejected": -377.21875,
      "loss": 0.5996,
      "loss/base_kto": 3.297459125518799,
      "metrics/advantage_var": 555.6985473632812,
      "regularization/lipschitz_norm": 1525.255126953125,
      "regularization/mmd": 0.21810658276081085,
      "regularization/rkhs_norm": 259.6507263183594,
      "regularization/w1": 1.2812143564224243,
      "rewards/chosen": 0.5720050039246468,
      "rewards/margins": 0.7381607107782019,
      "rewards/rejected": -0.1661557068535551,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 16.53514862060547,
      "kl": 11.54581356048584,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34169040.0,
      "logits/rejected": -35373100.8,
      "logps/chosen": -479.863671875,
      "logps/rejected": -374.3803466796875,
      "loss": 0.604,
      "loss/base_kto": 3.3393285274505615,
      "metrics/advantage_var": 592.9227905273438,
      "regularization/lipschitz_norm": 1516.8388671875,
      "regularization/mmd": 0.2185019999742508,
      "regularization/rkhs_norm": 260.1214294433594,
      "regularization/w1": 1.2741446495056152,
      "rewards/chosen": 0.3619662284851074,
      "rewards/margins": 0.6888193607330322,
      "rewards/rejected": -0.3268531322479248,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 19.912586212158203,
      "kl": 13.483344078063965,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -35517263.01234568,
      "logits/rejected": -35183846.07594936,
      "logps/chosen": -463.275993441358,
      "logps/rejected": -370.4110957278481,
      "loss": 0.5966,
      "loss/base_kto": 3.306652069091797,
      "metrics/advantage_var": 530.9874877929688,
      "regularization/lipschitz_norm": 1489.7181396484375,
      "regularization/mmd": 0.2145804613828659,
      "regularization/rkhs_norm": 255.4529266357422,
      "regularization/w1": 1.2513631582260132,
      "rewards/chosen": 0.4715083086932147,
      "rewards/margins": 0.7539580551269074,
      "rewards/rejected": -0.28244974643369264,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 15.422378540039062,
      "kl": 16.968021392822266,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -33524504.975609757,
      "logits/rejected": -34687527.384615384,
      "logps/chosen": -473.8185022865854,
      "logps/rejected": -375.02183493589746,
      "loss": 0.5938,
      "loss/base_kto": 3.32004714012146,
      "metrics/advantage_var": 483.416015625,
      "regularization/lipschitz_norm": 1461.2935791015625,
      "regularization/mmd": 0.20300164818763733,
      "regularization/rkhs_norm": 241.66860961914062,
      "regularization/w1": 1.2274866104125977,
      "rewards/chosen": 0.5240411991026344,
      "rewards/margins": 0.7307511831239435,
      "rewards/rejected": -0.2067099840213091,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 16.63542366027832,
      "kl": 13.394705772399902,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -34061198.222222224,
      "logits/rejected": -36468310.23975721,
      "logps/chosen": -497.9873691626409,
      "logps/rejected": -356.5102902124431,
      "loss": 0.5982,
      "loss/base_kto": 3.2797813415527344,
      "metrics/advantage_var": 549.18701171875,
      "regularization/lipschitz_norm": 1534.291015625,
      "regularization/mmd": 0.21697306632995605,
      "regularization/rkhs_norm": 258.30126953125,
      "regularization/w1": 1.2888044118881226,
      "rewards/chosen": 0.46995792603915054,
      "rewards/margins": 0.7730949802509652,
      "rewards/rejected": -0.30313705421181475,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 16.30461883544922,
      "kl": 11.491045951843262,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -33323437.827160493,
      "logits/rejected": -34646132.658227846,
      "logps/chosen": -442.68113425925924,
      "logps/rejected": -374.8646904667722,
      "loss": 0.6121,
      "loss/base_kto": 3.3129944801330566,
      "metrics/advantage_var": 655.145751953125,
      "regularization/lipschitz_norm": 1613.0155029296875,
      "regularization/mmd": 0.22887134552001953,
      "regularization/rkhs_norm": 272.4659118652344,
      "regularization/w1": 1.3549330234527588,
      "rewards/chosen": 0.42668396749614196,
      "rewards/margins": 0.7510802457809149,
      "rewards/rejected": -0.324396278284773,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 16.893821716308594,
      "kl": 13.179420471191406,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34527845.732899025,
      "logits/rejected": -34666079.61445783,
      "logps/chosen": -450.8351995114007,
      "logps/rejected": -354.81240587349396,
      "loss": 0.5771,
      "loss/base_kto": 3.2603137493133545,
      "metrics/advantage_var": 471.96337890625,
      "regularization/lipschitz_norm": 1374.1279296875,
      "regularization/mmd": 0.20227418839931488,
      "regularization/rkhs_norm": 240.80259704589844,
      "regularization/w1": 1.1542673110961914,
      "rewards/chosen": 0.48127850958112783,
      "rewards/margins": 0.7877240564781887,
      "rewards/rejected": -0.3064455468970609,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 17.633975982666016,
      "kl": 9.367804527282715,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34220206.63565891,
      "logits/rejected": -35114175.8992126,
      "logps/chosen": -482.91913759689925,
      "logps/rejected": -387.0054625984252,
      "loss": 0.568,
      "loss/base_kto": 3.1954779624938965,
      "metrics/advantage_var": 469.1932678222656,
      "regularization/lipschitz_norm": 1364.8748779296875,
      "regularization/mmd": 0.20240406692028046,
      "regularization/rkhs_norm": 240.9572296142578,
      "regularization/w1": 1.1464948654174805,
      "rewards/chosen": 0.40537838011749033,
      "rewards/margins": 0.8599942719420475,
      "rewards/rejected": -0.4546158918245571,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 15.641488075256348,
      "kl": 12.354438781738281,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -34263179.78412698,
      "logits/rejected": -36049595.470769234,
      "logps/chosen": -486.7924107142857,
      "logps/rejected": -356.91990384615383,
      "loss": 0.5675,
      "loss/base_kto": 3.1480367183685303,
      "metrics/advantage_var": 484.9521484375,
      "regularization/lipschitz_norm": 1406.8016357421875,
      "regularization/mmd": 0.20991837978363037,
      "regularization/rkhs_norm": 249.90283203125,
      "regularization/w1": 1.181713342666626,
      "rewards/chosen": 0.5818535698784723,
      "rewards/margins": 0.9040121955545539,
      "rewards/rejected": -0.3221586256760817,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 13.56850528717041,
      "kl": 6.684207439422607,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34272953.965996906,
      "logits/rejected": -35289689.78199052,
      "logps/chosen": -466.5400888717156,
      "logps/rejected": -386.13534261453395,
      "loss": 0.5643,
      "loss/base_kto": 3.2365081310272217,
      "metrics/advantage_var": 398.34033203125,
      "regularization/lipschitz_norm": 1295.4384765625,
      "regularization/mmd": 0.19008252024650574,
      "regularization/rkhs_norm": 226.2886962890625,
      "regularization/w1": 1.0881683826446533,
      "rewards/chosen": 0.3401842544767919,
      "rewards/margins": 0.8140668965026114,
      "rewards/rejected": -0.47388264202581953,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 14.188789367675781,
      "kl": 9.632389068603516,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34290703.28358209,
      "logits/rejected": -35077727.6852459,
      "logps/chosen": -487.1331623134328,
      "logps/rejected": -387.50548155737704,
      "loss": 0.5685,
      "loss/base_kto": 3.240670919418335,
      "metrics/advantage_var": 411.7604064941406,
      "regularization/lipschitz_norm": 1329.864990234375,
      "regularization/mmd": 0.19018827378749847,
      "regularization/rkhs_norm": 226.4146270751953,
      "regularization/w1": 1.11708664894104,
      "rewards/chosen": 0.41139394276177704,
      "rewards/margins": 0.816758682852069,
      "rewards/rejected": -0.405364740090292,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 12.198801040649414,
      "kl": 17.952430725097656,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -32834949.31629393,
      "logits/rejected": -35845780.74617737,
      "logps/chosen": -466.93140974440894,
      "logps/rejected": -365.73184250764524,
      "loss": 0.5581,
      "loss/base_kto": 3.1707189083099365,
      "metrics/advantage_var": 428.1657409667969,
      "regularization/lipschitz_norm": 1308.3392333984375,
      "regularization/mmd": 0.195432648062706,
      "regularization/rkhs_norm": 232.65794372558594,
      "regularization/w1": 1.0990049839019775,
      "rewards/chosen": 0.5989690908608726,
      "rewards/margins": 0.8394078496409758,
      "rewards/rejected": -0.2404387587801032,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 18.00127410888672,
      "kl": 12.2958402633667,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34211327.24148148,
      "logits/rejected": -34209643.054545455,
      "logps/chosen": -487.9811111111111,
      "logps/rejected": -378.4548811983471,
      "loss": 0.5609,
      "loss/base_kto": 3.2125489711761475,
      "metrics/advantage_var": 404.812744140625,
      "regularization/lipschitz_norm": 1289.4254150390625,
      "regularization/mmd": 0.19118943810462952,
      "regularization/rkhs_norm": 227.60647583007812,
      "regularization/w1": 1.0831173658370972,
      "rewards/chosen": 0.5194717520254629,
      "rewards/margins": 0.811769594508353,
      "rewards/rejected": -0.29229784248289,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 18.6423282623291,
      "kl": 17.075681686401367,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34602311.29113924,
      "logits/rejected": -35104764.83950617,
      "logps/chosen": -485.4895174050633,
      "logps/rejected": -370.2487461419753,
      "loss": 0.5543,
      "loss/base_kto": 3.145045280456543,
      "metrics/advantage_var": 455.00244140625,
      "regularization/lipschitz_norm": 1302.3179931640625,
      "regularization/mmd": 0.19558389484882355,
      "regularization/rkhs_norm": 232.8379364013672,
      "regularization/w1": 1.0939470529556274,
      "rewards/chosen": 0.5698854470554786,
      "rewards/margins": 0.8855800440878137,
      "rewards/rejected": -0.31569459703233504,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 19.949146270751953,
      "kl": 12.951250076293945,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34040358.25755167,
      "logits/rejected": -36859237.06298003,
      "logps/chosen": -496.67299284578695,
      "logps/rejected": -351.168946812596,
      "loss": 0.5526,
      "loss/base_kto": 3.169118642807007,
      "metrics/advantage_var": 390.81341552734375,
      "regularization/lipschitz_norm": 1265.3895263671875,
      "regularization/mmd": 0.18876340985298157,
      "regularization/rkhs_norm": 224.7183380126953,
      "regularization/w1": 1.06292724609375,
      "rewards/chosen": 0.5121310526691674,
      "rewards/margins": 0.8567477331367173,
      "rewards/rejected": -0.3446166804675499,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 17.865863800048828,
      "kl": 13.589856147766113,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -33942589.15773354,
      "logits/rejected": -34251989.94577353,
      "logps/chosen": -480.28589203675347,
      "logps/rejected": -387.3678478867624,
      "loss": 0.5584,
      "loss/base_kto": 3.1252965927124023,
      "metrics/advantage_var": 466.70489501953125,
      "regularization/lipschitz_norm": 1357.9537353515625,
      "regularization/mmd": 0.2011280059814453,
      "regularization/rkhs_norm": 239.43809509277344,
      "regularization/w1": 1.1406811475753784,
      "rewards/chosen": 0.5978173902900077,
      "rewards/margins": 0.926751617170241,
      "rewards/rejected": -0.32893422688023327,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 16.44749641418457,
      "kl": 16.37274169921875,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -33290848.0,
      "logits/rejected": -34991280.0,
      "logps/chosen": -503.111181640625,
      "logps/rejected": -400.1039794921875,
      "loss": 0.5648,
      "loss/base_kto": 3.1718239784240723,
      "metrics/advantage_var": 457.9571228027344,
      "regularization/lipschitz_norm": 1362.3905029296875,
      "regularization/mmd": 0.2020179033279419,
      "regularization/rkhs_norm": 240.4975128173828,
      "regularization/w1": 1.1444079875946045,
      "rewards/chosen": 0.5759394645690918,
      "rewards/margins": 0.86002836227417,
      "rewards/rejected": -0.2840888977050781,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 19.371198654174805,
      "kl": 12.099383354187012,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34977602.85350318,
      "logits/rejected": -35392822.969325155,
      "logps/chosen": -496.6204219745223,
      "logps/rejected": -370.08934049079755,
      "loss": 0.5502,
      "loss/base_kto": 3.127540349960327,
      "metrics/advantage_var": 432.763916015625,
      "regularization/lipschitz_norm": 1285.8685302734375,
      "regularization/mmd": 0.19408564269542694,
      "regularization/rkhs_norm": 231.05435180664062,
      "regularization/w1": 1.080129623413086,
      "rewards/chosen": 0.5249679468239948,
      "rewards/margins": 0.904853637899232,
      "rewards/rejected": -0.37988569107523723,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 14.959543228149414,
      "kl": 12.483757019042969,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -33898518.5408805,
      "logits/rejected": -35877837.11801242,
      "logps/chosen": -480.4602004716981,
      "logps/rejected": -372.8230298913044,
      "loss": 0.5484,
      "loss/base_kto": 3.1473729610443115,
      "metrics/advantage_var": 383.9928894042969,
      "regularization/lipschitz_norm": 1251.9901123046875,
      "regularization/mmd": 0.18830907344818115,
      "regularization/rkhs_norm": 224.17747497558594,
      "regularization/w1": 1.0516717433929443,
      "rewards/chosen": 0.539827694682955,
      "rewards/margins": 0.878365112266539,
      "rewards/rejected": -0.338537417583584,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 16.395963668823242,
      "kl": 12.685641288757324,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -33862333.59869494,
      "logits/rejected": -35913586.75862069,
      "logps/chosen": -463.567088091354,
      "logps/rejected": -380.15744471514245,
      "loss": 0.552,
      "loss/base_kto": 3.1507604122161865,
      "metrics/advantage_var": 395.13922119140625,
      "regularization/lipschitz_norm": 1282.7816162109375,
      "regularization/mmd": 0.1873190850019455,
      "regularization/rkhs_norm": 222.99893188476562,
      "regularization/w1": 1.0775364637374878,
      "rewards/chosen": 0.4674279864810614,
      "rewards/margins": 0.8704747846151974,
      "rewards/rejected": -0.4030467981341361,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 16.599842071533203,
      "kl": 12.55467700958252,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -33685516.60307692,
      "logits/rejected": -35014642.9968254,
      "logps/chosen": -477.19615384615383,
      "logps/rejected": -369.03978174603174,
      "loss": 0.5615,
      "loss/base_kto": 3.199857711791992,
      "metrics/advantage_var": 425.13525390625,
      "regularization/lipschitz_norm": 1306.302001953125,
      "regularization/mmd": 0.1948663741350174,
      "regularization/rkhs_norm": 231.98379516601562,
      "regularization/w1": 1.097293734550476,
      "rewards/chosen": 0.4951728703425481,
      "rewards/margins": 0.8536986097398696,
      "rewards/rejected": -0.3585257393973214,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 15.497091293334961,
      "kl": 12.510932922363281,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -33772782.4238806,
      "logits/rejected": -36386779.06885246,
      "logps/chosen": -481.1972014925373,
      "logps/rejected": -392.2569159836066,
      "loss": 0.5564,
      "loss/base_kto": 3.202043294906616,
      "metrics/advantage_var": 396.09527587890625,
      "regularization/lipschitz_norm": 1263.4398193359375,
      "regularization/mmd": 0.18781693279743195,
      "regularization/rkhs_norm": 223.59158325195312,
      "regularization/w1": 1.061289668083191,
      "rewards/chosen": 0.5571805583896922,
      "rewards/margins": 0.8350556894651917,
      "rewards/rejected": -0.2778751310754995,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 14.99384593963623,
      "kl": 17.907392501831055,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -33640813.598736174,
      "logits/rejected": -34056157.18083462,
      "logps/chosen": -498.21929304897316,
      "logps/rejected": -372.19914026275114,
      "loss": 0.5502,
      "loss/base_kto": 3.11967134475708,
      "metrics/advantage_var": 407.9004211425781,
      "regularization/lipschitz_norm": 1296.3050537109375,
      "regularization/mmd": 0.19289694726467133,
      "regularization/rkhs_norm": 229.6392059326172,
      "regularization/w1": 1.08889639377594,
      "rewards/chosen": 0.5916010900319658,
      "rewards/margins": 0.8917609040322798,
      "rewards/rejected": -0.30015981400031394,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 14.467615127563477,
      "kl": 12.251875877380371,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -34850161.9910045,
      "logits/rejected": -34718998.96900489,
      "logps/chosen": -486.04375937031483,
      "logps/rejected": -388.5187347063622,
      "loss": 0.5599,
      "loss/base_kto": 3.199481964111328,
      "metrics/advantage_var": 437.51171875,
      "regularization/lipschitz_norm": 1291.8138427734375,
      "regularization/mmd": 0.1944969892501831,
      "regularization/rkhs_norm": 231.54405212402344,
      "regularization/w1": 1.0851236581802368,
      "rewards/chosen": 0.5567921605603449,
      "rewards/margins": 0.8608295813839073,
      "rewards/rejected": -0.3040374208235624,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 17.1892147064209,
      "kl": 14.111940383911133,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -33855791.40740741,
      "logits/rejected": -35201098.58573596,
      "logps/chosen": -465.55626006441224,
      "logps/rejected": -373.59080993930195,
      "loss": 0.5627,
      "loss/base_kto": 3.1693296432495117,
      "metrics/advantage_var": 446.8841857910156,
      "regularization/lipschitz_norm": 1349.1328125,
      "regularization/mmd": 0.19930490851402283,
      "regularization/rkhs_norm": 237.26773071289062,
      "regularization/w1": 1.13327157497406,
      "rewards/chosen": 0.5713836129352858,
      "rewards/margins": 0.864196394493518,
      "rewards/rejected": -0.29281278155823215,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 20.75007438659668,
      "kl": 15.750589370727539,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -35293881.32515337,
      "logits/rejected": -35821000.560509555,
      "logps/chosen": -466.9783838190184,
      "logps/rejected": -394.8319566082803,
      "loss": 0.5558,
      "loss/base_kto": 3.197842836380005,
      "metrics/advantage_var": 419.5578308105469,
      "regularization/lipschitz_norm": 1263.475830078125,
      "regularization/mmd": 0.18733568489551544,
      "regularization/rkhs_norm": 223.0186767578125,
      "regularization/w1": 1.0613197088241577,
      "rewards/chosen": 0.48352930735956673,
      "rewards/margins": 0.8397992166776524,
      "rewards/rejected": -0.35626990931808566,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 18.520158767700195,
      "kl": 11.796889305114746,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -35220127.79617834,
      "logits/rejected": -36028525.93865031,
      "logps/chosen": -486.74293391719743,
      "logps/rejected": -375.61742714723925,
      "loss": 0.5652,
      "loss/base_kto": 3.2053322792053223,
      "metrics/advantage_var": 438.5023498535156,
      "regularization/lipschitz_norm": 1335.5159912109375,
      "regularization/mmd": 0.19403968751430511,
      "regularization/rkhs_norm": 230.9996337890625,
      "regularization/w1": 1.1218335628509521,
      "rewards/chosen": 0.5484309105356787,
      "rewards/margins": 0.8387103223664638,
      "rewards/rejected": -0.2902794118307851,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 13.852124214172363,
      "kl": 12.641082763671875,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -34629694.24924012,
      "logits/rejected": -36090721.95498392,
      "logps/chosen": -488.9081022036474,
      "logps/rejected": -382.36211314308684,
      "loss": 0.5489,
      "loss/base_kto": 3.1825757026672363,
      "metrics/advantage_var": 392.0561828613281,
      "regularization/lipschitz_norm": 1216.16796875,
      "regularization/mmd": 0.18730561435222626,
      "regularization/rkhs_norm": 222.9828643798828,
      "regularization/w1": 1.0215810537338257,
      "rewards/chosen": 0.495199835409147,
      "rewards/margins": 0.8462143682527337,
      "rewards/rejected": -0.3510145328435867,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 13.145265579223633,
      "kl": 12.742012023925781,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -33609220.61261261,
      "logits/rejected": -35441523.90879479,
      "logps/chosen": -485.61425487987987,
      "logps/rejected": -370.5197984527687,
      "loss": 0.5648,
      "loss/base_kto": 3.175414800643921,
      "metrics/advantage_var": 430.4765319824219,
      "regularization/lipschitz_norm": 1363.9232177734375,
      "regularization/mmd": 0.19717474281787872,
      "regularization/rkhs_norm": 234.73184204101562,
      "regularization/w1": 1.1456955671310425,
      "rewards/chosen": 0.5673646210908174,
      "rewards/margins": 0.8735518562737876,
      "rewards/rejected": -0.30618723518297025,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 14.446263313293457,
      "kl": 12.519579887390137,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34501147.44257274,
      "logits/rejected": -35055099.91706539,
      "logps/chosen": -489.0198124042879,
      "logps/rejected": -369.31708532695376,
      "loss": 0.5605,
      "loss/base_kto": 3.2099685668945312,
      "metrics/advantage_var": 428.7497253417969,
      "regularization/lipschitz_norm": 1285.4044189453125,
      "regularization/mmd": 0.19443576037883759,
      "regularization/rkhs_norm": 231.4711151123047,
      "regularization/w1": 1.0797398090362549,
      "rewards/chosen": 0.5657736061177977,
      "rewards/margins": 0.8485941905990203,
      "rewards/rejected": -0.2828205844812226,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 13.536280632019043,
      "kl": 14.868743896484375,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -34013258.41390205,
      "logits/rejected": -35506039.88871716,
      "logps/chosen": -453.9443127962085,
      "logps/rejected": -375.7507486476043,
      "loss": 0.5591,
      "loss/base_kto": 3.223512649536133,
      "metrics/advantage_var": 398.3023376464844,
      "regularization/lipschitz_norm": 1263.544189453125,
      "regularization/mmd": 0.18802706897258759,
      "regularization/rkhs_norm": 223.8417510986328,
      "regularization/w1": 1.0613771677017212,
      "rewards/chosen": 0.5673392297343997,
      "rewards/margins": 0.7916319435895122,
      "rewards/rejected": -0.22429271385511254,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 14.502313613891602,
      "kl": 16.450363159179688,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34611026.386185244,
      "logits/rejected": -36537320.11197512,
      "logps/chosen": -482.7138932496075,
      "logps/rejected": -383.52991349144634,
      "loss": 0.5555,
      "loss/base_kto": 3.1732280254364014,
      "metrics/advantage_var": 426.1412048339844,
      "regularization/lipschitz_norm": 1283.1588134765625,
      "regularization/mmd": 0.1930054873228073,
      "regularization/rkhs_norm": 229.76841735839844,
      "regularization/w1": 1.0778534412384033,
      "rewards/chosen": 0.5560097686911303,
      "rewards/margins": 0.8403388167173502,
      "rewards/rejected": -0.2843290480262199,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 21.743240356445312,
      "kl": 16.845134735107422,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -33126860.881141044,
      "logits/rejected": -35962286.74268105,
      "logps/chosen": -492.361529318542,
      "logps/rejected": -365.0382318952234,
      "loss": 0.5529,
      "loss/base_kto": 3.1860368251800537,
      "metrics/advantage_var": 422.7635803222656,
      "regularization/lipschitz_norm": 1245.07568359375,
      "regularization/mmd": 0.1915583610534668,
      "regularization/rkhs_norm": 228.045654296875,
      "regularization/w1": 1.0458635091781616,
      "rewards/chosen": 0.5459863629469592,
      "rewards/margins": 0.8514440438593003,
      "rewards/rejected": -0.3054576809123411,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 17.54193115234375,
      "kl": 15.537714004516602,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -34677190.757763974,
      "logits/rejected": -35669922.6163522,
      "logps/chosen": -481.9429833074534,
      "logps/rejected": -383.13536753144655,
      "loss": 0.563,
      "loss/base_kto": 3.186859369277954,
      "metrics/advantage_var": 453.5262756347656,
      "regularization/lipschitz_norm": 1331.4178466796875,
      "regularization/mmd": 0.19879555702209473,
      "regularization/rkhs_norm": 236.661376953125,
      "regularization/w1": 1.1183909177780151,
      "rewards/chosen": 0.5702908438925417,
      "rewards/margins": 0.854516403726956,
      "rewards/rejected": -0.28422555983441433,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 15.706538200378418,
      "kl": 13.580639839172363,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34258531.93993994,
      "logits/rejected": -36151379.38762215,
      "logps/chosen": -488.109375,
      "logps/rejected": -366.2739973534202,
      "loss": 0.555,
      "loss/base_kto": 3.148057460784912,
      "metrics/advantage_var": 436.72119140625,
      "regularization/lipschitz_norm": 1302.8299560546875,
      "regularization/mmd": 0.19719727337360382,
      "regularization/rkhs_norm": 234.7586669921875,
      "regularization/w1": 1.0943771600723267,
      "rewards/chosen": 0.6121360618430931,
      "rewards/margins": 0.8884072730509769,
      "rewards/rejected": -0.2762712112078837,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 15.945714950561523,
      "kl": 15.48679256439209,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -33477265.341935486,
      "logits/rejected": -35727934.06060606,
      "logps/chosen": -487.28397177419356,
      "logps/rejected": -381.74251893939396,
      "loss": 0.5554,
      "loss/base_kto": 3.197526216506958,
      "metrics/advantage_var": 382.0160827636719,
      "regularization/lipschitz_norm": 1259.7698974609375,
      "regularization/mmd": 0.18740373849868774,
      "regularization/rkhs_norm": 223.09971618652344,
      "regularization/w1": 1.0582067966461182,
      "rewards/chosen": 0.5319727251606603,
      "rewards/margins": 0.8168592645042919,
      "rewards/rejected": -0.2848865393436316,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 14.116644859313965,
      "kl": 12.542868614196777,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34326290.14781297,
      "logits/rejected": -36002339.68233387,
      "logps/chosen": -491.9751602564103,
      "logps/rejected": -387.92184967585086,
      "loss": 0.5657,
      "loss/base_kto": 3.202550172805786,
      "metrics/advantage_var": 420.9132385253906,
      "regularization/lipschitz_norm": 1341.544921875,
      "regularization/mmd": 0.1959245502948761,
      "regularization/rkhs_norm": 233.24351501464844,
      "regularization/w1": 1.1268978118896484,
      "rewards/chosen": 0.5192428289675245,
      "rewards/margins": 0.811094150888108,
      "rewards/rejected": -0.2918513219205835,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 15.54608154296875,
      "kl": 15.73474407196045,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35531926.01273885,
      "logits/rejected": -36403865.91411043,
      "logps/chosen": -491.41983479299364,
      "logps/rejected": -370.36450345092027,
      "loss": 0.5564,
      "loss/base_kto": 3.1753265857696533,
      "metrics/advantage_var": 405.5484313964844,
      "regularization/lipschitz_norm": 1290.739501953125,
      "regularization/mmd": 0.19158188998699188,
      "regularization/rkhs_norm": 228.0736846923828,
      "regularization/w1": 1.0842211246490479,
      "rewards/chosen": 0.5327730604038117,
      "rewards/margins": 0.8599869796396964,
      "rewards/rejected": -0.3272139192358848,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 14.386474609375,
      "kl": 13.991683006286621,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -35624960.0,
      "logits/rejected": -36200556.46105919,
      "logps/chosen": -484.3078957680251,
      "logps/rejected": -366.23009151090343,
      "loss": 0.554,
      "loss/base_kto": 3.1901745796203613,
      "metrics/advantage_var": 404.5918884277344,
      "regularization/lipschitz_norm": 1253.319580078125,
      "regularization/mmd": 0.18866340816020966,
      "regularization/rkhs_norm": 224.5992889404297,
      "regularization/w1": 1.0527886152267456,
      "rewards/chosen": 0.5278572871767241,
      "rewards/margins": 0.8375256409615883,
      "rewards/rejected": -0.3096683537848642,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 16.03128433227539,
      "kl": 15.179038047790527,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -33715310.479871176,
      "logits/rejected": -34859570.50075872,
      "logps/chosen": -489.4578301127214,
      "logps/rejected": -382.5035091047041,
      "loss": 0.5557,
      "loss/base_kto": 3.1808385848999023,
      "metrics/advantage_var": 406.5344543457031,
      "regularization/lipschitz_norm": 1280.5325927734375,
      "regularization/mmd": 0.18936872482299805,
      "regularization/rkhs_norm": 225.43896484375,
      "regularization/w1": 1.0756473541259766,
      "rewards/chosen": 0.5651349299579811,
      "rewards/margins": 0.8417423449792609,
      "rewards/rejected": -0.2766074150212799,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 17.106462478637695,
      "kl": 15.91718578338623,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -34661180.4789644,
      "logits/rejected": -34755144.700906344,
      "logps/chosen": -454.25151699029124,
      "logps/rejected": -389.61961858006043,
      "loss": 0.5563,
      "loss/base_kto": 3.1529488563537598,
      "metrics/advantage_var": 445.4698181152344,
      "regularization/lipschitz_norm": 1311.3211669921875,
      "regularization/mmd": 0.19628138840198517,
      "regularization/rkhs_norm": 233.66831970214844,
      "regularization/w1": 1.101509928703308,
      "rewards/chosen": 0.5545109128489077,
      "rewards/margins": 0.9146029606871937,
      "rewards/rejected": -0.36009204783828597,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 17.375688552856445,
      "kl": 14.020922660827637,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -34085769.39641109,
      "logits/rejected": -35753471.23238381,
      "logps/chosen": -469.96818923327896,
      "logps/rejected": -350.6618487631184,
      "loss": 0.5568,
      "loss/base_kto": 3.1888301372528076,
      "metrics/advantage_var": 385.19537353515625,
      "regularization/lipschitz_norm": 1283.8055419921875,
      "regularization/mmd": 0.18733106553554535,
      "regularization/rkhs_norm": 223.01316833496094,
      "regularization/w1": 1.0783965587615967,
      "rewards/chosen": 0.5275428361254842,
      "rewards/margins": 0.8230447967540566,
      "rewards/rejected": -0.2955019606285724,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 16.014177322387695,
      "kl": 11.869200706481934,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35327283.82822086,
      "logits/rejected": -33905820.53503185,
      "logps/chosen": -443.2274731595092,
      "logps/rejected": -357.488181727707,
      "loss": 0.5615,
      "loss/base_kto": 3.2101943492889404,
      "metrics/advantage_var": 410.871826171875,
      "regularization/lipschitz_norm": 1297.1297607421875,
      "regularization/mmd": 0.19185256958007812,
      "regularization/rkhs_norm": 228.39590454101562,
      "regularization/w1": 1.08958899974823,
      "rewards/chosen": 0.5072993647101467,
      "rewards/margins": 0.8425749539909737,
      "rewards/rejected": -0.335275589280827,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 18.9195613861084,
      "kl": 15.337427139282227,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34568424.21417323,
      "logits/rejected": -35940101.159689926,
      "logps/chosen": -468.53543307086613,
      "logps/rejected": -375.7328003875969,
      "loss": 0.5566,
      "loss/base_kto": 3.189122438430786,
      "metrics/advantage_var": 379.7440185546875,
      "regularization/lipschitz_norm": 1284.2718505859375,
      "regularization/mmd": 0.18473811447620392,
      "regularization/rkhs_norm": 219.9263153076172,
      "regularization/w1": 1.0787882804870605,
      "rewards/chosen": 0.5099421944205217,
      "rewards/margins": 0.8381716638972658,
      "rewards/rejected": -0.3282294694767442,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.97585917447635e+17,
      "train_loss": 0.5867518629228925,
      "train_runtime": 11108.9609,
      "train_samples_per_second": 13.342,
      "train_steps_per_second": 0.208
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.97585917447635e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}