{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 26.822582244873047,
      "kl": 6.688536167144775,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35990294.6835443,
      "logits/rejected": -37641911.30864198,
      "logps/chosen": -502.6991693037975,
      "logps/rejected": -375.73408564814815,
      "loss": 0.6253,
      "loss/base_kto": 3.941157579421997,
      "metrics/advantage_var": 271.9526062011719,
      "regularization/lipschitz_norm": 1082.0919189453125,
      "regularization/mmd": 0.15222804248332977,
      "regularization/rkhs_norm": 181.22386169433594,
      "regularization/w1": 0.9089571833610535,
      "rewards/chosen": -0.04996775373627868,
      "rewards/margins": 0.049426915031203,
      "rewards/rejected": -0.09939466876748168,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 18.23899269104004,
      "kl": 22.54376792907715,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35605139.52542373,
      "logits/rejected": -37668834.789223455,
      "logps/chosen": -490.5431432973806,
      "logps/rejected": -358.90397187004754,
      "loss": 0.5956,
      "loss/base_kto": 3.8734192848205566,
      "metrics/advantage_var": 181.184326171875,
      "regularization/lipschitz_norm": 904.0665893554688,
      "regularization/mmd": 0.13191047310829163,
      "regularization/rkhs_norm": 157.03627014160156,
      "regularization/w1": 0.7594159841537476,
      "rewards/chosen": 0.40319951179398594,
      "rewards/margins": 0.0966510682581797,
      "rewards/rejected": 0.30654844353580624,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 26.432907104492188,
      "kl": 9.919577598571777,
      "learning_rate": 5.9e-07,
      "logits/chosen": -34675935.179487176,
      "logits/rejected": -35587090.73170732,
      "logps/chosen": -469.19861778846155,
      "logps/rejected": -362.9036775914634,
      "loss": 0.5921,
      "loss/base_kto": 3.903366804122925,
      "metrics/advantage_var": 164.3740997314453,
      "regularization/lipschitz_norm": 849.7183837890625,
      "regularization/mmd": 0.11974674463272095,
      "regularization/rkhs_norm": 142.55564880371094,
      "regularization/w1": 0.713763415813446,
      "rewards/chosen": 0.14293929858085436,
      "rewards/margins": 0.06006812900211603,
      "rewards/rejected": 0.08287116957873833,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 20.65061378479004,
      "kl": 11.507179260253906,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37237064.97007874,
      "logits/rejected": -37540843.36124031,
      "logps/chosen": -505.36899606299215,
      "logps/rejected": -384.1623062015504,
      "loss": 0.5978,
      "loss/base_kto": 3.886489152908325,
      "metrics/advantage_var": 178.0410614013672,
      "regularization/lipschitz_norm": 915.6586303710938,
      "regularization/mmd": 0.1268511265516281,
      "regularization/rkhs_norm": 151.0132598876953,
      "regularization/w1": 0.7691532373428345,
      "rewards/chosen": 0.1602300688976378,
      "rewards/margins": 0.1150228663706877,
      "rewards/rejected": 0.0452072025269501,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 19.4213924407959,
      "kl": 6.736691951751709,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36571694.260336906,
      "logits/rejected": -37923727.31100479,
      "logps/chosen": -490.65974349157733,
      "logps/rejected": -373.0764553429027,
      "loss": 0.6046,
      "loss/base_kto": 3.8735275268554688,
      "metrics/advantage_var": 213.0327911376953,
      "regularization/lipschitz_norm": 984.1552734375,
      "regularization/mmd": 0.1367308646440506,
      "regularization/rkhs_norm": 162.77484130859375,
      "regularization/w1": 0.8266903758049011,
      "rewards/chosen": 0.05937138742911323,
      "rewards/margins": 0.13850803054224498,
      "rewards/rejected": -0.07913664311313173,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 22.295644760131836,
      "kl": 14.757791519165039,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -37726292.421374045,
      "logits/rejected": -37582910.2592,
      "logps/chosen": -491.5485687022901,
      "logps/rejected": -360.84155,
      "loss": 0.5976,
      "loss/base_kto": 3.8581607341766357,
      "metrics/advantage_var": 203.2979736328125,
      "regularization/lipschitz_norm": 937.5336303710938,
      "regularization/mmd": 0.1350373476743698,
      "regularization/rkhs_norm": 160.7587432861328,
      "regularization/w1": 0.7875282168388367,
      "rewards/chosen": 0.24400709312380725,
      "rewards/margins": 0.12322369468630726,
      "rewards/rejected": 0.1207833984375,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 21.928726196289062,
      "kl": 13.538284301757812,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35640385.96319018,
      "logits/rejected": -38540506.49681529,
      "logps/chosen": -485.2036042944785,
      "logps/rejected": -383.4859673566879,
      "loss": 0.5947,
      "loss/base_kto": 3.8243279457092285,
      "metrics/advantage_var": 187.7855682373047,
      "regularization/lipschitz_norm": 953.6726684570312,
      "regularization/mmd": 0.13212864100933075,
      "regularization/rkhs_norm": 157.29600524902344,
      "regularization/w1": 0.8010851144790649,
      "rewards/chosen": 0.2269869611307156,
      "rewards/margins": 0.15800791042760615,
      "rewards/rejected": 0.06897905070310945,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 20.098386764526367,
      "kl": 20.5661678314209,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -37250196.91754123,
      "logits/rejected": -36808780.84176183,
      "logps/chosen": -494.4382496251874,
      "logps/rejected": -372.70360929853183,
      "loss": 0.5873,
      "loss/base_kto": 3.7914528846740723,
      "metrics/advantage_var": 204.13919067382812,
      "regularization/lipschitz_norm": 918.5911254882812,
      "regularization/mmd": 0.13557849824428558,
      "regularization/rkhs_norm": 161.40296936035156,
      "regularization/w1": 0.7716165781021118,
      "rewards/chosen": 0.3903130069486741,
      "rewards/margins": 0.16678777762376076,
      "rewards/rejected": 0.22352522932491334,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 29.430130004882812,
      "kl": 24.841371536254883,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35907025.45454545,
      "logits/rejected": -37089340.532019705,
      "logps/chosen": -465.20985469448584,
      "logps/rejected": -357.6175595238095,
      "loss": 0.5938,
      "loss/base_kto": 3.787205457687378,
      "metrics/advantage_var": 213.7475128173828,
      "regularization/lipschitz_norm": 978.7572631835938,
      "regularization/mmd": 0.1411084234714508,
      "regularization/rkhs_norm": 167.9862060546875,
      "regularization/w1": 0.8221561312675476,
      "rewards/chosen": 0.45094488061487054,
      "rewards/margins": 0.18808537953568077,
      "rewards/rejected": 0.26285950107918976,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 17.58774757385254,
      "kl": 28.271289825439453,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -37944043.48961424,
      "logits/rejected": -39779676.09240924,
      "logps/chosen": -486.9328171364985,
      "logps/rejected": -380.546875,
      "loss": 0.5937,
      "loss/base_kto": 3.806340456008911,
      "metrics/advantage_var": 222.38003540039062,
      "regularization/lipschitz_norm": 955.1566772460938,
      "regularization/mmd": 0.14101631939411163,
      "regularization/rkhs_norm": 167.87657165527344,
      "regularization/w1": 0.8023315668106079,
      "rewards/chosen": 0.43112114699728765,
      "rewards/margins": 0.15921002949393576,
      "rewards/rejected": 0.2719111175033519,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 20.11609649658203,
      "kl": 17.88020896911621,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37807972.586846545,
      "logits/rejected": -38566248.710334785,
      "logps/chosen": -478.32419898819563,
      "logps/rejected": -379.9218522561863,
      "loss": 0.5909,
      "loss/base_kto": 3.7347412109375,
      "metrics/advantage_var": 216.2038116455078,
      "regularization/lipschitz_norm": 1012.9171752929688,
      "regularization/mmd": 0.14182019233703613,
      "regularization/rkhs_norm": 168.8335723876953,
      "regularization/w1": 0.850850522518158,
      "rewards/chosen": 0.24950945437658095,
      "rewards/margins": 0.237535000276973,
      "rewards/rejected": 0.011974454099607953,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 17.779966354370117,
      "kl": 8.485206604003906,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36586348.32329635,
      "logits/rejected": -36844611.84591679,
      "logps/chosen": -465.98350832012676,
      "logps/rejected": -377.4280383281972,
      "loss": 0.5997,
      "loss/base_kto": 3.7884018421173096,
      "metrics/advantage_var": 250.99400329589844,
      "regularization/lipschitz_norm": 1028.31787109375,
      "regularization/mmd": 0.14561444520950317,
      "regularization/rkhs_norm": 173.35052490234375,
      "regularization/w1": 0.8637871146202087,
      "rewards/chosen": 0.0761179311906479,
      "rewards/margins": 0.21856869585112315,
      "rewards/rejected": -0.14245076466047524,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 22.252639770507812,
      "kl": 10.02391529083252,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36825978.01869159,
      "logits/rejected": -36777714.35736677,
      "logps/chosen": -478.9975175233645,
      "logps/rejected": -378.5579447492163,
      "loss": 0.5986,
      "loss/base_kto": 3.78965163230896,
      "metrics/advantage_var": 300.36932373046875,
      "regularization/lipschitz_norm": 1015.0419921875,
      "regularization/mmd": 0.14683066308498383,
      "regularization/rkhs_norm": 174.7983856201172,
      "regularization/w1": 0.8526352047920227,
      "rewards/chosen": 0.1416795558275835,
      "rewards/margins": 0.18282197167847097,
      "rewards/rejected": -0.04114241585088748,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 15.103630065917969,
      "kl": 9.661877632141113,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -35989077.19547658,
      "logits/rejected": -38817792.0,
      "logps/chosen": -504.87838247172857,
      "logps/rejected": -364.2305692133132,
      "loss": 0.5882,
      "loss/base_kto": 3.754521608352661,
      "metrics/advantage_var": 207.8542938232422,
      "regularization/lipschitz_norm": 967.5964965820312,
      "regularization/mmd": 0.13858331739902496,
      "regularization/rkhs_norm": 164.9801483154297,
      "regularization/w1": 0.8127809762954712,
      "rewards/chosen": 0.16231197917981371,
      "rewards/margins": 0.21434921681174568,
      "rewards/rejected": -0.052037237631931965,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 27.652835845947266,
      "kl": 5.51276159286499,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36078726.8982036,
      "logits/rejected": -38148705.04575163,
      "logps/chosen": -463.51300523952096,
      "logps/rejected": -374.59369893790847,
      "loss": 0.5975,
      "loss/base_kto": 3.810443162918091,
      "metrics/advantage_var": 228.3494110107422,
      "regularization/lipschitz_norm": 980.9369506835938,
      "regularization/mmd": 0.14567025005817413,
      "regularization/rkhs_norm": 173.41697692871094,
      "regularization/w1": 0.8239870071411133,
      "rewards/chosen": 0.030363008647621748,
      "rewards/margins": 0.18544664625916052,
      "rewards/rejected": -0.15508363761153876,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 24.82827377319336,
      "kl": 14.267743110656738,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -37787016.06462035,
      "logits/rejected": -38199646.11195159,
      "logps/chosen": -523.109147819063,
      "logps/rejected": -394.60088880484113,
      "loss": 0.5992,
      "loss/base_kto": 3.7817025184631348,
      "metrics/advantage_var": 229.16064453125,
      "regularization/lipschitz_norm": 1031.0673828125,
      "regularization/mmd": 0.14609156548976898,
      "regularization/rkhs_norm": 173.91854858398438,
      "regularization/w1": 0.8660966753959656,
      "rewards/chosen": 0.20830596282909683,
      "rewards/margins": 0.1782946487024776,
      "rewards/rejected": 0.030011314126619233,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.098224639892578,
      "kl": 7.649110317230225,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -34977713.48242812,
      "logits/rejected": -36509104.14678899,
      "logps/chosen": -470.78973642172525,
      "logps/rejected": -364.20943233944956,
      "loss": 0.5915,
      "loss/base_kto": 3.7403557300567627,
      "metrics/advantage_var": 246.9385223388672,
      "regularization/lipschitz_norm": 1009.1304931640625,
      "regularization/mmd": 0.1441594511270523,
      "regularization/rkhs_norm": 171.61839294433594,
      "regularization/w1": 0.8476696014404297,
      "rewards/chosen": 0.07622077746894032,
      "rewards/margins": 0.2398331781888192,
      "rewards/rejected": -0.16361240071987887,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 22.88916778564453,
      "kl": 8.33399486541748,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -34745693.053311795,
      "logits/rejected": -36200144.36308623,
      "logps/chosen": -495.8392568659128,
      "logps/rejected": -364.34351361573374,
      "loss": 0.5983,
      "loss/base_kto": 3.7419095039367676,
      "metrics/advantage_var": 240.70654296875,
      "regularization/lipschitz_norm": 1065.1556396484375,
      "regularization/mmd": 0.15013444423675537,
      "regularization/rkhs_norm": 178.7314910888672,
      "regularization/w1": 0.8947307467460632,
      "rewards/chosen": 0.08383417206550069,
      "rewards/margins": 0.238408914000365,
      "rewards/rejected": -0.1545747419348643,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 19.463855743408203,
      "kl": 7.266137599945068,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35963957.04866562,
      "logits/rejected": -36229820.715396576,
      "logps/chosen": -481.81171507064363,
      "logps/rejected": -354.65685750388803,
      "loss": 0.6024,
      "loss/base_kto": 3.806239366531372,
      "metrics/advantage_var": 234.2755126953125,
      "regularization/lipschitz_norm": 1030.6988525390625,
      "regularization/mmd": 0.1471906453371048,
      "regularization/rkhs_norm": 175.22695922851562,
      "regularization/w1": 0.8657870292663574,
      "rewards/chosen": 0.04450525855718639,
      "rewards/margins": 0.17270566497728748,
      "rewards/rejected": -0.12820040642010108,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 16.1120548248291,
      "kl": 5.923459529876709,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36382160.863492064,
      "logits/rejected": -36331059.98769231,
      "logps/chosen": -482.41001984126984,
      "logps/rejected": -389.11834134615384,
      "loss": 0.5892,
      "loss/base_kto": 3.751215696334839,
      "metrics/advantage_var": 213.98587036132812,
      "regularization/lipschitz_norm": 978.8876953125,
      "regularization/mmd": 0.14040164649486542,
      "regularization/rkhs_norm": 167.1448211669922,
      "regularization/w1": 0.822265625,
      "rewards/chosen": -0.003133468022422185,
      "rewards/margins": 0.23518008366027013,
      "rewards/rejected": -0.2383135516826923,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 20.187463760375977,
      "kl": 3.1745717525482178,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -37297491.179810725,
      "logits/rejected": -38113194.40247678,
      "logps/chosen": -480.70174487381706,
      "logps/rejected": -360.6059162151703,
      "loss": 0.6016,
      "loss/base_kto": 3.8223042488098145,
      "metrics/advantage_var": 226.72300720214844,
      "regularization/lipschitz_norm": 1004.8407592773438,
      "regularization/mmd": 0.14660242199897766,
      "regularization/rkhs_norm": 174.52670288085938,
      "regularization/w1": 0.8440662622451782,
      "rewards/chosen": -0.150642840268108,
      "rewards/margins": 0.16019357379741542,
      "rewards/rejected": -0.3108364140655234,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 20.050668716430664,
      "kl": 9.79570484161377,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -34902970.469135806,
      "logits/rejected": -35876883.44303797,
      "logps/chosen": -455.43648726851853,
      "logps/rejected": -377.20446004746833,
      "loss": 0.6015,
      "loss/base_kto": 3.7855632305145264,
      "metrics/advantage_var": 250.9638214111328,
      "regularization/lipschitz_norm": 1041.472412109375,
      "regularization/mmd": 0.15120147168636322,
      "regularization/rkhs_norm": 180.00173950195312,
      "regularization/w1": 0.8748367428779602,
      "rewards/chosen": 0.09103574870545188,
      "rewards/margins": 0.19406917665615403,
      "rewards/rejected": -0.10303342795070214,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 21.8386173248291,
      "kl": 16.655717849731445,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -34515184.75725191,
      "logits/rejected": -35051955.8144,
      "logps/chosen": -464.76564885496185,
      "logps/rejected": -355.4643,
      "loss": 0.5902,
      "loss/base_kto": 3.6980209350585938,
      "metrics/advantage_var": 242.9734344482422,
      "regularization/lipschitz_norm": 1039.7606201171875,
      "regularization/mmd": 0.15043532848358154,
      "regularization/rkhs_norm": 179.08969116210938,
      "regularization/w1": 0.8733989000320435,
      "rewards/chosen": 0.32243722231333494,
      "rewards/margins": 0.27308180239145996,
      "rewards/rejected": 0.049355419921875,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 21.20807647705078,
      "kl": 15.139017105102539,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36116527.18894009,
      "logits/rejected": -36718432.457869634,
      "logps/chosen": -494.6800115207373,
      "logps/rejected": -387.47717110492846,
      "loss": 0.603,
      "loss/base_kto": 3.7427093982696533,
      "metrics/advantage_var": 287.2233581542969,
      "regularization/lipschitz_norm": 1094.8924560546875,
      "regularization/mmd": 0.16165395081043243,
      "regularization/rkhs_norm": 192.44517517089844,
      "regularization/w1": 0.9197096228599548,
      "rewards/chosen": 0.3005254836309524,
      "rewards/margins": 0.23888388148624104,
      "rewards/rejected": 0.06164160214471135,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 19.53720474243164,
      "kl": 22.42978858947754,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35044256.049155146,
      "logits/rejected": -36973999.41494436,
      "logps/chosen": -464.2317588325653,
      "logps/rejected": -383.77931240063594,
      "loss": 0.5915,
      "loss/base_kto": 3.733435869216919,
      "metrics/advantage_var": 259.2154235839844,
      "regularization/lipschitz_norm": 1009.2023315429688,
      "regularization/mmd": 0.1510559767484665,
      "regularization/rkhs_norm": 179.82855224609375,
      "regularization/w1": 0.8477298617362976,
      "rewards/chosen": 0.30200598463301653,
      "rewards/margins": 0.2217405937790134,
      "rewards/rejected": 0.08026539085400312,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 17.993221282958984,
      "kl": 8.867077827453613,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -35755495.77287066,
      "logits/rejected": -36845174.88544892,
      "logps/chosen": -465.98945189274446,
      "logps/rejected": -379.63191756965944,
      "loss": 0.5932,
      "loss/base_kto": 3.7875168323516846,
      "metrics/advantage_var": 214.8356170654297,
      "regularization/lipschitz_norm": 972.3076171875,
      "regularization/mmd": 0.14126591384410858,
      "regularization/rkhs_norm": 168.17369079589844,
      "regularization/w1": 0.8167383074760437,
      "rewards/chosen": 0.06241883238783394,
      "rewards/margins": 0.18564694684368,
      "rewards/rejected": -0.12322811445584607,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 24.116201400756836,
      "kl": 17.905044555664062,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36314500.0913242,
      "logits/rejected": -36291855.203852326,
      "logps/chosen": -499.87661719939115,
      "logps/rejected": -359.70856741573033,
      "loss": 0.5796,
      "loss/base_kto": 3.6291000843048096,
      "metrics/advantage_var": 258.013671875,
      "regularization/lipschitz_norm": 1020.1943359375,
      "regularization/mmd": 0.15099453926086426,
      "regularization/rkhs_norm": 179.75540161132812,
      "regularization/w1": 0.8569631576538086,
      "rewards/chosen": 0.3302828528928249,
      "rewards/margins": 0.34321509911785436,
      "rewards/rejected": -0.012932246225029469,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 33.473541259765625,
      "kl": 21.3460693359375,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35796976.50832073,
      "logits/rejected": -36226168.762520194,
      "logps/chosen": -495.66442889561273,
      "logps/rejected": -384.9128634894992,
      "loss": 0.6024,
      "loss/base_kto": 3.696981906890869,
      "metrics/advantage_var": 294.5236511230469,
      "regularization/lipschitz_norm": 1138.978271484375,
      "regularization/mmd": 0.16508857905864716,
      "regularization/rkhs_norm": 196.53404235839844,
      "regularization/w1": 0.9567417502403259,
      "rewards/chosen": 0.3779589816048719,
      "rewards/margins": 0.29664941193329036,
      "rewards/rejected": 0.08130956967158155,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 18.26300811767578,
      "kl": 21.616954803466797,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -36925939.358024694,
      "logits/rejected": -37991884.151898734,
      "logps/chosen": -503.7965374228395,
      "logps/rejected": -367.091426028481,
      "loss": 0.5929,
      "loss/base_kto": 3.695453643798828,
      "metrics/advantage_var": 244.77578735351562,
      "regularization/lipschitz_norm": 1066.0438232421875,
      "regularization/mmd": 0.15256932377815247,
      "regularization/rkhs_norm": 181.630126953125,
      "regularization/w1": 0.8954768180847168,
      "rewards/chosen": 0.4246550195011092,
      "rewards/margins": 0.2642496552834717,
      "rewards/rejected": 0.16040536421763746,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 20.769075393676758,
      "kl": 11.691008567810059,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36831582.81481481,
      "logits/rejected": -37781082.73417722,
      "logps/chosen": -524.2831790123457,
      "logps/rejected": -366.26864121835445,
      "loss": 0.5904,
      "loss/base_kto": 3.6856117248535156,
      "metrics/advantage_var": 238.05186462402344,
      "regularization/lipschitz_norm": 1057.029541015625,
      "regularization/mmd": 0.15006379783153534,
      "regularization/rkhs_norm": 178.64736938476562,
      "regularization/w1": 0.8879047632217407,
      "rewards/chosen": 0.1865536254129292,
      "rewards/margins": 0.2912244276621133,
      "rewards/rejected": -0.10467080224918414,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 19.232282638549805,
      "kl": 13.548723220825195,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -34448511.39527559,
      "logits/rejected": -36557920.84341085,
      "logps/chosen": -474.90871062992125,
      "logps/rejected": -382.872238372093,
      "loss": 0.5904,
      "loss/base_kto": 3.714409589767456,
      "metrics/advantage_var": 242.79356384277344,
      "regularization/lipschitz_norm": 1022.59619140625,
      "regularization/mmd": 0.15015900135040283,
      "regularization/rkhs_norm": 178.76072692871094,
      "regularization/w1": 0.8589809536933899,
      "rewards/chosen": 0.27847155473363683,
      "rewards/margins": 0.25291948228254063,
      "rewards/rejected": 0.025552072451096173,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 24.14682960510254,
      "kl": 9.039995193481445,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35461607.61904762,
      "logits/rejected": -37804727.89269747,
      "logps/chosen": -481.61473727422003,
      "logps/rejected": -369.2634593889717,
      "loss": 0.5981,
      "loss/base_kto": 3.751467227935791,
      "metrics/advantage_var": 252.6370849609375,
      "regularization/lipschitz_norm": 1049.42431640625,
      "regularization/mmd": 0.15219371020793915,
      "regularization/rkhs_norm": 181.18301391601562,
      "regularization/w1": 0.8815164566040039,
      "rewards/chosen": 0.010386488120544133,
      "rewards/margins": 0.2020670779584726,
      "rewards/rejected": -0.19168058983792846,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 16.865453720092773,
      "kl": 7.734155178070068,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -34780896.89719626,
      "logits/rejected": -35494452.96551724,
      "logps/chosen": -455.0786117601246,
      "logps/rejected": -385.95273315047024,
      "loss": 0.6081,
      "loss/base_kto": 3.7895705699920654,
      "metrics/advantage_var": 264.3589172363281,
      "regularization/lipschitz_norm": 1095.6732177734375,
      "regularization/mmd": 0.15475480258464813,
      "regularization/rkhs_norm": 184.23191833496094,
      "regularization/w1": 0.9203655123710632,
      "rewards/chosen": 0.07494597865785023,
      "rewards/margins": 0.2042531186614014,
      "rewards/rejected": -0.12930714000355115,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 18.38845443725586,
      "kl": 14.334578514099121,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35359368.42633229,
      "logits/rejected": -37021437.60747664,
      "logps/chosen": -494.8095121473354,
      "logps/rejected": -377.56313278816197,
      "loss": 0.5954,
      "loss/base_kto": 3.746176242828369,
      "metrics/advantage_var": 240.9512176513672,
      "regularization/lipschitz_norm": 1032.4136962890625,
      "regularization/mmd": 0.14993715286254883,
      "regularization/rkhs_norm": 178.4966278076172,
      "regularization/w1": 0.8672273755073547,
      "rewards/chosen": 0.22056825930795698,
      "rewards/margins": 0.22055451570238785,
      "rewards/rejected": 1.3743605569144275e-05,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 22.69530487060547,
      "kl": 7.90990686416626,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -35112337.537007876,
      "logits/rejected": -36394392.0124031,
      "logps/chosen": -488.8363681102362,
      "logps/rejected": -346.1170542635659,
      "loss": 0.5947,
      "loss/base_kto": 3.8013062477111816,
      "metrics/advantage_var": 205.14932250976562,
      "regularization/lipschitz_norm": 974.9683837890625,
      "regularization/mmd": 0.13693474233150482,
      "regularization/rkhs_norm": 163.01756286621094,
      "regularization/w1": 0.8189733624458313,
      "rewards/chosen": 0.026237824207215796,
      "rewards/margins": 0.16785384115700988,
      "rewards/rejected": -0.1416160169497941,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 21.700618743896484,
      "kl": 5.915856838226318,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36401620.47604328,
      "logits/rejected": -37539542.344391786,
      "logps/chosen": -501.3424942040185,
      "logps/rejected": -380.0257207740916,
      "loss": 0.6003,
      "loss/base_kto": 3.782858371734619,
      "metrics/advantage_var": 250.19741821289062,
      "regularization/lipschitz_norm": 1036.060791015625,
      "regularization/mmd": 0.14888663589954376,
      "regularization/rkhs_norm": 177.24598693847656,
      "regularization/w1": 0.8702909350395203,
      "rewards/chosen": 0.07331632351764755,
      "rewards/margins": 0.21358003010458476,
      "rewards/rejected": -0.1402637065869372,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 19.367273330688477,
      "kl": 14.19812297821045,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36714834.01941747,
      "logits/rejected": -37470514.27190332,
      "logps/chosen": -492.8945691747573,
      "logps/rejected": -386.22467428247734,
      "loss": 0.5986,
      "loss/base_kto": 3.7310502529144287,
      "metrics/advantage_var": 269.1242980957031,
      "regularization/lipschitz_norm": 1073.406494140625,
      "regularization/mmd": 0.15581868588924408,
      "regularization/rkhs_norm": 185.49844360351562,
      "regularization/w1": 0.9016615152359009,
      "rewards/chosen": 0.13576719907495197,
      "rewards/margins": 0.21811820368412044,
      "rewards/rejected": -0.08235100460916847,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 14.528448104858398,
      "kl": 10.813498497009277,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35483170.13333333,
      "logits/rejected": -37314793.15692308,
      "logps/chosen": -477.52708333333334,
      "logps/rejected": -360.9183173076923,
      "loss": 0.5935,
      "loss/base_kto": 3.7103450298309326,
      "metrics/advantage_var": 266.1695251464844,
      "regularization/lipschitz_norm": 1050.378662109375,
      "regularization/mmd": 0.1554114669561386,
      "regularization/rkhs_norm": 185.01365661621094,
      "regularization/w1": 0.8823180198669434,
      "rewards/chosen": 0.026440117851136223,
      "rewards/margins": 0.2533165733048622,
      "rewards/rejected": -0.22687645545372595,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 18.759897232055664,
      "kl": 4.533371925354004,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -34546959.741100326,
      "logits/rejected": -35124968.72727273,
      "logps/chosen": -468.8022855987055,
      "logps/rejected": -388.53063446969696,
      "loss": 0.5938,
      "loss/base_kto": 3.6106491088867188,
      "metrics/advantage_var": 319.78607177734375,
      "regularization/lipschitz_norm": 1158.6326904296875,
      "regularization/mmd": 0.16680000722408295,
      "regularization/rkhs_norm": 198.5714569091797,
      "regularization/w1": 0.9732513427734375,
      "rewards/chosen": -0.010402836845916452,
      "rewards/margins": 0.40733120523504945,
      "rewards/rejected": -0.41773404208096593,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 17.778566360473633,
      "kl": 8.78848934173584,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36056717.91730475,
      "logits/rejected": -36985096.57416268,
      "logps/chosen": -464.6512251148545,
      "logps/rejected": -386.83268540669854,
      "loss": 0.6112,
      "loss/base_kto": 3.3962676525115967,
      "metrics/advantage_var": 572.8226318359375,
      "regularization/lipschitz_norm": 1521.9007568359375,
      "regularization/mmd": 0.21491070091724396,
      "regularization/rkhs_norm": 255.84609985351562,
      "regularization/w1": 1.2783966064453125,
      "rewards/chosen": 0.3514991281226072,
      "rewards/margins": 0.6808625966639,
      "rewards/rejected": -0.32936346854129284,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 19.549013137817383,
      "kl": 13.9061918258667,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -34939704.35220126,
      "logits/rejected": -36651221.06832298,
      "logps/chosen": -442.9081662735849,
      "logps/rejected": -391.614494371118,
      "loss": 0.6016,
      "loss/base_kto": 3.3266537189483643,
      "metrics/advantage_var": 532.240234375,
      "regularization/lipschitz_norm": 1514.8626708984375,
      "regularization/mmd": 0.2138865739107132,
      "regularization/rkhs_norm": 254.62686157226562,
      "regularization/w1": 1.2724847793579102,
      "rewards/chosen": 0.5109047559822131,
      "rewards/margins": 0.748533574280932,
      "rewards/rejected": -0.23762881829871896,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 17.792404174804688,
      "kl": 3.4995524883270264,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35142991.76508972,
      "logits/rejected": -35455104.95952024,
      "logps/chosen": -514.744545269168,
      "logps/rejected": -366.4920352323838,
      "loss": 0.6144,
      "loss/base_kto": 3.391146183013916,
      "metrics/advantage_var": 613.748046875,
      "regularization/lipschitz_norm": 1552.6650390625,
      "regularization/mmd": 0.21969342231750488,
      "regularization/rkhs_norm": 261.539794921875,
      "regularization/w1": 1.3042386770248413,
      "rewards/chosen": 0.03567127769183957,
      "rewards/margins": 0.6398779634114797,
      "rewards/rejected": -0.6042066857196402,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 18.40005874633789,
      "kl": 8.422476768493652,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -36076973.728549145,
      "logits/rejected": -36445333.03286385,
      "logps/chosen": -461.4642648205928,
      "logps/rejected": -374.2261345852895,
      "loss": 0.6134,
      "loss/base_kto": 3.3130805492401123,
      "metrics/advantage_var": 659.9691772460938,
      "regularization/lipschitz_norm": 1623.8614501953125,
      "regularization/mmd": 0.22984924912452698,
      "regularization/rkhs_norm": 273.6300048828125,
      "regularization/w1": 1.3640435934066772,
      "rewards/chosen": 0.31854762227598477,
      "rewards/margins": 0.7969921351103695,
      "rewards/rejected": -0.47844451283438477,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 18.619529724121094,
      "kl": 11.051854133605957,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35030811.74961832,
      "logits/rejected": -36247548.7232,
      "logps/chosen": -462.54064885496183,
      "logps/rejected": -370.33355,
      "loss": 0.6078,
      "loss/base_kto": 3.329913377761841,
      "metrics/advantage_var": 571.1303100585938,
      "regularization/lipschitz_norm": 1562.1920166015625,
      "regularization/mmd": 0.22055485844612122,
      "regularization/rkhs_norm": 262.5653076171875,
      "regularization/w1": 1.3122414350509644,
      "rewards/chosen": 0.34584243424976147,
      "rewards/margins": 0.7301825709685115,
      "rewards/rejected": -0.38434013671875,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 20.384653091430664,
      "kl": 18.162588119506836,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -36808110.01812689,
      "logits/rejected": -36869875.57281554,
      "logps/chosen": -507.0894070996979,
      "logps/rejected": -381.21050768608416,
      "loss": 0.6195,
      "loss/base_kto": 3.315048933029175,
      "metrics/advantage_var": 618.730712890625,
      "regularization/lipschitz_norm": 1677.330078125,
      "regularization/mmd": 0.23201961815357208,
      "regularization/rkhs_norm": 276.2138366699219,
      "regularization/w1": 1.4089572429656982,
      "rewards/chosen": 0.5819440305773225,
      "rewards/margins": 0.7377112429851969,
      "rewards/rejected": -0.15576721240787444,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 15.094950675964355,
      "kl": 5.735419750213623,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34146068.21052632,
      "logits/rejected": -34838994.28571428,
      "logps/chosen": -490.48118832236844,
      "logps/rejected": -377.0299479166667,
      "loss": 0.6013,
      "loss/base_kto": 3.279555082321167,
      "metrics/advantage_var": 617.1185302734375,
      "regularization/lipschitz_norm": 1553.6285400390625,
      "regularization/mmd": 0.2261827439069748,
      "regularization/rkhs_norm": 269.26519775390625,
      "regularization/w1": 1.3050479888916016,
      "rewards/chosen": 0.28274867409153986,
      "rewards/margins": 0.8007513204015286,
      "rewards/rejected": -0.5180026463099888,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 18.639392852783203,
      "kl": 7.991387844085693,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35504902.56410257,
      "logits/rejected": -36857094.24390244,
      "logps/chosen": -490.0498297275641,
      "logps/rejected": -373.9388814786585,
      "loss": 0.6003,
      "loss/base_kto": 3.2353203296661377,
      "metrics/advantage_var": 627.2642211914062,
      "regularization/lipschitz_norm": 1591.6224365234375,
      "regularization/mmd": 0.22979405522346497,
      "regularization/rkhs_norm": 273.5643615722656,
      "regularization/w1": 1.3369629383087158,
      "rewards/chosen": 0.3509660378480569,
      "rewards/margins": 0.8579493529204655,
      "rewards/rejected": -0.5069833150724086,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 20.658008575439453,
      "kl": 6.227613925933838,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36547915.80775194,
      "logits/rejected": -38035633.385826774,
      "logps/chosen": -494.5046511627907,
      "logps/rejected": -381.33375984251967,
      "loss": 0.6224,
      "loss/base_kto": 3.3543832302093506,
      "metrics/advantage_var": 606.0228881835938,
      "regularization/lipschitz_norm": 1661.4329833984375,
      "regularization/mmd": 0.22887349128723145,
      "regularization/rkhs_norm": 272.4684753417969,
      "regularization/w1": 1.3956036567687988,
      "rewards/chosen": 0.2094398439392563,
      "rewards/margins": 0.7421111460738528,
      "rewards/rejected": -0.5326713021345965,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 21.802608489990234,
      "kl": 10.756336212158203,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35302135.22741433,
      "logits/rejected": -36563354.88401254,
      "logps/chosen": -476.05592873831773,
      "logps/rejected": -394.12382445141066,
      "loss": 0.6147,
      "loss/base_kto": 3.2948577404022217,
      "metrics/advantage_var": 664.4406127929688,
      "regularization/lipschitz_norm": 1653.151611328125,
      "regularization/mmd": 0.23432253301143646,
      "regularization/rkhs_norm": 278.9554138183594,
      "regularization/w1": 1.388647437095642,
      "rewards/chosen": 0.39019937010197625,
      "rewards/margins": 0.7748431639554489,
      "rewards/rejected": -0.38464379385347275,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 16.32952117919922,
      "kl": 14.149995803833008,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -34725425.33965245,
      "logits/rejected": -34796032.791344665,
      "logps/chosen": -470.53297788309635,
      "logps/rejected": -404.87053226429674,
      "loss": 0.624,
      "loss/base_kto": 3.3575081825256348,
      "metrics/advantage_var": 635.5565795898438,
      "regularization/lipschitz_norm": 1671.5345458984375,
      "regularization/mmd": 0.2306479513645172,
      "regularization/rkhs_norm": 274.5809020996094,
      "regularization/w1": 1.404089093208313,
      "rewards/chosen": 0.4045377372753752,
      "rewards/margins": 0.716084664348646,
      "rewards/rejected": -0.31154692707327086,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 14.87328815460205,
      "kl": 12.064723014831543,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35413856.5748503,
      "logits/rejected": -35093209.51633987,
      "logps/chosen": -477.73409431137725,
      "logps/rejected": -385.7476256127451,
      "loss": 0.6032,
      "loss/base_kto": 3.2641656398773193,
      "metrics/advantage_var": 568.94287109375,
      "regularization/lipschitz_norm": 1593.8173828125,
      "regularization/mmd": 0.22247324883937836,
      "regularization/rkhs_norm": 264.8490905761719,
      "regularization/w1": 1.3388065099716187,
      "rewards/chosen": 0.477585432772151,
      "rewards/margins": 0.8137759992900993,
      "rewards/rejected": -0.3361905665179483,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 26.503816604614258,
      "kl": 20.650165557861328,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -34678882.55037594,
      "logits/rejected": -36262544.02601626,
      "logps/chosen": -509.61287593984963,
      "logps/rejected": -363.3782774390244,
      "loss": 0.6127,
      "loss/base_kto": 3.330998182296753,
      "metrics/advantage_var": 603.1930541992188,
      "regularization/lipschitz_norm": 1596.963134765625,
      "regularization/mmd": 0.22921793162822723,
      "regularization/rkhs_norm": 272.8785095214844,
      "regularization/w1": 1.3414490222930908,
      "rewards/chosen": 0.6033118685385338,
      "rewards/margins": 0.714094249921918,
      "rewards/rejected": -0.11078238138338414,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 14.60009479522705,
      "kl": 15.492996215820312,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35416559.69426752,
      "logits/rejected": -35820883.2392638,
      "logps/chosen": -491.2158140923567,
      "logps/rejected": -376.93153278374234,
      "loss": 0.6125,
      "loss/base_kto": 3.222810745239258,
      "metrics/advantage_var": 688.3303833007812,
      "regularization/lipschitz_norm": 1711.7630615234375,
      "regularization/mmd": 0.23897671699523926,
      "regularization/rkhs_norm": 284.49609375,
      "regularization/w1": 1.4378809928894043,
      "rewards/chosen": 0.6643336107776423,
      "rewards/margins": 0.8913463854732904,
      "rewards/rejected": -0.227012774695648,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 19.115020751953125,
      "kl": 14.2912015914917,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -36602624.40699523,
      "logits/rejected": -38045623.64362519,
      "logps/chosen": -504.0611089030207,
      "logps/rejected": -366.39544930875576,
      "loss": 0.5952,
      "loss/base_kto": 3.26883864402771,
      "metrics/advantage_var": 534.1740112304688,
      "regularization/lipschitz_norm": 1521.0423583984375,
      "regularization/mmd": 0.2153901606798172,
      "regularization/rkhs_norm": 256.4168395996094,
      "regularization/w1": 1.2776756286621094,
      "rewards/chosen": 0.4696582497019078,
      "rewards/margins": 0.759111430955508,
      "rewards/rejected": -0.28945318125360026,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 13.423221588134766,
      "kl": 11.066169738769531,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34700488.971962616,
      "logits/rejected": -36084604.38871473,
      "logps/chosen": -470.60153816199374,
      "logps/rejected": -377.42750783699057,
      "loss": 0.6023,
      "loss/base_kto": 3.35111403465271,
      "metrics/advantage_var": 568.9474487304688,
      "regularization/lipschitz_norm": 1493.8076171875,
      "regularization/mmd": 0.21226052939891815,
      "regularization/rkhs_norm": 252.6911163330078,
      "regularization/w1": 1.2547982931137085,
      "rewards/chosen": 0.3544506417628018,
      "rewards/margins": 0.6904167460131446,
      "rewards/rejected": -0.33596610425034285,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 22.768653869628906,
      "kl": 8.966629981994629,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -35089128.426494345,
      "logits/rejected": -36029642.94099849,
      "logps/chosen": -471.4241720516963,
      "logps/rejected": -378.2768532526475,
      "loss": 0.5958,
      "loss/base_kto": 3.3208909034729004,
      "metrics/advantage_var": 496.6685485839844,
      "regularization/lipschitz_norm": 1471.4638671875,
      "regularization/mmd": 0.20927010476589203,
      "regularization/rkhs_norm": 249.13108825683594,
      "regularization/w1": 1.2360297441482544,
      "rewards/chosen": 0.318765027072395,
      "rewards/margins": 0.7169578324217709,
      "rewards/rejected": -0.39819280534937596,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 18.214309692382812,
      "kl": 8.788565635681152,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34333707.39586645,
      "logits/rejected": -35671485.14900154,
      "logps/chosen": -486.2912857710652,
      "logps/rejected": -376.6211357526882,
      "loss": 0.5974,
      "loss/base_kto": 3.2602412700653076,
      "metrics/advantage_var": 572.3546752929688,
      "regularization/lipschitz_norm": 1548.5469970703125,
      "regularization/mmd": 0.21799610555171967,
      "regularization/rkhs_norm": 259.5191955566406,
      "regularization/w1": 1.3007795810699463,
      "rewards/chosen": 0.3632470936009539,
      "rewards/margins": 0.8114203704406717,
      "rewards/rejected": -0.44817327683971775,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 19.935007095336914,
      "kl": 10.534090042114258,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34666082.429725364,
      "logits/rejected": -36020598.89863843,
      "logps/chosen": -483.5410945072698,
      "logps/rejected": -373.60795669440245,
      "loss": 0.6119,
      "loss/base_kto": 3.2726731300354004,
      "metrics/advantage_var": 650.3192749023438,
      "regularization/lipschitz_norm": 1655.721923828125,
      "regularization/mmd": 0.23140673339366913,
      "regularization/rkhs_norm": 275.4842224121094,
      "regularization/w1": 1.3908065557479858,
      "rewards/chosen": 0.39531334305425836,
      "rewards/margins": 0.7906553633862601,
      "rewards/rejected": -0.3953420203320017,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 18.949359893798828,
      "kl": 6.50945520401001,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -33921636.4244373,
      "logits/rejected": -35300809.5319149,
      "logps/chosen": -500.0674236334405,
      "logps/rejected": -346.7652688069909,
      "loss": 0.6003,
      "loss/base_kto": 3.265681505203247,
      "metrics/advantage_var": 583.9192504882812,
      "regularization/lipschitz_norm": 1563.104248046875,
      "regularization/mmd": 0.22389140725135803,
      "regularization/rkhs_norm": 266.53741455078125,
      "regularization/w1": 1.3130074739456177,
      "rewards/chosen": 0.3065753703715334,
      "rewards/margins": 0.8083490538673302,
      "rewards/rejected": -0.5017736834957969,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 17.29044532775879,
      "kl": 11.243669509887695,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -33964555.4115899,
      "logits/rejected": -36896923.202635914,
      "logps/chosen": -503.8058135215453,
      "logps/rejected": -364.0481363261944,
      "loss": 0.6147,
      "loss/base_kto": 3.266183614730835,
      "metrics/advantage_var": 679.18994140625,
      "regularization/lipschitz_norm": 1679.4996337890625,
      "regularization/mmd": 0.24045300483703613,
      "regularization/rkhs_norm": 286.25360107421875,
      "regularization/w1": 1.4107798337936401,
      "rewards/chosen": 0.5210249335415119,
      "rewards/margins": 0.8321277613897924,
      "rewards/rejected": -0.31110282784828047,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 18.00981330871582,
      "kl": 8.955912590026855,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34107723.763239875,
      "logits/rejected": -35853202.85893417,
      "logps/chosen": -464.2046339563863,
      "logps/rejected": -376.59502351097177,
      "loss": 0.6164,
      "loss/base_kto": 3.2926902770996094,
      "metrics/advantage_var": 973.8168334960938,
      "regularization/lipschitz_norm": 1667.1356201171875,
      "regularization/mmd": 0.23807401955127716,
      "regularization/rkhs_norm": 283.42144775390625,
      "regularization/w1": 1.4003939628601074,
      "rewards/chosen": 0.4130081699644665,
      "rewards/margins": 0.761906265361459,
      "rewards/rejected": -0.34889809539699257,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 20.81473731994629,
      "kl": 13.247878074645996,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34005499.123809524,
      "logits/rejected": -36260350.42461538,
      "logps/chosen": -508.7732638888889,
      "logps/rejected": -376.1651923076923,
      "loss": 0.6046,
      "loss/base_kto": 3.281367063522339,
      "metrics/advantage_var": 645.3544921875,
      "regularization/lipschitz_norm": 1575.2410888671875,
      "regularization/mmd": 0.232137069106102,
      "regularization/rkhs_norm": 276.3536071777344,
      "regularization/w1": 1.3232024908065796,
      "rewards/chosen": 0.45817687019469244,
      "rewards/margins": 0.7560264138395242,
      "rewards/rejected": -0.29784954364483174,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 21.589284896850586,
      "kl": 9.518244743347168,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -33770334.82988871,
      "logits/rejected": -35405043.809523806,
      "logps/chosen": -484.3436009538951,
      "logps/rejected": -356.1745871735791,
      "loss": 0.6135,
      "loss/base_kto": 3.3802649974823,
      "metrics/advantage_var": 610.53466796875,
      "regularization/lipschitz_norm": 1561.3052978515625,
      "regularization/mmd": 0.2164938896894455,
      "regularization/rkhs_norm": 257.7308044433594,
      "regularization/w1": 1.311496615409851,
      "rewards/chosen": 0.30456038385582274,
      "rewards/margins": 0.6745935894184787,
      "rewards/rejected": -0.370033205562656,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 19.481111526489258,
      "kl": 8.323575019836426,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34992025.1177394,
      "logits/rejected": -35880778.45101088,
      "logps/chosen": -473.9120879120879,
      "logps/rejected": -369.423551710731,
      "loss": 0.6238,
      "loss/base_kto": 3.34150767326355,
      "metrics/advantage_var": 633.9444580078125,
      "regularization/lipschitz_norm": 1686.3128662109375,
      "regularization/mmd": 0.2320529669523239,
      "regularization/rkhs_norm": 276.2535400390625,
      "regularization/w1": 1.416502833366394,
      "rewards/chosen": 0.33816430108326384,
      "rewards/margins": 0.7282971280114812,
      "rewards/rejected": -0.3901328269282173,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 15.917203903198242,
      "kl": 14.068951606750488,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33293421.20556414,
      "logits/rejected": -34839262.4328594,
      "logps/chosen": -470.7300521638331,
      "logps/rejected": -367.746198657188,
      "loss": 0.609,
      "loss/base_kto": 3.3531925678253174,
      "metrics/advantage_var": 532.55517578125,
      "regularization/lipschitz_norm": 1554.1697998046875,
      "regularization/mmd": 0.21358957886695862,
      "regularization/rkhs_norm": 254.27330017089844,
      "regularization/w1": 1.3055025339126587,
      "rewards/chosen": 0.414887983979545,
      "rewards/margins": 0.6779400098453379,
      "rewards/rejected": -0.26305202586579285,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 14.823602676391602,
      "kl": 10.622819900512695,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34610425.35064935,
      "logits/rejected": -35481131.18072289,
      "logps/chosen": -484.32112418831167,
      "logps/rejected": -387.072218561747,
      "loss": 0.598,
      "loss/base_kto": 3.270538330078125,
      "metrics/advantage_var": 601.8978881835938,
      "regularization/lipschitz_norm": 1537.4549560546875,
      "regularization/mmd": 0.22194503247737885,
      "regularization/rkhs_norm": 264.22027587890625,
      "regularization/w1": 1.2914621829986572,
      "rewards/chosen": 0.4453907756062297,
      "rewards/margins": 0.8184043983957275,
      "rewards/rejected": -0.37301362278949785,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 15.449972152709961,
      "kl": 6.428173065185547,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34536957.64777948,
      "logits/rejected": -35821342.62200957,
      "logps/chosen": -484.12557427258804,
      "logps/rejected": -375.1648225677831,
      "loss": 0.6036,
      "loss/base_kto": 3.31561541557312,
      "metrics/advantage_var": 600.2677612304688,
      "regularization/lipschitz_norm": 1538.493896484375,
      "regularization/mmd": 0.22086206078529358,
      "regularization/rkhs_norm": 262.9310302734375,
      "regularization/w1": 1.2923349142074585,
      "rewards/chosen": 0.31995327585871697,
      "rewards/margins": 0.7706536129051683,
      "rewards/rejected": -0.45070033704645135,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 20.079334259033203,
      "kl": 13.06872272491455,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -33904351.01840491,
      "logits/rejected": -35549324.22929937,
      "logps/chosen": -484.3055023006135,
      "logps/rejected": -382.65057722929936,
      "loss": 0.5937,
      "loss/base_kto": 3.2971794605255127,
      "metrics/advantage_var": 488.62335205078125,
      "regularization/lipschitz_norm": 1478.4853515625,
      "regularization/mmd": 0.21024754643440247,
      "regularization/rkhs_norm": 250.29470825195312,
      "regularization/w1": 1.2419277429580688,
      "rewards/chosen": 0.44681623962027894,
      "rewards/margins": 0.7049084759872799,
      "rewards/rejected": -0.2580922363670009,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 22.235336303710938,
      "kl": 14.48465633392334,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34517573.9277865,
      "logits/rejected": -35482412.19284604,
      "logps/chosen": -474.99617346938777,
      "logps/rejected": -393.0787811041991,
      "loss": 0.6048,
      "loss/base_kto": 3.3372159004211426,
      "metrics/advantage_var": 549.857421875,
      "regularization/lipschitz_norm": 1529.2337646484375,
      "regularization/mmd": 0.21660156548023224,
      "regularization/rkhs_norm": 257.8589782714844,
      "regularization/w1": 1.2845563888549805,
      "rewards/chosen": 0.3548815381395948,
      "rewards/margins": 0.709813264194829,
      "rewards/rejected": -0.35493172605523426,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 18.25558090209961,
      "kl": 10.816429138183594,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34766032.50678733,
      "logits/rejected": -35377693.04376013,
      "logps/chosen": -474.4428261689291,
      "logps/rejected": -350.2752228525122,
      "loss": 0.6127,
      "loss/base_kto": 3.3443429470062256,
      "metrics/advantage_var": 577.3859252929688,
      "regularization/lipschitz_norm": 1594.7008056640625,
      "regularization/mmd": 0.21744489669799805,
      "regularization/rkhs_norm": 258.86297607421875,
      "regularization/w1": 1.3395487070083618,
      "rewards/chosen": 0.39128736493094834,
      "rewards/margins": 0.7063987199628567,
      "rewards/rejected": -0.3151113550319084,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 16.051433563232422,
      "kl": 8.202681541442871,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -34601436.88676236,
      "logits/rejected": -35412253.402756505,
      "logps/chosen": -485.18620414673046,
      "logps/rejected": -359.6713246554364,
      "loss": 0.6037,
      "loss/base_kto": 3.3309967517852783,
      "metrics/advantage_var": 613.6174926757812,
      "regularization/lipschitz_norm": 1523.8358154296875,
      "regularization/mmd": 0.21878688037395477,
      "regularization/rkhs_norm": 260.4605712890625,
      "regularization/w1": 1.2800220251083374,
      "rewards/chosen": 0.375434692966881,
      "rewards/margins": 0.754728768574952,
      "rewards/rejected": -0.37929407560807094,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 20.815223693847656,
      "kl": 11.590879440307617,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -32725838.25499232,
      "logits/rejected": -33893693.45627981,
      "logps/chosen": -476.3749039938556,
      "logps/rejected": -352.7370578298887,
      "loss": 0.5987,
      "loss/base_kto": 3.261859178543091,
      "metrics/advantage_var": 574.8982543945312,
      "regularization/lipschitz_norm": 1554.3128662109375,
      "regularization/mmd": 0.22218680381774902,
      "regularization/rkhs_norm": 264.5081481933594,
      "regularization/w1": 1.305622935295105,
      "rewards/chosen": 0.47808050340221775,
      "rewards/margins": 0.8157705080754342,
      "rewards/rejected": -0.3376900046732164,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 18.088031768798828,
      "kl": 11.878664016723633,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34259679.50464396,
      "logits/rejected": -35375110.460567825,
      "logps/chosen": -469.6266447368421,
      "logps/rejected": -393.8571322949527,
      "loss": 0.6088,
      "loss/base_kto": 3.3371567726135254,
      "metrics/advantage_var": 551.409423828125,
      "regularization/lipschitz_norm": 1562.9835205078125,
      "regularization/mmd": 0.2202134132385254,
      "regularization/rkhs_norm": 262.1588439941406,
      "regularization/w1": 1.3129061460494995,
      "rewards/chosen": 0.4428910293815306,
      "rewards/margins": 0.7090021445573734,
      "rewards/rejected": -0.26611111517584285,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 21.351125717163086,
      "kl": 11.486132621765137,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -34189819.27384616,
      "logits/rejected": -35112222.06984127,
      "logps/chosen": -495.61538461538464,
      "logps/rejected": -391.8570436507936,
      "loss": 0.6038,
      "loss/base_kto": 3.287785768508911,
      "metrics/advantage_var": 554.07470703125,
      "regularization/lipschitz_norm": 1572.2630615234375,
      "regularization/mmd": 0.22212015092372894,
      "regularization/rkhs_norm": 264.4287109375,
      "regularization/w1": 1.320701003074646,
      "rewards/chosen": 0.42416682316706733,
      "rewards/margins": 0.7961476658785009,
      "rewards/rejected": -0.37198084271143356,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 16.664878845214844,
      "kl": 13.020438194274902,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -34901265.93630573,
      "logits/rejected": -37257592.93251534,
      "logps/chosen": -474.5579219745223,
      "logps/rejected": -370.8956096625767,
      "loss": 0.5925,
      "loss/base_kto": 3.231579303741455,
      "metrics/advantage_var": 534.8076171875,
      "regularization/lipschitz_norm": 1537.1300048828125,
      "regularization/mmd": 0.21699519455432892,
      "regularization/rkhs_norm": 258.3276062011719,
      "regularization/w1": 1.291189193725586,
      "rewards/chosen": 0.5680030774159036,
      "rewards/margins": 0.8133468532528716,
      "rewards/rejected": -0.24534377583696798,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 18.43654441833496,
      "kl": 13.266098022460938,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33752704.0,
      "logits/rejected": -34829000.64864865,
      "logps/chosen": -477.6653797238372,
      "logps/rejected": -375.7274070945946,
      "loss": 0.5963,
      "loss/base_kto": 3.269211530685425,
      "metrics/advantage_var": 561.1244506835938,
      "regularization/lipschitz_norm": 1529.3062744140625,
      "regularization/mmd": 0.21642573177814484,
      "regularization/rkhs_norm": 257.649658203125,
      "regularization/w1": 1.284617304801941,
      "rewards/chosen": 0.5461674180141715,
      "rewards/margins": 0.8230417540506474,
      "rewards/rejected": -0.27687433603647593,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 18.059795379638672,
      "kl": 15.921954154968262,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -35747996.28616852,
      "logits/rejected": -36362991.877112135,
      "logps/chosen": -485.92259538950714,
      "logps/rejected": -389.21193356374806,
      "loss": 0.5947,
      "loss/base_kto": 3.2594032287597656,
      "metrics/advantage_var": 546.0824584960938,
      "regularization/lipschitz_norm": 1527.6932373046875,
      "regularization/mmd": 0.21503780782222748,
      "regularization/rkhs_norm": 255.9973602294922,
      "regularization/w1": 1.2832622528076172,
      "rewards/chosen": 0.5455893317921552,
      "rewards/margins": 0.8161443035603484,
      "rewards/rejected": -0.27055497176819315,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 18.50031089782715,
      "kl": 11.904764175415039,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -33491071.163398694,
      "logits/rejected": -34679386.71471471,
      "logps/chosen": -488.5717422385621,
      "logps/rejected": -376.6834177927928,
      "loss": 0.5673,
      "loss/base_kto": 3.265979051589966,
      "metrics/advantage_var": 435.43310546875,
      "regularization/lipschitz_norm": 1286.2176513671875,
      "regularization/mmd": 0.191904217004776,
      "regularization/rkhs_norm": 228.45742797851562,
      "regularization/w1": 1.0804227590560913,
      "rewards/chosen": 0.41821299035564746,
      "rewards/margins": 0.7594718670301736,
      "rewards/rejected": -0.3412588766745261,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 18.242860794067383,
      "kl": 8.851688385009766,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -33713933.99113737,
      "logits/rejected": -34132294.049751244,
      "logps/chosen": -459.3748153618907,
      "logps/rejected": -383.0402155887231,
      "loss": 0.5647,
      "loss/base_kto": 3.2155025005340576,
      "metrics/advantage_var": 463.7978515625,
      "regularization/lipschitz_norm": 1314.51904296875,
      "regularization/mmd": 0.19804874062538147,
      "regularization/rkhs_norm": 235.77232360839844,
      "regularization/w1": 1.1041959524154663,
      "rewards/chosen": 0.43894311631623895,
      "rewards/margins": 0.8680322592247434,
      "rewards/rejected": -0.42908914290850436,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 14.94932746887207,
      "kl": 14.95853328704834,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -32988485.2327044,
      "logits/rejected": -35565593.440993786,
      "logps/chosen": -498.71555621069183,
      "logps/rejected": -362.0689538043478,
      "loss": 0.5556,
      "loss/base_kto": 3.146251916885376,
      "metrics/advantage_var": 433.46002197265625,
      "regularization/lipschitz_norm": 1312.9205322265625,
      "regularization/mmd": 0.19606690108776093,
      "regularization/rkhs_norm": 233.4129638671875,
      "regularization/w1": 1.1028531789779663,
      "rewards/chosen": 0.6007800791998329,
      "rewards/margins": 0.8850090971327957,
      "rewards/rejected": -0.2842290179329629,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 16.971399307250977,
      "kl": 17.06403350830078,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -33604675.43032159,
      "logits/rejected": -35877353.95215311,
      "logps/chosen": -482.5694391271057,
      "logps/rejected": -385.4413875598086,
      "loss": 0.556,
      "loss/base_kto": 3.1591320037841797,
      "metrics/advantage_var": 440.32861328125,
      "regularization/lipschitz_norm": 1301.7017822265625,
      "regularization/mmd": 0.19571247696876526,
      "regularization/rkhs_norm": 232.9910430908203,
      "regularization/w1": 1.093429446220398,
      "rewards/chosen": 0.6331080952245645,
      "rewards/margins": 0.8811377888767524,
      "rewards/rejected": -0.248029693652188,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 15.447782516479492,
      "kl": 14.525923728942871,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34164105.84615385,
      "logits/rejected": -34123059.359253496,
      "logps/chosen": -482.33330062794346,
      "logps/rejected": -374.1341125583204,
      "loss": 0.5665,
      "loss/base_kto": 3.206890821456909,
      "metrics/advantage_var": 435.4579162597656,
      "regularization/lipschitz_norm": 1344.9830322265625,
      "regularization/mmd": 0.1954275369644165,
      "regularization/rkhs_norm": 232.6518096923828,
      "regularization/w1": 1.1297857761383057,
      "rewards/chosen": 0.5523423894046802,
      "rewards/margins": 0.8329416245952541,
      "rewards/rejected": -0.28059923519057395,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 16.84161949157715,
      "kl": 14.022412300109863,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -33639749.81818182,
      "logits/rejected": -34408898.313253015,
      "logps/chosen": -472.515726461039,
      "logps/rejected": -370.6802757906627,
      "loss": 0.5624,
      "loss/base_kto": 3.1868457794189453,
      "metrics/advantage_var": 410.8081970214844,
      "regularization/lipschitz_norm": 1333.5665283203125,
      "regularization/mmd": 0.1918194591999054,
      "regularization/rkhs_norm": 228.3564910888672,
      "regularization/w1": 1.1201958656311035,
      "rewards/chosen": 0.5040625039633219,
      "rewards/margins": 0.8412723893422398,
      "rewards/rejected": -0.337209885378918,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 18.6634464263916,
      "kl": 12.651629447937012,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -33675174.95652174,
      "logits/rejected": -36437268.93081761,
      "logps/chosen": -489.09171195652175,
      "logps/rejected": -361.3551985062893,
      "loss": 0.5655,
      "loss/base_kto": 3.1931161880493164,
      "metrics/advantage_var": 440.7278747558594,
      "regularization/lipschitz_norm": 1350.2635498046875,
      "regularization/mmd": 0.19630078971385956,
      "regularization/rkhs_norm": 233.69140625,
      "regularization/w1": 1.1342214345932007,
      "rewards/chosen": 0.5269959254294448,
      "rewards/margins": 0.8607994501905248,
      "rewards/rejected": -0.33380352476108,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 17.905786514282227,
      "kl": 13.652798652648926,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -33972603.37833595,
      "logits/rejected": -36312052.852255054,
      "logps/chosen": -524.2321428571429,
      "logps/rejected": -371.7416650466563,
      "loss": 0.5547,
      "loss/base_kto": 3.133366346359253,
      "metrics/advantage_var": 459.79296875,
      "regularization/lipschitz_norm": 1314.7943115234375,
      "regularization/mmd": 0.19959811866283417,
      "regularization/rkhs_norm": 237.6168212890625,
      "regularization/w1": 1.1044272184371948,
      "rewards/chosen": 0.5978972286781299,
      "rewards/margins": 0.9215670656924305,
      "rewards/rejected": -0.32366983701430063,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 13.987251281738281,
      "kl": 13.4874906539917,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -33498057.28244275,
      "logits/rejected": -35428125.9008,
      "logps/chosen": -473.863358778626,
      "logps/rejected": -391.28785,
      "loss": 0.5616,
      "loss/base_kto": 3.1569135189056396,
      "metrics/advantage_var": 447.26593017578125,
      "regularization/lipschitz_norm": 1351.6658935546875,
      "regularization/mmd": 0.20029079914093018,
      "regularization/rkhs_norm": 238.44143676757812,
      "regularization/w1": 1.135399341583252,
      "rewards/chosen": 0.5698268278864503,
      "rewards/margins": 0.8870999968317628,
      "rewards/rejected": -0.3172731689453125,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 24.192644119262695,
      "kl": 8.75890827178955,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -33442620.58015267,
      "logits/rejected": -35251050.9056,
      "logps/chosen": -492.8916984732824,
      "logps/rejected": -365.31045,
      "loss": 0.5596,
      "loss/base_kto": 3.1864590644836426,
      "metrics/advantage_var": 441.95703125,
      "regularization/lipschitz_norm": 1303.9627685546875,
      "regularization/mmd": 0.19536092877388,
      "regularization/rkhs_norm": 232.572509765625,
      "regularization/w1": 1.095328688621521,
      "rewards/chosen": 0.5029198100548664,
      "rewards/margins": 0.8839807231408039,
      "rewards/rejected": -0.3810609130859375,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 13.031659126281738,
      "kl": 14.070281028747559,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -33599494.27258805,
      "logits/rejected": -34143909.767145135,
      "logps/chosen": -443.59355857580397,
      "logps/rejected": -372.660735645933,
      "loss": 0.551,
      "loss/base_kto": 3.18375563621521,
      "metrics/advantage_var": 358.3189392089844,
      "regularization/lipschitz_norm": 1239.441650390625,
      "regularization/mmd": 0.1828051209449768,
      "regularization/rkhs_norm": 217.62515258789062,
      "regularization/w1": 1.0411310195922852,
      "rewards/chosen": 0.5493479987198507,
      "rewards/margins": 0.8359266014878681,
      "rewards/rejected": -0.28657860276801733,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 16.931217193603516,
      "kl": 15.399487495422363,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34551310.35514019,
      "logits/rejected": -35247601.55485894,
      "logps/chosen": -487.81580996884736,
      "logps/rejected": -392.4702929075235,
      "loss": 0.559,
      "loss/base_kto": 3.1606571674346924,
      "metrics/advantage_var": 443.51495361328125,
      "regularization/lipschitz_norm": 1328.6239013671875,
      "regularization/mmd": 0.19551421701908112,
      "regularization/rkhs_norm": 232.75503540039062,
      "regularization/w1": 1.116044044494629,
      "rewards/chosen": 0.559925970630111,
      "rewards/margins": 0.8630176117602295,
      "rewards/rejected": -0.30309164113011855,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 18.24913215637207,
      "kl": 12.832463264465332,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -35501554.46345256,
      "logits/rejected": -34190289.381475665,
      "logps/chosen": -489.16854587869364,
      "logps/rejected": -367.9146144034537,
      "loss": 0.5611,
      "loss/base_kto": 3.201253652572632,
      "metrics/advantage_var": 400.7730407714844,
      "regularization/lipschitz_norm": 1306.8446044921875,
      "regularization/mmd": 0.18958069384098053,
      "regularization/rkhs_norm": 225.69131469726562,
      "regularization/w1": 1.0977494716644287,
      "rewards/chosen": 0.5137816528510157,
      "rewards/margins": 0.8428591653914886,
      "rewards/rejected": -0.3290775125404729,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 17.598134994506836,
      "kl": 11.90916633605957,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -33998500.716323294,
      "logits/rejected": -35748152.40677966,
      "logps/chosen": -464.4553783676704,
      "logps/rejected": -355.8242969953775,
      "loss": 0.5528,
      "loss/base_kto": 3.1689717769622803,
      "metrics/advantage_var": 385.138427734375,
      "regularization/lipschitz_norm": 1268.771728515625,
      "regularization/mmd": 0.18765221536159515,
      "regularization/rkhs_norm": 223.39549255371094,
      "regularization/w1": 1.0657681226730347,
      "rewards/chosen": 0.5189987158435272,
      "rewards/margins": 0.86975529785973,
      "rewards/rejected": -0.3507565820162028,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 21.09761619567871,
      "kl": 10.740870475769043,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -34534776.92259084,
      "logits/rejected": -35369858.96754251,
      "logps/chosen": -455.9173578199052,
      "logps/rejected": -396.7879153786708,
      "loss": 0.5521,
      "loss/base_kto": 3.195344924926758,
      "metrics/advantage_var": 408.6285705566406,
      "regularization/lipschitz_norm": 1231.0389404296875,
      "regularization/mmd": 0.1872580349445343,
      "regularization/rkhs_norm": 222.9262237548828,
      "regularization/w1": 1.034072756767273,
      "rewards/chosen": 0.4403811389989386,
      "rewards/margins": 0.8548842807529475,
      "rewards/rejected": -0.4145031417540089,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 17.74950408935547,
      "kl": 10.207816123962402,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35081552.02488336,
      "logits/rejected": -34916361.64521193,
      "logps/chosen": -471.7910672628305,
      "logps/rejected": -393.3380592621664,
      "loss": 0.5677,
      "loss/base_kto": 3.2030608654022217,
      "metrics/advantage_var": 436.63531494140625,
      "regularization/lipschitz_norm": 1359.7098388671875,
      "regularization/mmd": 0.19673502445220947,
      "regularization/rkhs_norm": 234.2083740234375,
      "regularization/w1": 1.1421562433242798,
      "rewards/chosen": 0.4566846405551371,
      "rewards/margins": 0.8489532301198692,
      "rewards/rejected": -0.39226858956473215,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 15.340740203857422,
      "kl": 11.331315994262695,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -31437489.046728972,
      "logits/rejected": -34931175.924764894,
      "logps/chosen": -467.008128894081,
      "logps/rejected": -386.536417515674,
      "loss": 0.5648,
      "loss/base_kto": 3.2069878578186035,
      "metrics/advantage_var": 402.5113220214844,
      "regularization/lipschitz_norm": 1330.2119140625,
      "regularization/mmd": 0.1936829388141632,
      "regularization/rkhs_norm": 230.5749053955078,
      "regularization/w1": 1.1173781156539917,
      "rewards/chosen": 0.4889666120582652,
      "rewards/margins": 0.8321314207561002,
      "rewards/rejected": -0.34316480869783506,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 17.722761154174805,
      "kl": 11.810046195983887,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -33378682.041733548,
      "logits/rejected": -35160207.39117199,
      "logps/chosen": -475.239165329053,
      "logps/rejected": -384.67006754185695,
      "loss": 0.5629,
      "loss/base_kto": 3.2113449573516846,
      "metrics/advantage_var": 413.6997985839844,
      "regularization/lipschitz_norm": 1309.820556640625,
      "regularization/mmd": 0.19192849099636078,
      "regularization/rkhs_norm": 228.4862823486328,
      "regularization/w1": 1.1002492904663086,
      "rewards/chosen": 0.45422147747792935,
      "rewards/margins": 0.8251634756853352,
      "rewards/rejected": -0.3709419982074058,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 14.714356422424316,
      "kl": 8.73776912689209,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34605199.56509299,
      "logits/rejected": -35368466.5060241,
      "logps/chosen": -482.5021012160229,
      "logps/rejected": -389.9052818416523,
      "loss": 0.5641,
      "loss/base_kto": 3.1989588737487793,
      "metrics/advantage_var": 478.6230163574219,
      "regularization/lipschitz_norm": 1323.5806884765625,
      "regularization/mmd": 0.20208410918712616,
      "regularization/rkhs_norm": 240.57632446289062,
      "regularization/w1": 1.1118078231811523,
      "rewards/chosen": 0.44558039107206726,
      "rewards/margins": 0.8767346257563401,
      "rewards/rejected": -0.43115423468427283,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 18.295270919799805,
      "kl": 13.62393856048584,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34508696.935064934,
      "logits/rejected": -36238909.686746985,
      "logps/chosen": -530.406047077922,
      "logps/rejected": -362.4706325301205,
      "loss": 0.5539,
      "loss/base_kto": 3.1301259994506836,
      "metrics/advantage_var": 445.87139892578125,
      "regularization/lipschitz_norm": 1313.7607421875,
      "regularization/mmd": 0.19779208302497864,
      "regularization/rkhs_norm": 235.46678161621094,
      "regularization/w1": 1.1035590171813965,
      "rewards/chosen": 0.5494497720297281,
      "rewards/margins": 0.9158441597000382,
      "rewards/rejected": -0.36639438767031013,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 17.067087173461914,
      "kl": 12.12473201751709,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -32465490.372239746,
      "logits/rejected": -34711282.526315786,
      "logps/chosen": -473.06111987381706,
      "logps/rejected": -357.53052438080493,
      "loss": 0.5545,
      "loss/base_kto": 3.1900439262390137,
      "metrics/advantage_var": 419.9374084472656,
      "regularization/lipschitz_norm": 1256.192626953125,
      "regularization/mmd": 0.19065237045288086,
      "regularization/rkhs_norm": 226.9671173095703,
      "regularization/w1": 1.055201768875122,
      "rewards/chosen": 0.512193649725207,
      "rewards/margins": 0.8468726775542148,
      "rewards/rejected": -0.3346790278290078,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 20.583120346069336,
      "kl": 12.695433616638184,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -34495723.56807512,
      "logits/rejected": -34164614.58970359,
      "logps/chosen": -461.06875978090767,
      "logps/rejected": -383.80703978159124,
      "loss": 0.5507,
      "loss/base_kto": 3.2116639614105225,
      "metrics/advantage_var": 362.9190673828125,
      "regularization/lipschitz_norm": 1205.908447265625,
      "regularization/mmd": 0.18123646080493927,
      "regularization/rkhs_norm": 215.7576904296875,
      "regularization/w1": 1.0129631757736206,
      "rewards/chosen": 0.4543029594122897,
      "rewards/margins": 0.8202510458220742,
      "rewards/rejected": -0.3659480864097845,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 16.20465660095215,
      "kl": 12.763381958007812,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -34506081.689291105,
      "logits/rejected": -35597977.51701783,
      "logps/chosen": -472.95512820512823,
      "logps/rejected": -369.6152248784441,
      "loss": 0.5539,
      "loss/base_kto": 3.140606641769409,
      "metrics/advantage_var": 419.79345703125,
      "regularization/lipschitz_norm": 1304.9725341796875,
      "regularization/mmd": 0.1942991465330124,
      "regularization/rkhs_norm": 231.30850219726562,
      "regularization/w1": 1.0961769819259644,
      "rewards/chosen": 0.5681098489200368,
      "rewards/margins": 0.9044474655298296,
      "rewards/rejected": -0.33633761660979283,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 16.107254028320312,
      "kl": 14.743173599243164,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -33537325.17647059,
      "logits/rejected": -35016920.42902208,
      "logps/chosen": -491.8492647058824,
      "logps/rejected": -352.57445287854887,
      "loss": 0.5587,
      "loss/base_kto": 3.1763598918914795,
      "metrics/advantage_var": 435.5909118652344,
      "regularization/lipschitz_norm": 1305.9234619140625,
      "regularization/mmd": 0.19652919471263885,
      "regularization/rkhs_norm": 233.9633331298828,
      "regularization/w1": 1.0969756841659546,
      "rewards/chosen": 0.5630381203288264,
      "rewards/margins": 0.8732626353592631,
      "rewards/rejected": -0.3102245150304367,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 21.481182098388672,
      "kl": 12.255326271057129,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -34864933.22255193,
      "logits/rejected": -34594741.650165014,
      "logps/chosen": -468.8024851632047,
      "logps/rejected": -391.146349009901,
      "loss": 0.5618,
      "loss/base_kto": 3.2057454586029053,
      "metrics/advantage_var": 422.9691467285156,
      "regularization/lipschitz_norm": 1303.2862548828125,
      "regularization/mmd": 0.194209486246109,
      "regularization/rkhs_norm": 231.20176696777344,
      "regularization/w1": 1.0947604179382324,
      "rewards/chosen": 0.5268299317855156,
      "rewards/margins": 0.8234247592675314,
      "rewards/rejected": -0.29659482748201577,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 14.624382972717285,
      "kl": 17.491533279418945,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34489799.84516129,
      "logits/rejected": -35485540.84848485,
      "logps/chosen": -480.8692540322581,
      "logps/rejected": -394.1696732954546,
      "loss": 0.5608,
      "loss/base_kto": 3.1768290996551514,
      "metrics/advantage_var": 424.4175720214844,
      "regularization/lipschitz_norm": 1324.0650634765625,
      "regularization/mmd": 0.1969681978225708,
      "regularization/rkhs_norm": 234.4859619140625,
      "regularization/w1": 1.1122145652770996,
      "rewards/chosen": 0.5332314768145161,
      "rewards/margins": 0.8496638054721866,
      "rewards/rejected": -0.31643232865767046,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 13.645767211914062,
      "kl": 13.762918472290039,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -34331302.43962848,
      "logits/rejected": -35779464.479495265,
      "logps/chosen": -468.5385061919505,
      "logps/rejected": -383.44178824921136,
      "loss": 0.5633,
      "loss/base_kto": 3.2114410400390625,
      "metrics/advantage_var": 440.93072509765625,
      "regularization/lipschitz_norm": 1309.8765869140625,
      "regularization/mmd": 0.1943253129720688,
      "regularization/rkhs_norm": 231.33963012695312,
      "regularization/w1": 1.1002962589263916,
      "rewards/chosen": 0.5259264305280089,
      "rewards/margins": 0.8284625856751402,
      "rewards/rejected": -0.30253615514713134,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 15.173432350158691,
      "kl": 14.218057632446289,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -32425249.391304348,
      "logits/rejected": -34971718.8427673,
      "logps/chosen": -503.3925659937888,
      "logps/rejected": -353.57033706761007,
      "loss": 0.5588,
      "loss/base_kto": 3.1940276622772217,
      "metrics/advantage_var": 400.8150329589844,
      "regularization/lipschitz_norm": 1293.98583984375,
      "regularization/mmd": 0.18915407359600067,
      "regularization/rkhs_norm": 225.1834259033203,
      "regularization/w1": 1.086948037147522,
      "rewards/chosen": 0.5563917456206328,
      "rewards/margins": 0.836030096005423,
      "rewards/rejected": -0.2796383503847902,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 14.659303665161133,
      "kl": 14.139813423156738,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -33578268.62619808,
      "logits/rejected": -36143078.94801223,
      "logps/chosen": -490.8719049520767,
      "logps/rejected": -378.82160263761466,
      "loss": 0.5556,
      "loss/base_kto": 3.136568546295166,
      "metrics/advantage_var": 440.8300476074219,
      "regularization/lipschitz_norm": 1323.760009765625,
      "regularization/mmd": 0.19664309918880463,
      "regularization/rkhs_norm": 234.0989227294922,
      "regularization/w1": 1.111958384513855,
      "rewards/chosen": 0.5801238526170627,
      "rewards/margins": 0.9139969364491298,
      "rewards/rejected": -0.3338730838320671,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 13.051947593688965,
      "kl": 17.49016761779785,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -33601833.18611988,
      "logits/rejected": -35005481.213622294,
      "logps/chosen": -489.55293769716087,
      "logps/rejected": -371.22288602941177,
      "loss": 0.5558,
      "loss/base_kto": 3.148218870162964,
      "metrics/advantage_var": 409.5477600097656,
      "regularization/lipschitz_norm": 1313.3740234375,
      "regularization/mmd": 0.19498609006404877,
      "regularization/rkhs_norm": 232.1262969970703,
      "regularization/w1": 1.1032341718673706,
      "rewards/chosen": 0.5496336157765674,
      "rewards/margins": 0.8620950200159976,
      "rewards/rejected": -0.31246140423943014,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 18.560632705688477,
      "kl": 18.206375122070312,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -33359756.2003231,
      "logits/rejected": -34790556.46596067,
      "logps/chosen": -497.78165387722134,
      "logps/rejected": -370.53711232980334,
      "loss": 0.5655,
      "loss/base_kto": 3.1513259410858154,
      "metrics/advantage_var": 684.828857421875,
      "regularization/lipschitz_norm": 1390.2115478515625,
      "regularization/mmd": 0.20519761741161346,
      "regularization/rkhs_norm": 244.2828826904297,
      "regularization/w1": 1.16777765750885,
      "rewards/chosen": 0.6012356315945577,
      "rewards/margins": 0.8369443097780629,
      "rewards/rejected": -0.2357086781835051,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 14.579899787902832,
      "kl": 15.151265144348145,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -34251067.59344263,
      "logits/rejected": -36550078.280597016,
      "logps/chosen": -491.97843237704916,
      "logps/rejected": -378.91923973880597,
      "loss": 0.5523,
      "loss/base_kto": 3.1356801986694336,
      "metrics/advantage_var": 447.5450134277344,
      "regularization/lipschitz_norm": 1292.7020263671875,
      "regularization/mmd": 0.19711998105049133,
      "regularization/rkhs_norm": 234.66665649414062,
      "regularization/w1": 1.0858697891235352,
      "rewards/chosen": 0.585163354091957,
      "rewards/margins": 0.8870121271942191,
      "rewards/rejected": -0.3018487731022621,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 17.755054473876953,
      "kl": 14.024542808532715,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -34082644.508856684,
      "logits/rejected": -36033904.26707132,
      "logps/chosen": -469.70632045088564,
      "logps/rejected": -371.58156297420334,
      "loss": 0.5474,
      "loss/base_kto": 3.205434560775757,
      "metrics/advantage_var": 382.1590270996094,
      "regularization/lipschitz_norm": 1180.8150634765625,
      "regularization/mmd": 0.18211029469966888,
      "regularization/rkhs_norm": 216.79795837402344,
      "regularization/w1": 0.9918846487998962,
      "rewards/chosen": 0.5159232958119464,
      "rewards/margins": 0.8224332833492899,
      "rewards/rejected": -0.3065099875373435,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 16.099609375,
      "kl": 16.29318618774414,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -34512733.717908084,
      "logits/rejected": -35207549.83050848,
      "logps/chosen": -467.43304278922346,
      "logps/rejected": -385.44551714175657,
      "loss": 0.556,
      "loss/base_kto": 3.1834332942962646,
      "metrics/advantage_var": 420.833251953125,
      "regularization/lipschitz_norm": 1275.8087158203125,
      "regularization/mmd": 0.1927143633365631,
      "regularization/rkhs_norm": 229.42185974121094,
      "regularization/w1": 1.0716793537139893,
      "rewards/chosen": 0.5675009169782587,
      "rewards/margins": 0.8446744221529866,
      "rewards/rejected": -0.27717350517472794,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 22.56321907043457,
      "kl": 15.587442398071289,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -34523900.67532468,
      "logits/rejected": -36209404.915662654,
      "logps/chosen": -493.51861810064935,
      "logps/rejected": -401.04593373493975,
      "loss": 0.5611,
      "loss/base_kto": 3.170428991317749,
      "metrics/advantage_var": 408.4739685058594,
      "regularization/lipschitz_norm": 1338.758056640625,
      "regularization/mmd": 0.19401060044765472,
      "regularization/rkhs_norm": 230.96499633789062,
      "regularization/w1": 1.1245567798614502,
      "rewards/chosen": 0.5469281085125812,
      "rewards/margins": 0.8478588670692688,
      "rewards/rejected": -0.30093075855668766,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 17.010610580444336,
      "kl": 13.440454483032227,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -33436187.28526646,
      "logits/rejected": -35349768.77258567,
      "logps/chosen": -464.2762539184953,
      "logps/rejected": -364.92752141744546,
      "loss": 0.5536,
      "loss/base_kto": 3.212057590484619,
      "metrics/advantage_var": 375.243408203125,
      "regularization/lipschitz_norm": 1230.5667724609375,
      "regularization/mmd": 0.18288172781467438,
      "regularization/rkhs_norm": 217.71633911132812,
      "regularization/w1": 1.033676028251648,
      "rewards/chosen": 0.49261713775347765,
      "rewards/margins": 0.7861720914504208,
      "rewards/rejected": -0.2935549536969431,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 17.789718627929688,
      "kl": 13.67638874053955,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34302437.80465116,
      "logits/rejected": -35579897.5496063,
      "logps/chosen": -493.7002906976744,
      "logps/rejected": -393.74451279527557,
      "loss": 0.571,
      "loss/base_kto": 3.2242684364318848,
      "metrics/advantage_var": 441.3744812011719,
      "regularization/lipschitz_norm": 1364.3621826171875,
      "regularization/mmd": 0.19798366725444794,
      "regularization/rkhs_norm": 235.69485473632812,
      "regularization/w1": 1.1460641622543335,
      "rewards/chosen": 0.5166667612948159,
      "rewards/margins": 0.8272236806630492,
      "rewards/rejected": -0.31055691936823326,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 12.054067611694336,
      "kl": 12.401045799255371,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -33550323.569044005,
      "logits/rejected": -33446841.095008053,
      "logps/chosen": -470.05202010622156,
      "logps/rejected": -343.1029589371981,
      "loss": 0.548,
      "loss/base_kto": 3.1565399169921875,
      "metrics/advantage_var": 404.8399963378906,
      "regularization/lipschitz_norm": 1240.16796875,
      "regularization/mmd": 0.18549861013889313,
      "regularization/rkhs_norm": 220.83169555664062,
      "regularization/w1": 1.0417410135269165,
      "rewards/chosen": 0.5369251118082559,
      "rewards/margins": 0.8793018207926309,
      "rewards/rejected": -0.342376708984375,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.975174191323546e+17,
      "train_loss": 0.5871853746074881,
      "train_runtime": 11079.3298,
      "train_samples_per_second": 13.378,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.975174191323546e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}