{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 15.571517944335938,
      "kl": 9.538911819458008,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36670855.86141732,
      "logits/rejected": -37428016.0248062,
      "logps/chosen": -466.5294291338583,
      "logps/rejected": -375.1725048449612,
      "loss": 0.5935,
      "loss/base_kto": 3.9124398231506348,
      "metrics/advantage_var": 246.2978973388672,
      "regularization/mmd": 0.8353201150894165,
      "regularization/rkhs_norm": 160.94802856445312,
      "rewards/chosen": 0.10693647730068898,
      "rewards/margins": 0.0829938729400346,
      "rewards/rejected": 0.023942604360654374,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 15.307286262512207,
      "kl": 13.87574291229248,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36265901.95377504,
      "logits/rejected": -37981487.467511885,
      "logps/chosen": -484.0126155624037,
      "logps/rejected": -376.0209488906498,
      "loss": 0.5871,
      "loss/base_kto": 3.873422622680664,
      "metrics/advantage_var": 193.25048828125,
      "regularization/mmd": 0.8232390284538269,
      "regularization/rkhs_norm": 158.6202392578125,
      "rewards/chosen": 0.183217758390312,
      "rewards/margins": 0.11884899200464566,
      "rewards/rejected": 0.06436876638566635,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 18.164213180541992,
      "kl": 5.212839603424072,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35386226.70387597,
      "logits/rejected": -35990765.05196851,
      "logps/chosen": -475.18464147286824,
      "logps/rejected": -359.7203248031496,
      "loss": 0.5808,
      "loss/base_kto": 3.8760621547698975,
      "metrics/advantage_var": 169.1094512939453,
      "regularization/mmd": 0.7703275084495544,
      "regularization/rkhs_norm": 148.4253387451172,
      "rewards/chosen": 0.020032202550607135,
      "rewards/margins": 0.1362361594549809,
      "rewards/rejected": -0.11620395690437377,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 16.888856887817383,
      "kl": 10.327300071716309,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37882630.66462481,
      "logits/rejected": -38822433.480063796,
      "logps/chosen": -491.083269525268,
      "logps/rejected": -376.98327850877195,
      "loss": 0.5776,
      "loss/base_kto": 3.8541600704193115,
      "metrics/advantage_var": 165.6694793701172,
      "regularization/mmd": 0.7665037512779236,
      "regularization/rkhs_norm": 147.68858337402344,
      "rewards/chosen": 0.15566759562236793,
      "rewards/margins": 0.13684607704529114,
      "rewards/rejected": 0.01882151857707679,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 18.683528900146484,
      "kl": 3.284613847732544,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36451557.45819398,
      "logits/rejected": -37508225.126099706,
      "logps/chosen": -508.8169941471572,
      "logps/rejected": -383.2355434384164,
      "loss": 0.5815,
      "loss/base_kto": 3.8655879497528076,
      "metrics/advantage_var": 177.95997619628906,
      "regularization/mmd": 0.7861188650131226,
      "regularization/rkhs_norm": 151.4679718017578,
      "rewards/chosen": -0.03227000411936272,
      "rewards/margins": 0.11076771918697435,
      "rewards/rejected": -0.14303772330633707,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 11.06960678100586,
      "kl": 11.392313003540039,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -35050959.12363067,
      "logits/rejected": -36804598.4149766,
      "logps/chosen": -455.0399061032864,
      "logps/rejected": -358.3792414196568,
      "loss": 0.5873,
      "loss/base_kto": 3.881218671798706,
      "metrics/advantage_var": 196.70751953125,
      "regularization/mmd": 0.8169800043106079,
      "regularization/rkhs_norm": 157.4142608642578,
      "rewards/chosen": 0.115661728550011,
      "rewards/margins": 0.10624722828225615,
      "rewards/rejected": 0.00941450026775485,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 13.950078010559082,
      "kl": 5.728837490081787,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36749391.50310559,
      "logits/rejected": -37346703.295597486,
      "logps/chosen": -500.00349378881987,
      "logps/rejected": -381.5553508254717,
      "loss": 0.5863,
      "loss/base_kto": 3.8131446838378906,
      "metrics/advantage_var": 231.8813934326172,
      "regularization/mmd": 0.8771948218345642,
      "regularization/rkhs_norm": 169.01634216308594,
      "rewards/chosen": 0.05716702952888442,
      "rewards/margins": 0.1870318241820512,
      "rewards/rejected": -0.12986479465316678,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 13.237503051757812,
      "kl": 8.087309837341309,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -35618828.52599388,
      "logits/rejected": -36276878.31309904,
      "logps/chosen": -473.78631498470946,
      "logps/rejected": -364.7073682108626,
      "loss": 0.5756,
      "loss/base_kto": 3.8134605884552,
      "metrics/advantage_var": 181.14297485351562,
      "regularization/mmd": 0.7911037802696228,
      "regularization/rkhs_norm": 152.428466796875,
      "rewards/chosen": 0.08350314569035801,
      "rewards/margins": 0.15822221822118368,
      "rewards/rejected": -0.07471907253082567,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 14.653188705444336,
      "kl": 15.084848403930664,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36715595.38650307,
      "logits/rejected": -37102428.94267516,
      "logps/chosen": -461.5396855828221,
      "logps/rejected": -364.40159733280257,
      "loss": 0.5793,
      "loss/base_kto": 3.7889657020568848,
      "metrics/advantage_var": 204.2191162109375,
      "regularization/mmd": 0.8457900881767273,
      "regularization/rkhs_norm": 162.96536254882812,
      "rewards/chosen": 0.30467050938518503,
      "rewards/margins": 0.18966756745177787,
      "rewards/rejected": 0.11500294193340714,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 18.993202209472656,
      "kl": 4.71709680557251,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -38046149.71076923,
      "logits/rejected": -37486377.44761905,
      "logps/chosen": -493.4542307692308,
      "logps/rejected": -403.08010912698415,
      "loss": 0.5922,
      "loss/base_kto": 3.8570022583007812,
      "metrics/advantage_var": 212.0679931640625,
      "regularization/mmd": 0.8802224397659302,
      "regularization/rkhs_norm": 169.59970092773438,
      "rewards/chosen": 0.022366710075965295,
      "rewards/margins": 0.14284531289404565,
      "rewards/rejected": -0.12047860281808036,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 13.953816413879395,
      "kl": 10.587577819824219,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37216005.22448979,
      "logits/rejected": -37344576.09953344,
      "logps/chosen": -492.0768249607535,
      "logps/rejected": -378.4935361586314,
      "loss": 0.5776,
      "loss/base_kto": 3.748526096343994,
      "metrics/advantage_var": 219.2888641357422,
      "regularization/mmd": 0.8723026514053345,
      "regularization/rkhs_norm": 168.07373046875,
      "rewards/chosen": 0.2412963340196355,
      "rewards/margins": 0.24378986309467665,
      "rewards/rejected": -0.0024935290750411586,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 14.870015144348145,
      "kl": 6.1637091636657715,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -38293815.508090615,
      "logits/rejected": -38198009.03927492,
      "logps/chosen": -481.43669093851133,
      "logps/rejected": -382.0043665030212,
      "loss": 0.5936,
      "loss/base_kto": 3.8108742237091064,
      "metrics/advantage_var": 250.5630340576172,
      "regularization/mmd": 0.937593936920166,
      "regularization/rkhs_norm": 180.6539306640625,
      "rewards/chosen": -0.0679220366246492,
      "rewards/margins": 0.1553817207807039,
      "rewards/rejected": -0.2233037574053531,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 14.09887981414795,
      "kl": 5.046163558959961,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -37280157.602605864,
      "logits/rejected": -38798062.318318315,
      "logps/chosen": -517.2627239413681,
      "logps/rejected": -373.02587743993996,
      "loss": 0.5833,
      "loss/base_kto": 3.738848924636841,
      "metrics/advantage_var": 246.8894805908203,
      "regularization/mmd": 0.9277191162109375,
      "regularization/rkhs_norm": 178.7512664794922,
      "rewards/chosen": 0.027881609888729134,
      "rewards/margins": 0.25842825761066984,
      "rewards/rejected": -0.2305466477219407,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 13.609427452087402,
      "kl": 2.7668838500976562,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -37892866.048,
      "logits/rejected": -39231050.25954199,
      "logps/chosen": -516.8509,
      "logps/rejected": -391.6226383587786,
      "loss": 0.5856,
      "loss/base_kto": 3.731579542160034,
      "metrics/advantage_var": 260.7747497558594,
      "regularization/mmd": 0.9528587460517883,
      "regularization/rkhs_norm": 183.5951385498047,
      "rewards/chosen": -0.024977178955078123,
      "rewards/margins": 0.26962076709543475,
      "rewards/rejected": -0.29459794605051287,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 16.254486083984375,
      "kl": 10.548303604125977,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36928320.0,
      "logits/rejected": -37830617.6,
      "logps/chosen": -495.545849609375,
      "logps/rejected": -350.2385498046875,
      "loss": 0.577,
      "loss/base_kto": 3.7107200622558594,
      "metrics/advantage_var": 235.25523376464844,
      "regularization/mmd": 0.9048828482627869,
      "regularization/rkhs_norm": 174.35121154785156,
      "rewards/chosen": 0.22926266193389894,
      "rewards/margins": 0.27641730308532714,
      "rewards/rejected": -0.04715464115142822,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 12.882833480834961,
      "kl": 7.569650173187256,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -36503129.24770642,
      "logits/rejected": -39226525.03514377,
      "logps/chosen": -489.1180237003058,
      "logps/rejected": -384.365964456869,
      "loss": 0.5802,
      "loss/base_kto": 3.788947343826294,
      "metrics/advantage_var": 206.8526153564453,
      "regularization/mmd": 0.8527728319168091,
      "regularization/rkhs_norm": 164.31076049804688,
      "rewards/chosen": 0.1230209770552609,
      "rewards/margins": 0.2077236120190064,
      "rewards/rejected": -0.08470263496374551,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 20.220701217651367,
      "kl": 12.921002388000488,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -38969773.67228178,
      "logits/rejected": -39175548.52950558,
      "logps/chosen": -491.80560872894335,
      "logps/rejected": -378.8329096889952,
      "loss": 0.5904,
      "loss/base_kto": 3.8205342292785645,
      "metrics/advantage_var": 231.05068969726562,
      "regularization/mmd": 0.9027809500694275,
      "regularization/rkhs_norm": 173.9462127685547,
      "rewards/chosen": 0.20647296058456163,
      "rewards/margins": 0.1769182832327515,
      "rewards/rejected": 0.029554677351810145,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 14.828634262084961,
      "kl": 4.833045959472656,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -36490830.76923077,
      "logits/rejected": -37599659.70731708,
      "logps/chosen": -496.5906951121795,
      "logps/rejected": -385.7433307926829,
      "loss": 0.5778,
      "loss/base_kto": 3.7393481731414795,
      "metrics/advantage_var": 224.12744140625,
      "regularization/mmd": 0.8829835057258606,
      "regularization/rkhs_norm": 170.13169860839844,
      "rewards/chosen": 0.005749409015362079,
      "rewards/margins": 0.24529392857936266,
      "rewards/rejected": -0.23954451956400058,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 13.264820098876953,
      "kl": 12.306333541870117,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35814038.49455676,
      "logits/rejected": -37112094.14128728,
      "logps/chosen": -483.37218118195955,
      "logps/rejected": -352.5729984301413,
      "loss": 0.5764,
      "loss/base_kto": 3.719007968902588,
      "metrics/advantage_var": 225.1729278564453,
      "regularization/mmd": 0.8922660946846008,
      "regularization/rkhs_norm": 171.9202423095703,
      "rewards/chosen": 0.22712349187159797,
      "rewards/margins": 0.24754749481393326,
      "rewards/rejected": -0.020424002942335286,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 14.280707359313965,
      "kl": 3.4532058238983154,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -35375217.42769231,
      "logits/rejected": -37906880.60952381,
      "logps/chosen": -508.5975480769231,
      "logps/rejected": -366.37363591269843,
      "loss": 0.5886,
      "loss/base_kto": 3.743018388748169,
      "metrics/advantage_var": 267.3111267089844,
      "regularization/mmd": 0.9658026099205017,
      "regularization/rkhs_norm": 186.089111328125,
      "rewards/chosen": 0.09360163175142729,
      "rewards/margins": 0.2721868174533122,
      "rewards/rejected": -0.17858518570188492,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 18.786409378051758,
      "kl": 7.645198822021484,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -36261818.469135806,
      "logits/rejected": -36752089.11392405,
      "logps/chosen": -511.5447048611111,
      "logps/rejected": -390.44521360759495,
      "loss": 0.5841,
      "loss/base_kto": 3.760416030883789,
      "metrics/advantage_var": 239.8439483642578,
      "regularization/mmd": 0.9127063751220703,
      "regularization/rkhs_norm": 175.858642578125,
      "rewards/chosen": 0.11989230874144001,
      "rewards/margins": 0.2177080094655951,
      "rewards/rejected": -0.09781570072415509,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 22.86400032043457,
      "kl": 7.945260524749756,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -36764127.38853503,
      "logits/rejected": -37778906.306748465,
      "logps/chosen": -489.1418192675159,
      "logps/rejected": -377.61277799079755,
      "loss": 0.5949,
      "loss/base_kto": 3.8170158863067627,
      "metrics/advantage_var": 266.03265380859375,
      "regularization/mmd": 0.9423967599868774,
      "regularization/rkhs_norm": 181.579345703125,
      "rewards/chosen": 0.05191928717740782,
      "rewards/margins": 0.16989548168783847,
      "rewards/rejected": -0.11797619451043065,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 16.6367244720459,
      "kl": 6.752360820770264,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35805528.523364484,
      "logits/rejected": -36757918.09404389,
      "logps/chosen": -476.0342192367601,
      "logps/rejected": -365.6922756661442,
      "loss": 0.5741,
      "loss/base_kto": 3.688326358795166,
      "metrics/advantage_var": 238.8223876953125,
      "regularization/mmd": 0.9042009711265564,
      "regularization/rkhs_norm": 174.21983337402344,
      "rewards/chosen": 0.15758937467295805,
      "rewards/margins": 0.3021852951519941,
      "rewards/rejected": -0.14459592047903605,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 17.560890197753906,
      "kl": 9.86482048034668,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -35733777.710691825,
      "logits/rejected": -36475484.22360248,
      "logps/chosen": -473.0930129716981,
      "logps/rejected": -381.6515916149068,
      "loss": 0.581,
      "loss/base_kto": 3.7668347358703613,
      "metrics/advantage_var": 221.4756317138672,
      "regularization/mmd": 0.8811573386192322,
      "regularization/rkhs_norm": 169.7798309326172,
      "rewards/chosen": 0.07517084085716391,
      "rewards/margins": 0.19803682575608061,
      "rewards/rejected": -0.12286598489891669,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 12.482565879821777,
      "kl": 16.427331924438477,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35223902.72782875,
      "logits/rejected": -35434437.111821085,
      "logps/chosen": -471.33720374617735,
      "logps/rejected": -371.2310553115016,
      "loss": 0.5728,
      "loss/base_kto": 3.7031056880950928,
      "metrics/advantage_var": 218.29490661621094,
      "regularization/mmd": 0.8796938061714172,
      "regularization/rkhs_norm": 169.4978485107422,
      "rewards/chosen": 0.2948127303283878,
      "rewards/margins": 0.28652009339611223,
      "rewards/rejected": 0.008292636932275547,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 19.035781860351562,
      "kl": 12.132369041442871,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36218929.46859904,
      "logits/rejected": -36743385.5417299,
      "logps/chosen": -460.4609500805153,
      "logps/rejected": -368.035375569044,
      "loss": 0.5769,
      "loss/base_kto": 3.7111313343048096,
      "metrics/advantage_var": 236.9758758544922,
      "regularization/mmd": 0.9044443964958191,
      "regularization/rkhs_norm": 174.26675415039062,
      "rewards/chosen": 0.15490230001305227,
      "rewards/margins": 0.26908599508696873,
      "rewards/rejected": -0.11418369507391644,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 16.07984161376953,
      "kl": 12.516695022583008,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36478836.01783061,
      "logits/rejected": -38328242.39868204,
      "logps/chosen": -502.0854383358098,
      "logps/rejected": -379.00779962932455,
      "loss": 0.5774,
      "loss/base_kto": 3.7033092975616455,
      "metrics/advantage_var": 255.03062438964844,
      "regularization/mmd": 0.9161953926086426,
      "regularization/rkhs_norm": 176.53091430664062,
      "rewards/chosen": 0.2599937586337992,
      "rewards/margins": 0.2863752622464369,
      "rewards/rejected": -0.026381503612637715,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 19.324129104614258,
      "kl": 9.51673412322998,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36140524.8,
      "logits/rejected": -37949420.8,
      "logps/chosen": -487.704736328125,
      "logps/rejected": -384.366455078125,
      "loss": 0.5837,
      "loss/base_kto": 3.736816883087158,
      "metrics/advantage_var": 251.3058624267578,
      "regularization/mmd": 0.9326297640800476,
      "regularization/rkhs_norm": 179.6974639892578,
      "rewards/chosen": 0.11220818758010864,
      "rewards/margins": 0.22923604249954224,
      "rewards/rejected": -0.1170278549194336,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 15.10949420928955,
      "kl": 7.855568885803223,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -35668759.127725855,
      "logits/rejected": -37519687.42319749,
      "logps/chosen": -473.3497371495327,
      "logps/rejected": -355.66881367554856,
      "loss": 0.5856,
      "loss/base_kto": 3.7436959743499756,
      "metrics/advantage_var": 280.8514709472656,
      "regularization/mmd": 0.9414481520652771,
      "regularization/rkhs_norm": 181.3965606689453,
      "rewards/chosen": 0.12582903710481164,
      "rewards/margins": 0.2381549625431933,
      "rewards/rejected": -0.11232592543838166,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 15.881586074829102,
      "kl": 13.679503440856934,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36254788.69266771,
      "logits/rejected": -38482833.427230045,
      "logps/chosen": -490.9366224648986,
      "logps/rejected": -356.3986697965571,
      "loss": 0.5881,
      "loss/base_kto": 3.7355542182922363,
      "metrics/advantage_var": 268.415771484375,
      "regularization/mmd": 0.9695192575454712,
      "regularization/rkhs_norm": 186.8052520751953,
      "rewards/chosen": 0.244347916006484,
      "rewards/margins": 0.23980130205953054,
      "rewards/rejected": 0.004546613946953476,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 14.873223304748535,
      "kl": 11.618257522583008,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -37046560.04945904,
      "logits/rejected": -37583783.02685624,
      "logps/chosen": -493.97198608964453,
      "logps/rejected": -364.0125394944708,
      "loss": 0.5783,
      "loss/base_kto": 3.7623164653778076,
      "metrics/advantage_var": 207.55296325683594,
      "regularization/mmd": 0.864192008972168,
      "regularization/rkhs_norm": 166.51101684570312,
      "rewards/chosen": 0.10723678848290186,
      "rewards/margins": 0.20618302687561574,
      "rewards/rejected": -0.09894623839271388,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 15.5108003616333,
      "kl": 15.256669998168945,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35425102.58966565,
      "logits/rejected": -37325359.74276527,
      "logps/chosen": -465.72791603343467,
      "logps/rejected": -376.3284515675241,
      "loss": 0.5781,
      "loss/base_kto": 3.698437452316284,
      "metrics/advantage_var": 239.4070587158203,
      "regularization/mmd": 0.9260997772216797,
      "regularization/rkhs_norm": 178.4392547607422,
      "rewards/chosen": 0.30974333047142144,
      "rewards/margins": 0.25810140013353394,
      "rewards/rejected": 0.051641930337887484,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 21.99886131286621,
      "kl": 7.631244659423828,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -35972634.0338983,
      "logits/rejected": -37005282.789223455,
      "logps/chosen": -483.33484206471496,
      "logps/rejected": -385.09984152139464,
      "loss": 0.5807,
      "loss/base_kto": 3.710510730743408,
      "metrics/advantage_var": 263.571044921875,
      "regularization/mmd": 0.9350858926773071,
      "regularization/rkhs_norm": 180.17068481445312,
      "rewards/chosen": 0.13394736361981174,
      "rewards/margins": 0.2811491677068264,
      "rewards/rejected": -0.14720180408701466,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 16.300622940063477,
      "kl": 17.796476364135742,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -34486187.74683544,
      "logits/rejected": -35447084.24691358,
      "logps/chosen": -471.895272943038,
      "logps/rejected": -369.14689429012344,
      "loss": 0.5823,
      "loss/base_kto": 3.7178444862365723,
      "metrics/advantage_var": 250.9248046875,
      "regularization/mmd": 0.9402643442153931,
      "regularization/rkhs_norm": 181.16845703125,
      "rewards/chosen": 0.26546992531305624,
      "rewards/margins": 0.22068698653244823,
      "rewards/rejected": 0.044782938780608,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 18.066877365112305,
      "kl": 13.168902397155762,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -35066087.122257054,
      "logits/rejected": -36025867.16510903,
      "logps/chosen": -466.1370493730407,
      "logps/rejected": -354.39505451713393,
      "loss": 0.5882,
      "loss/base_kto": 3.8002800941467285,
      "metrics/advantage_var": 233.62783813476562,
      "regularization/mmd": 0.9055353403091431,
      "regularization/rkhs_norm": 174.47695922851562,
      "rewards/chosen": 0.13914042505724677,
      "rewards/margins": 0.17859217866763788,
      "rewards/rejected": -0.03945175361039111,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 15.254158973693848,
      "kl": 5.895008563995361,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36683634.09133858,
      "logits/rejected": -36912262.94573643,
      "logps/chosen": -494.22691929133856,
      "logps/rejected": -368.55382751937987,
      "loss": 0.5798,
      "loss/base_kto": 3.7205963134765625,
      "metrics/advantage_var": 235.41050720214844,
      "regularization/mmd": 0.9181786775588989,
      "regularization/rkhs_norm": 176.9130401611328,
      "rewards/chosen": 0.07600033977838952,
      "rewards/margins": 0.26991844668548737,
      "rewards/rejected": -0.19391810690709788,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 19.226804733276367,
      "kl": 14.0280179977417,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -35125136.522550546,
      "logits/rejected": -37260231.73626374,
      "logps/chosen": -476.140066096423,
      "logps/rejected": -388.09259713500785,
      "loss": 0.576,
      "loss/base_kto": 3.7070813179016113,
      "metrics/advantage_var": 227.0825653076172,
      "regularization/mmd": 0.9011588096618652,
      "regularization/rkhs_norm": 173.63369750976562,
      "rewards/chosen": 0.18713741984745577,
      "rewards/margins": 0.2513987140821343,
      "rewards/rejected": -0.06426129423467855,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 13.873737335205078,
      "kl": 12.9469575881958,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -34681182.70759289,
      "logits/rejected": -35259774.64447806,
      "logps/chosen": -476.08678311793216,
      "logps/rejected": -360.8672465960666,
      "loss": 0.5726,
      "loss/base_kto": 3.683901309967041,
      "metrics/advantage_var": 236.10203552246094,
      "regularization/mmd": 0.8970678448677063,
      "regularization/rkhs_norm": 172.84544372558594,
      "rewards/chosen": 0.25019873401845216,
      "rewards/margins": 0.28955164155416097,
      "rewards/rejected": -0.039352907535708796,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 14.483478546142578,
      "kl": 7.9873785972595215,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -36476209.45904173,
      "logits/rejected": -37305842.20602219,
      "logps/chosen": -505.45319744976814,
      "logps/rejected": -376.2053783676704,
      "loss": 0.5841,
      "loss/base_kto": 3.5255863666534424,
      "metrics/advantage_var": 409.21575927734375,
      "regularization/mmd": 1.1474695205688477,
      "regularization/rkhs_norm": 221.09239196777344,
      "rewards/chosen": 0.23478642129087374,
      "rewards/margins": 0.5181088356830217,
      "rewards/rejected": -0.2833224143921479,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 11.03209114074707,
      "kl": 9.305520057678223,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -35367011.2,
      "logits/rejected": -36813363.2,
      "logps/chosen": -476.743505859375,
      "logps/rejected": -373.8144775390625,
      "loss": 0.5677,
      "loss/base_kto": 3.2241909503936768,
      "metrics/advantage_var": 555.5560913085938,
      "regularization/mmd": 1.3173788785934448,
      "regularization/rkhs_norm": 253.83023071289062,
      "rewards/chosen": 0.4903388977050781,
      "rewards/margins": 0.8678805112838746,
      "rewards/rejected": -0.3775416135787964,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 14.804159164428711,
      "kl": 13.329559326171875,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -35032850.638930164,
      "logits/rejected": -36724776.48764415,
      "logps/chosen": -484.1767737741456,
      "logps/rejected": -374.4861511532125,
      "loss": 0.5617,
      "loss/base_kto": 3.169060230255127,
      "metrics/advantage_var": 662.4751586914062,
      "regularization/mmd": 1.3242170810699463,
      "regularization/rkhs_norm": 255.14781188964844,
      "rewards/chosen": 0.5468870619253807,
      "rewards/margins": 0.9381494368787245,
      "rewards/rejected": -0.3912623749533438,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 10.822148323059082,
      "kl": 17.54160499572754,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36543880.101265825,
      "logits/rejected": -36830517.72839506,
      "logps/chosen": -472.810176028481,
      "logps/rejected": -364.96195023148147,
      "loss": 0.5547,
      "loss/base_kto": 3.14931321144104,
      "metrics/advantage_var": 500.598876953125,
      "regularization/mmd": 1.2882076501846313,
      "regularization/rkhs_norm": 248.20957946777344,
      "rewards/chosen": 0.6308662800849238,
      "rewards/margins": 0.9442392806184163,
      "rewards/rejected": -0.3133730005334925,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 11.496271133422852,
      "kl": 16.021413803100586,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -34998066.5775076,
      "logits/rejected": -37187412.78456592,
      "logps/chosen": -488.53804141337383,
      "logps/rejected": -373.63987138263667,
      "loss": 0.5752,
      "loss/base_kto": 3.2147233486175537,
      "metrics/advantage_var": 640.8287963867188,
      "regularization/mmd": 1.3867143392562866,
      "regularization/rkhs_norm": 267.189697265625,
      "rewards/chosen": 0.578866326700228,
      "rewards/margins": 0.8966528347513735,
      "rewards/rejected": -0.3177865080511455,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 17.11268424987793,
      "kl": 13.3150053024292,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -33797192.46769231,
      "logits/rejected": -36451581.56190476,
      "logps/chosen": -490.08384615384614,
      "logps/rejected": -360.4661706349206,
      "loss": 0.562,
      "loss/base_kto": 3.1174914836883545,
      "metrics/advantage_var": 571.0593872070312,
      "regularization/mmd": 1.378857970237732,
      "regularization/rkhs_norm": 265.6759033203125,
      "rewards/chosen": 0.6120276817908654,
      "rewards/margins": 0.9400860033046923,
      "rewards/rejected": -0.3280583215138269,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 11.711071968078613,
      "kl": 14.93961238861084,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -34375409.033846155,
      "logits/rejected": -36298563.45396826,
      "logps/chosen": -487.9735576923077,
      "logps/rejected": -358.37366071428573,
      "loss": 0.565,
      "loss/base_kto": 3.148419141769409,
      "metrics/advantage_var": 573.4874877929688,
      "regularization/mmd": 1.37118399143219,
      "regularization/rkhs_norm": 264.19732666015625,
      "rewards/chosen": 0.5945619553786058,
      "rewards/margins": 0.930740495805193,
      "rewards/rejected": -0.3361785404265873,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 14.130123138427734,
      "kl": 9.879761695861816,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34500308.29268292,
      "logits/rejected": -37149728.336842105,
      "logps/chosen": -474.78231707317076,
      "logps/rejected": -395.69891917293234,
      "loss": 0.5611,
      "loss/base_kto": 3.209912061691284,
      "metrics/advantage_var": 538.355712890625,
      "regularization/mmd": 1.2786645889282227,
      "regularization/rkhs_norm": 246.37083435058594,
      "rewards/chosen": 0.3794711725498603,
      "rewards/margins": 0.8551711035296535,
      "rewards/rejected": -0.47569993097979324,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 13.540090560913086,
      "kl": 5.989752292633057,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -34450699.50083195,
      "logits/rejected": -35318646.7628866,
      "logps/chosen": -492.9532029950083,
      "logps/rejected": -387.5404086892489,
      "loss": 0.5696,
      "loss/base_kto": 3.2576324939727783,
      "metrics/advantage_var": 524.0263061523438,
      "regularization/mmd": 1.2991846799850464,
      "regularization/rkhs_norm": 250.32461547851562,
      "rewards/chosen": 0.3055762165596402,
      "rewards/margins": 0.795464426041645,
      "rewards/rejected": -0.4898882094820048,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 10.40711784362793,
      "kl": 9.336389541625977,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -35843078.440251574,
      "logits/rejected": -37510671.900621116,
      "logps/chosen": -495.6369889937107,
      "logps/rejected": -369.85721564440996,
      "loss": 0.5743,
      "loss/base_kto": 3.193208932876587,
      "metrics/advantage_var": 623.3612060546875,
      "regularization/mmd": 1.401095986366272,
      "regularization/rkhs_norm": 269.9606628417969,
      "rewards/chosen": 0.383165827337301,
      "rewards/margins": 0.8949085554343992,
      "rewards/rejected": -0.5117427280970982,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 14.492140769958496,
      "kl": 11.189831733703613,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35120141.2987013,
      "logits/rejected": -37403182.26506024,
      "logps/chosen": -501.71935876623377,
      "logps/rejected": -403.6578501506024,
      "loss": 0.5654,
      "loss/base_kto": 3.1605660915374756,
      "metrics/advantage_var": 598.8322143554688,
      "regularization/mmd": 1.3623825311660767,
      "regularization/rkhs_norm": 262.5014343261719,
      "rewards/chosen": 0.4411466524198458,
      "rewards/margins": 0.9260114503567105,
      "rewards/rejected": -0.48486479793686466,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 14.895859718322754,
      "kl": 5.659992218017578,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -34601870.9202454,
      "logits/rejected": -35387046.318471335,
      "logps/chosen": -481.83291794478527,
      "logps/rejected": -385.2407941878981,
      "loss": 0.571,
      "loss/base_kto": 3.239588499069214,
      "metrics/advantage_var": 549.1942749023438,
      "regularization/mmd": 1.3287962675094604,
      "regularization/rkhs_norm": 256.0301208496094,
      "rewards/chosen": 0.3637069514924032,
      "rewards/margins": 0.8523527602043359,
      "rewards/rejected": -0.48864580871193275,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 14.638245582580566,
      "kl": 17.762792587280273,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35373422.15757576,
      "logits/rejected": -37802558.76129032,
      "logps/chosen": -491.02272727272725,
      "logps/rejected": -361.95703125,
      "loss": 0.5642,
      "loss/base_kto": 3.2021262645721436,
      "metrics/advantage_var": 522.0460205078125,
      "regularization/mmd": 1.3116295337677002,
      "regularization/rkhs_norm": 252.7224578857422,
      "rewards/chosen": 0.6330557158499053,
      "rewards/margins": 0.8610706064591542,
      "rewards/rejected": -0.22801489060924898,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 15.502215385437012,
      "kl": 12.964693069458008,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35737295.18139535,
      "logits/rejected": -37757989.08976378,
      "logps/chosen": -491.22093023255815,
      "logps/rejected": -369.54714566929135,
      "loss": 0.5683,
      "loss/base_kto": 3.2542412281036377,
      "metrics/advantage_var": 542.7702026367188,
      "regularization/mmd": 1.291926622390747,
      "regularization/rkhs_norm": 248.92613220214844,
      "rewards/chosen": 0.512014652222626,
      "rewards/margins": 0.8132746677553524,
      "rewards/rejected": -0.30126001553272636,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 13.078570365905762,
      "kl": 8.791579246520996,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35871753.69085173,
      "logits/rejected": -37875309.374613,
      "logps/chosen": -490.7807570977918,
      "logps/rejected": -354.6810903637771,
      "loss": 0.5692,
      "loss/base_kto": 3.150142192840576,
      "metrics/advantage_var": 969.3246459960938,
      "regularization/mmd": 1.403619647026062,
      "regularization/rkhs_norm": 270.44696044921875,
      "rewards/chosen": 0.48477798606319006,
      "rewards/margins": 0.8774978808030556,
      "rewards/rejected": -0.3927198947398655,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 10.45084285736084,
      "kl": 14.742172241210938,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -34520927.24646782,
      "logits/rejected": -35291588.27993779,
      "logps/chosen": -470.88000392464676,
      "logps/rejected": -392.1973658631415,
      "loss": 0.5621,
      "loss/base_kto": 3.209953784942627,
      "metrics/advantage_var": 507.49639892578125,
      "regularization/mmd": 1.2871525287628174,
      "regularization/rkhs_norm": 248.00625610351562,
      "rewards/chosen": 0.4810376129869015,
      "rewards/margins": 0.8173346081587644,
      "rewards/rejected": -0.33629699517186284,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 11.959739685058594,
      "kl": 10.621394157409668,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -36385666.51739788,
      "logits/rejected": -36326851.6187399,
      "logps/chosen": -468.92274962178516,
      "logps/rejected": -365.3991569063005,
      "loss": 0.5649,
      "loss/base_kto": 3.174787998199463,
      "metrics/advantage_var": 563.9235229492188,
      "regularization/mmd": 1.3442481756210327,
      "regularization/rkhs_norm": 259.00738525390625,
      "rewards/chosen": 0.5609467429941614,
      "rewards/margins": 0.9389897991062752,
      "rewards/rejected": -0.3780430561121138,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 15.207474708557129,
      "kl": 11.307988166809082,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -36114899.37242472,
      "logits/rejected": -37777389.06625578,
      "logps/chosen": -478.51361925515056,
      "logps/rejected": -401.49157357473035,
      "loss": 0.5768,
      "loss/base_kto": 3.2136363983154297,
      "metrics/advantage_var": 624.7119140625,
      "regularization/mmd": 1.4008406400680542,
      "regularization/rkhs_norm": 269.9114685058594,
      "rewards/chosen": 0.41678248192353406,
      "rewards/margins": 0.8572625775032721,
      "rewards/rejected": -0.44048009557973805,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 9.94320297241211,
      "kl": 22.93610954284668,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -36681682.776025236,
      "logits/rejected": -36529548.28482972,
      "logps/chosen": -471.74773264984225,
      "logps/rejected": -379.4496662151703,
      "loss": 0.5604,
      "loss/base_kto": 3.148862838745117,
      "metrics/advantage_var": 561.9501953125,
      "regularization/mmd": 1.3347347974777222,
      "regularization/rkhs_norm": 257.17431640625,
      "rewards/chosen": 0.6471507556806979,
      "rewards/margins": 0.9064340990164903,
      "rewards/rejected": -0.2592833433357924,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 11.12475872039795,
      "kl": 15.809300422668457,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34522435.368421055,
      "logits/rejected": -36607589.75394322,
      "logps/chosen": -505.07410990712077,
      "logps/rejected": -361.7321322949527,
      "loss": 0.5637,
      "loss/base_kto": 3.187002658843994,
      "metrics/advantage_var": 580.5582885742188,
      "regularization/mmd": 1.3225512504577637,
      "regularization/rkhs_norm": 254.82681274414062,
      "rewards/chosen": 0.5610994037828947,
      "rewards/margins": 0.9047024958200596,
      "rewards/rejected": -0.3436030920371648,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 15.943488121032715,
      "kl": 12.461132049560547,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -36111878.64935065,
      "logits/rejected": -36577773.4939759,
      "logps/chosen": -490.8744419642857,
      "logps/rejected": -393.5507577183735,
      "loss": 0.5517,
      "loss/base_kto": 3.2027695178985596,
      "metrics/advantage_var": 430.74444580078125,
      "regularization/mmd": 1.2105010747909546,
      "regularization/rkhs_norm": 233.23721313476562,
      "rewards/chosen": 0.4537938601010806,
      "rewards/margins": 0.8291066193166539,
      "rewards/rejected": -0.3753127592155732,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 19.020092010498047,
      "kl": 8.088566780090332,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -33876174.80130293,
      "logits/rejected": -35707811.74774775,
      "logps/chosen": -498.2650651465798,
      "logps/rejected": -364.7563344594595,
      "loss": 0.5748,
      "loss/base_kto": 3.2149109840393066,
      "metrics/advantage_var": 621.2483520507812,
      "regularization/mmd": 1.3834377527236938,
      "regularization/rkhs_norm": 266.5583190917969,
      "rewards/chosen": 0.383180301430171,
      "rewards/margins": 0.8799245432305651,
      "rewards/rejected": -0.49674424180039417,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 9.668807029724121,
      "kl": 14.72839641571045,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -36003460.62166405,
      "logits/rejected": -36985449.903576985,
      "logps/chosen": -470.77315541601257,
      "logps/rejected": -384.33944887247276,
      "loss": 0.5584,
      "loss/base_kto": 3.1495158672332764,
      "metrics/advantage_var": 551.7531127929688,
      "regularization/mmd": 1.3180047273635864,
      "regularization/rkhs_norm": 253.95083618164062,
      "rewards/chosen": 0.5112585909138295,
      "rewards/margins": 0.9135487097112148,
      "rewards/rejected": -0.4022901187973853,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 46.552490234375,
      "kl": 10.026848793029785,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34977852.43278688,
      "logits/rejected": -35357549.27761194,
      "logps/chosen": -452.09149590163935,
      "logps/rejected": -358.71585820895524,
      "loss": 0.569,
      "loss/base_kto": 3.180851697921753,
      "metrics/advantage_var": 694.2982177734375,
      "regularization/mmd": 1.3709758520126343,
      "regularization/rkhs_norm": 264.1571960449219,
      "rewards/chosen": 0.41222759309362195,
      "rewards/margins": 0.8554363387938319,
      "rewards/rejected": -0.4432087457002099,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 14.278236389160156,
      "kl": 14.127997398376465,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -34378638.901492536,
      "logits/rejected": -35840933.35081967,
      "logps/chosen": -477.92793843283584,
      "logps/rejected": -385.291931352459,
      "loss": 0.5693,
      "loss/base_kto": 3.2170772552490234,
      "metrics/advantage_var": 563.250244140625,
      "regularization/mmd": 1.3377063274383545,
      "regularization/rkhs_norm": 257.74688720703125,
      "rewards/chosen": 0.519941552005597,
      "rewards/margins": 0.8542181453233479,
      "rewards/rejected": -0.334276593317751,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 11.11368179321289,
      "kl": 18.76359748840332,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34820701.01737757,
      "logits/rejected": -34516112.02472952,
      "logps/chosen": -446.1201125592417,
      "logps/rejected": -352.66953245749613,
      "loss": 0.5618,
      "loss/base_kto": 3.2049381732940674,
      "metrics/advantage_var": 503.1366882324219,
      "regularization/mmd": 1.2894788980484009,
      "regularization/rkhs_norm": 248.4545135498047,
      "rewards/chosen": 0.6195756812796208,
      "rewards/margins": 0.8582325286128656,
      "rewards/rejected": -0.23865684733324477,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 11.907867431640625,
      "kl": 8.349406242370605,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33516892.870662462,
      "logits/rejected": -34123313.13931888,
      "logps/chosen": -463.0879337539432,
      "logps/rejected": -351.16802921826627,
      "loss": 0.5566,
      "loss/base_kto": 3.2693023681640625,
      "metrics/advantage_var": 432.8820495605469,
      "regularization/mmd": 1.1837867498397827,
      "regularization/rkhs_norm": 228.08995056152344,
      "rewards/chosen": 0.3898502434089733,
      "rewards/margins": 0.7927915427057273,
      "rewards/rejected": -0.40294129929675404,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 19.638782501220703,
      "kl": 10.250836372375488,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34289782.53376206,
      "logits/rejected": -35860732.10942249,
      "logps/chosen": -482.92614549839226,
      "logps/rejected": -387.59719319908817,
      "loss": 0.5681,
      "loss/base_kto": 3.2245795726776123,
      "metrics/advantage_var": 564.8365478515625,
      "regularization/mmd": 1.3204599618911743,
      "regularization/rkhs_norm": 254.4238739013672,
      "rewards/chosen": 0.40070024103980356,
      "rewards/margins": 0.8202269644203126,
      "rewards/rejected": -0.4195267233805091,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 13.80681324005127,
      "kl": 9.39556884765625,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -35517312.192481205,
      "logits/rejected": -36264885.07317073,
      "logps/chosen": -496.2058270676692,
      "logps/rejected": -379.8111788617886,
      "loss": 0.5666,
      "loss/base_kto": 3.2209389209747314,
      "metrics/advantage_var": 521.9708251953125,
      "regularization/mmd": 1.312155842781067,
      "regularization/rkhs_norm": 252.82388305664062,
      "rewards/chosen": 0.4581374548431626,
      "rewards/margins": 0.8154349769150631,
      "rewards/rejected": -0.3572975220719004,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 13.431105613708496,
      "kl": 11.714164733886719,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -35926315.251141556,
      "logits/rejected": -36230875.428571425,
      "logps/chosen": -507.626997716895,
      "logps/rejected": -377.32115268860355,
      "loss": 0.5645,
      "loss/base_kto": 3.2468605041503906,
      "metrics/advantage_var": 504.2196960449219,
      "regularization/mmd": 1.2694696187973022,
      "regularization/rkhs_norm": 244.59915161132812,
      "rewards/chosen": 0.5132574955259466,
      "rewards/margins": 0.7778916784268545,
      "rewards/rejected": -0.2646341829009079,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 11.592087745666504,
      "kl": 12.167591094970703,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34635315.2,
      "logits/rejected": -35708259.2,
      "logps/chosen": -470.50869140625,
      "logps/rejected": -374.138134765625,
      "loss": 0.5728,
      "loss/base_kto": 3.1993470191955566,
      "metrics/advantage_var": 635.9799194335938,
      "regularization/mmd": 1.3829056024551392,
      "regularization/rkhs_norm": 266.455810546875,
      "rewards/chosen": 0.4886659622192383,
      "rewards/margins": 0.8927920818328858,
      "rewards/rejected": -0.40412611961364747,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 16.77651023864746,
      "kl": 10.939823150634766,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34838283.80286168,
      "logits/rejected": -37276078.99231951,
      "logps/chosen": -466.2969495230525,
      "logps/rejected": -381.3138440860215,
      "loss": 0.5565,
      "loss/base_kto": 3.169776678085327,
      "metrics/advantage_var": 550.9974975585938,
      "regularization/mmd": 1.2822946310043335,
      "regularization/rkhs_norm": 247.07029724121094,
      "rewards/chosen": 0.4813091720799384,
      "rewards/margins": 0.9120447704082314,
      "rewards/rejected": -0.430735598328293,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 14.149236679077148,
      "kl": 10.224510192871094,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -36272958.52534562,
      "logits/rejected": -36580581.54531001,
      "logps/chosen": -474.05721966205834,
      "logps/rejected": -389.4228686406995,
      "loss": 0.5606,
      "loss/base_kto": 3.1691484451293945,
      "metrics/advantage_var": 546.0302734375,
      "regularization/mmd": 1.3153948783874512,
      "regularization/rkhs_norm": 253.4479522705078,
      "rewards/chosen": 0.49803646808395735,
      "rewards/margins": 0.9197941524974609,
      "rewards/rejected": -0.4217576844135036,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 12.257226943969727,
      "kl": 16.28075408935547,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -36167804.5184,
      "logits/rejected": -36414096.610687025,
      "logps/chosen": -457.8833,
      "logps/rejected": -380.27562022900764,
      "loss": 0.5725,
      "loss/base_kto": 3.246429443359375,
      "metrics/advantage_var": 582.8483276367188,
      "regularization/mmd": 1.3332990407943726,
      "regularization/rkhs_norm": 256.897705078125,
      "rewards/chosen": 0.505484912109375,
      "rewards/margins": 0.8409347365517652,
      "rewards/rejected": -0.33544982444239024,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 16.778846740722656,
      "kl": 12.644331932067871,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34810601.3164557,
      "logits/rejected": -36375523.55555555,
      "logps/chosen": -483.8537381329114,
      "logps/rejected": -379.38635706018516,
      "loss": 0.5672,
      "loss/base_kto": 3.255591630935669,
      "metrics/advantage_var": 539.2044677734375,
      "regularization/mmd": 1.2822104692459106,
      "regularization/rkhs_norm": 247.0540313720703,
      "rewards/chosen": 0.45679077921034417,
      "rewards/margins": 0.7998983559337962,
      "rewards/rejected": -0.343107576723452,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 10.346834182739258,
      "kl": 12.630763053894043,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -36129978.90795632,
      "logits/rejected": -36763853.12050078,
      "logps/chosen": -500.6096918876755,
      "logps/rejected": -381.59634194053206,
      "loss": 0.5595,
      "loss/base_kto": 3.1904945373535156,
      "metrics/advantage_var": 562.1464233398438,
      "regularization/mmd": 1.2853113412857056,
      "regularization/rkhs_norm": 247.65151977539062,
      "rewards/chosen": 0.5136202188810208,
      "rewards/margins": 0.8641619082806564,
      "rewards/rejected": -0.35054168939963565,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 15.149313926696777,
      "kl": 14.863423347473145,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35679784.358578056,
      "logits/rejected": -37479060.01895735,
      "logps/chosen": -484.0900792117465,
      "logps/rejected": -370.00083925750397,
      "loss": 0.5752,
      "loss/base_kto": 3.2095329761505127,
      "metrics/advantage_var": 710.6679077148438,
      "regularization/mmd": 1.3923863172531128,
      "regularization/rkhs_norm": 268.28253173828125,
      "rewards/chosen": 0.5660976309680497,
      "rewards/margins": 0.8733764735023605,
      "rewards/rejected": -0.3072788425343108,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 12.813681602478027,
      "kl": 12.500152587890625,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33813792.49535604,
      "logits/rejected": -35740904.58044164,
      "logps/chosen": -477.83813854489165,
      "logps/rejected": -390.43276813880124,
      "loss": 0.5578,
      "loss/base_kto": 3.1661529541015625,
      "metrics/advantage_var": 496.9759826660156,
      "regularization/mmd": 1.2963389158248901,
      "regularization/rkhs_norm": 249.77627563476562,
      "rewards/chosen": 0.5235694436465993,
      "rewards/margins": 0.8826646831354596,
      "rewards/rejected": -0.3590952394888604,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 13.28852653503418,
      "kl": 20.830835342407227,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -34747848.48192771,
      "logits/rejected": -35427341.2987013,
      "logps/chosen": -459.82516001506025,
      "logps/rejected": -382.30509334415586,
      "loss": 0.5688,
      "loss/base_kto": 3.1892714500427246,
      "metrics/advantage_var": 584.1787109375,
      "regularization/mmd": 1.3608081340789795,
      "regularization/rkhs_norm": 262.1980895996094,
      "rewards/chosen": 0.6476653685052711,
      "rewards/margins": 0.8488772730557987,
      "rewards/rejected": -0.2012119045505276,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 11.812023162841797,
      "kl": 17.362991333007812,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34660524.53042122,
      "logits/rejected": -36000039.78649922,
      "logps/chosen": -485.6875975039002,
      "logps/rejected": -363.7608418367347,
      "loss": 0.5396,
      "loss/base_kto": 3.0937564373016357,
      "metrics/advantage_var": 465.77960205078125,
      "regularization/mmd": 1.2231942415237427,
      "regularization/rkhs_norm": 235.6829071044922,
      "rewards/chosen": 0.686273155271914,
      "rewards/margins": 0.964398600818973,
      "rewards/rejected": -0.278125445547059,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 13.598008155822754,
      "kl": 12.415908813476562,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -35139206.4,
      "logits/rejected": -35781446.4,
      "logps/chosen": -468.61357421875,
      "logps/rejected": -395.6599853515625,
      "loss": 0.5442,
      "loss/base_kto": 3.1144955158233643,
      "metrics/advantage_var": 742.1534423828125,
      "regularization/mmd": 1.2394458055496216,
      "regularization/rkhs_norm": 238.814208984375,
      "rewards/chosen": 0.5431042671203613,
      "rewards/margins": 0.897245216369629,
      "rewards/rejected": -0.3541409492492676,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 10.59531021118164,
      "kl": 15.029504776000977,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -34486255.64217252,
      "logits/rejected": -36448938.666666664,
      "logps/chosen": -480.65460263578274,
      "logps/rejected": -389.69399847094803,
      "loss": 0.5218,
      "loss/base_kto": 3.02362322807312,
      "metrics/advantage_var": 382.8936462402344,
      "regularization/mmd": 1.1508859395980835,
      "regularization/rkhs_norm": 221.7506561279297,
      "rewards/chosen": 0.5973086616101737,
      "rewards/margins": 0.9933514183236676,
      "rewards/rejected": -0.39604275671349387,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 13.953946113586426,
      "kl": 10.442084312438965,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -35427420.50079239,
      "logits/rejected": -36524697.8366718,
      "logps/chosen": -498.19284865293184,
      "logps/rejected": -387.099311440678,
      "loss": 0.5242,
      "loss/base_kto": 3.0552632808685303,
      "metrics/advantage_var": 368.9991149902344,
      "regularization/mmd": 1.1386114358901978,
      "regularization/rkhs_norm": 219.3856201171875,
      "rewards/chosen": 0.5488621731378764,
      "rewards/margins": 0.965508839583851,
      "rewards/rejected": -0.4166466664459746,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 13.173537254333496,
      "kl": 13.659466743469238,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -35595632.145015106,
      "logits/rejected": -35881510.110032365,
      "logps/chosen": -505.66550226586105,
      "logps/rejected": -363.38647855987057,
      "loss": 0.5277,
      "loss/base_kto": 3.065598249435425,
      "metrics/advantage_var": 378.0130310058594,
      "regularization/mmd": 1.1558254957199097,
      "regularization/rkhs_norm": 222.702392578125,
      "rewards/chosen": 0.5783516229819675,
      "rewards/margins": 0.9472788471504672,
      "rewards/rejected": -0.3689272241684997,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 11.931377410888672,
      "kl": 10.723335266113281,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -34421327.92581144,
      "logits/rejected": -36117059.1342812,
      "logps/chosen": -458.4371136012365,
      "logps/rejected": -395.20228574249603,
      "loss": 0.5336,
      "loss/base_kto": 3.097973346710205,
      "metrics/advantage_var": 413.72314453125,
      "regularization/mmd": 1.1707619428634644,
      "regularization/rkhs_norm": 225.580322265625,
      "rewards/chosen": 0.5291910304168277,
      "rewards/margins": 0.9380135706518198,
      "rewards/rejected": -0.4088225402349921,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 10.052943229675293,
      "kl": 11.873779296875,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -35897449.66312595,
      "logits/rejected": -37358936.631239936,
      "logps/chosen": -480.7319328528073,
      "logps/rejected": -378.75747282608694,
      "loss": 0.525,
      "loss/base_kto": 3.065223217010498,
      "metrics/advantage_var": 365.3321228027344,
      "regularization/mmd": 1.1347383260726929,
      "regularization/rkhs_norm": 218.63938903808594,
      "rewards/chosen": 0.5719722116843703,
      "rewards/margins": 0.9500131581165361,
      "rewards/rejected": -0.37804094643216585,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 11.726381301879883,
      "kl": 12.947540283203125,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34227976.152866244,
      "logits/rejected": -36331997.44785276,
      "logps/chosen": -489.10091560509557,
      "logps/rejected": -359.5610381518405,
      "loss": 0.5288,
      "loss/base_kto": 3.102567434310913,
      "metrics/advantage_var": 368.41070556640625,
      "regularization/mmd": 1.1274595260620117,
      "regularization/rkhs_norm": 217.23692321777344,
      "rewards/chosen": 0.5757263572352707,
      "rewards/margins": 0.9222583714852945,
      "rewards/rejected": -0.34653201425002395,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 11.503512382507324,
      "kl": 13.424036026000977,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34937517.82716049,
      "logits/rejected": -35918433.215189874,
      "logps/chosen": -516.2775848765432,
      "logps/rejected": -386.8339102056962,
      "loss": 0.5292,
      "loss/base_kto": 3.084811210632324,
      "metrics/advantage_var": 368.7038879394531,
      "regularization/mmd": 1.148498773574829,
      "regularization/rkhs_norm": 221.2906951904297,
      "rewards/chosen": 0.5573343347620081,
      "rewards/margins": 0.9301758197010039,
      "rewards/rejected": -0.37284148493899577,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 7.271444797515869,
      "kl": 15.21033000946045,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -34667300.00618238,
      "logits/rejected": -35852171.52606635,
      "logps/chosen": -471.90813369397216,
      "logps/rejected": -373.40131319115324,
      "loss": 0.5246,
      "loss/base_kto": 3.0619912147521973,
      "metrics/advantage_var": 368.8582458496094,
      "regularization/mmd": 1.134839415550232,
      "regularization/rkhs_norm": 218.65884399414062,
      "rewards/chosen": 0.633236963192499,
      "rewards/margins": 0.976519605526869,
      "rewards/rejected": -0.34328264233437006,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 12.789861679077148,
      "kl": 11.074790954589844,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -35224399.11251981,
      "logits/rejected": -37280491.882896766,
      "logps/chosen": -475.46805665610145,
      "logps/rejected": -370.1136363636364,
      "loss": 0.5254,
      "loss/base_kto": 3.0926616191864014,
      "metrics/advantage_var": 383.5738525390625,
      "regularization/mmd": 1.1108897924423218,
      "regularization/rkhs_norm": 214.04429626464844,
      "rewards/chosen": 0.544474896462708,
      "rewards/margins": 0.9385429752076522,
      "rewards/rejected": -0.39406807874494415,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 9.382735252380371,
      "kl": 12.817514419555664,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34063755.90212072,
      "logits/rejected": -36143705.811094455,
      "logps/chosen": -472.3943719412724,
      "logps/rejected": -364.0275487256372,
      "loss": 0.5204,
      "loss/base_kto": 3.0654656887054443,
      "metrics/advantage_var": 342.53643798828125,
      "regularization/mmd": 1.098124384880066,
      "regularization/rkhs_norm": 211.5846710205078,
      "rewards/chosen": 0.5396833061977467,
      "rewards/margins": 0.9245164633496396,
      "rewards/rejected": -0.3848331571518928,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 10.125913619995117,
      "kl": 8.758418083190918,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34721252.71473354,
      "logits/rejected": -35780895.10280374,
      "logps/chosen": -482.1507641065831,
      "logps/rejected": -382.7769421728972,
      "loss": 0.5296,
      "loss/base_kto": 3.088571548461914,
      "metrics/advantage_var": 377.4120178222656,
      "regularization/mmd": 1.1480095386505127,
      "regularization/rkhs_norm": 221.19644165039062,
      "rewards/chosen": 0.5008663069865547,
      "rewards/margins": 0.9467187589646261,
      "rewards/rejected": -0.44585245197807144,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 10.050665855407715,
      "kl": 9.842815399169922,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -36315312.71494607,
      "logits/rejected": -36899818.90332805,
      "logps/chosen": -493.86951078582433,
      "logps/rejected": -396.1964391838352,
      "loss": 0.5311,
      "loss/base_kto": 3.056138753890991,
      "metrics/advantage_var": 409.9040222167969,
      "regularization/mmd": 1.1925348043441772,
      "regularization/rkhs_norm": 229.7755126953125,
      "rewards/chosen": 0.5343352848282213,
      "rewards/margins": 0.9792982110490018,
      "rewards/rejected": -0.4449629262207805,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 10.840620994567871,
      "kl": 13.91283893585205,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -33469242.273155417,
      "logits/rejected": -35610678.146189734,
      "logps/chosen": -486.79341640502355,
      "logps/rejected": -389.954801710731,
      "loss": 0.5374,
      "loss/base_kto": 3.0902035236358643,
      "metrics/advantage_var": 433.07330322265625,
      "regularization/mmd": 1.209328293800354,
      "regularization/rkhs_norm": 233.01123046875,
      "rewards/chosen": 0.5972605275583791,
      "rewards/margins": 0.9716978745433373,
      "rewards/rejected": -0.37443734698495823,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 8.343278884887695,
      "kl": 12.181922912597656,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -33700287.37864078,
      "logits/rejected": -34702329.81268882,
      "logps/chosen": -461.6702063106796,
      "logps/rejected": -376.6431032854985,
      "loss": 0.5248,
      "loss/base_kto": 3.1118314266204834,
      "metrics/advantage_var": 344.9362487792969,
      "regularization/mmd": 1.0864795446395874,
      "regularization/rkhs_norm": 209.3409423828125,
      "rewards/chosen": 0.5032996514082727,
      "rewards/margins": 0.8964650784404669,
      "rewards/rejected": -0.3931654270321941,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 9.558717727661133,
      "kl": 11.3455228805542,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -34344430.03508772,
      "logits/rejected": -36126331.70469799,
      "logps/chosen": -456.03682383040933,
      "logps/rejected": -363.75760276845637,
      "loss": 0.5249,
      "loss/base_kto": 3.1258723735809326,
      "metrics/advantage_var": 312.35992431640625,
      "regularization/mmd": 1.0734361410140991,
      "regularization/rkhs_norm": 206.8277587890625,
      "rewards/chosen": 0.5453165065475375,
      "rewards/margins": 0.8938541820338526,
      "rewards/rejected": -0.34853767548631504,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 8.555154800415039,
      "kl": 16.892311096191406,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35235413.19547658,
      "logits/rejected": -35423521.69440242,
      "logps/chosen": -476.1195476575121,
      "logps/rejected": -373.17310419818455,
      "loss": 0.5232,
      "loss/base_kto": 3.075972557067871,
      "metrics/advantage_var": 347.23553466796875,
      "regularization/mmd": 1.1093342304229736,
      "regularization/rkhs_norm": 213.74453735351562,
      "rewards/chosen": 0.5682295150633583,
      "rewards/margins": 0.8934559222515084,
      "rewards/rejected": -0.32522640718815005,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 13.49815559387207,
      "kl": 14.067483901977539,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34997816.88888889,
      "logits/rejected": -36328561.412520066,
      "logps/chosen": -481.9634703196347,
      "logps/rejected": -379.16470204654894,
      "loss": 0.5331,
      "loss/base_kto": 3.0894711017608643,
      "metrics/advantage_var": 401.41070556640625,
      "regularization/mmd": 1.1757266521453857,
      "regularization/rkhs_norm": 226.5369110107422,
      "rewards/chosen": 0.560076987906678,
      "rewards/margins": 0.9336913037172719,
      "rewards/rejected": -0.3736143158105939,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 14.797484397888184,
      "kl": 14.350302696228027,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -36001405.267651886,
      "logits/rejected": -36866606.54545455,
      "logps/chosen": -478.26472701149424,
      "logps/rejected": -379.9672596870343,
      "loss": 0.5311,
      "loss/base_kto": 3.098508834838867,
      "metrics/advantage_var": 388.4842224121094,
      "regularization/mmd": 1.15003502368927,
      "regularization/rkhs_norm": 221.5867156982422,
      "rewards/chosen": 0.5771887768274067,
      "rewards/margins": 0.9156153923761445,
      "rewards/rejected": -0.3384266155487379,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 8.583003044128418,
      "kl": 11.744126319885254,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34785482.27160494,
      "logits/rejected": -35954642.63291139,
      "logps/chosen": -503.56037808641975,
      "logps/rejected": -367.19931764240505,
      "loss": 0.5307,
      "loss/base_kto": 3.065383195877075,
      "metrics/advantage_var": 400.19427490234375,
      "regularization/mmd": 1.180153489112854,
      "regularization/rkhs_norm": 227.38987731933594,
      "rewards/chosen": 0.5370976954330633,
      "rewards/margins": 0.9577066988139473,
      "rewards/rejected": -0.4206090033808841,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 9.572432518005371,
      "kl": 13.085634231567383,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -34060697.6,
      "logits/rejected": -36925001.37322835,
      "logps/chosen": -482.0249031007752,
      "logps/rejected": -365.8713090551181,
      "loss": 0.5286,
      "loss/base_kto": 3.038194179534912,
      "metrics/advantage_var": 450.6354675292969,
      "regularization/mmd": 1.1905380487442017,
      "regularization/rkhs_norm": 229.39077758789062,
      "rewards/chosen": 0.5976823200551115,
      "rewards/margins": 1.0057066168532167,
      "rewards/rejected": -0.4080242967981053,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 8.293378829956055,
      "kl": 12.306334495544434,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -33829552.98765432,
      "logits/rejected": -35876339.037974685,
      "logps/chosen": -473.9470486111111,
      "logps/rejected": -360.09342859968353,
      "loss": 0.5253,
      "loss/base_kto": 3.0924527645111084,
      "metrics/advantage_var": 365.6189880371094,
      "regularization/mmd": 1.1100748777389526,
      "regularization/rkhs_norm": 213.8872528076172,
      "rewards/chosen": 0.5788733870894821,
      "rewards/margins": 0.941765272388199,
      "rewards/rejected": -0.36289188529871685,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 9.167913436889648,
      "kl": 10.900675773620605,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35208733.538461536,
      "logits/rejected": -36294346.92682927,
      "logps/chosen": -478.72000200320514,
      "logps/rejected": -382.6992663871951,
      "loss": 0.5297,
      "loss/base_kto": 3.080029010772705,
      "metrics/advantage_var": 387.3747253417969,
      "regularization/mmd": 1.1578792333602905,
      "regularization/rkhs_norm": 223.0980987548828,
      "rewards/chosen": 0.5555437528170072,
      "rewards/margins": 0.9557336875242766,
      "rewards/rejected": -0.40018993470726943,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 9.086071968078613,
      "kl": 10.250889778137207,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -34755010.81761006,
      "logits/rejected": -36850360.44720497,
      "logps/chosen": -499.359768081761,
      "logps/rejected": -360.2353454968944,
      "loss": 0.5276,
      "loss/base_kto": 3.0793330669403076,
      "metrics/advantage_var": 378.3411560058594,
      "regularization/mmd": 1.1418160200119019,
      "regularization/rkhs_norm": 220.00308227539062,
      "rewards/chosen": 0.5264859949267885,
      "rewards/margins": 0.9395983746761576,
      "rewards/rejected": -0.41311237974936915,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 10.675963401794434,
      "kl": 10.080180168151855,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34271557.178625956,
      "logits/rejected": -34417416.6016,
      "logps/chosen": -491.93396946564883,
      "logps/rejected": -365.516775,
      "loss": 0.5283,
      "loss/base_kto": 3.093646764755249,
      "metrics/advantage_var": 370.02587890625,
      "regularization/mmd": 1.1326717138290405,
      "regularization/rkhs_norm": 218.2411651611328,
      "rewards/chosen": 0.53616943359375,
      "rewards/margins": 0.9338271484375,
      "rewards/rejected": -0.39765771484375,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 10.079834938049316,
      "kl": 10.984450340270996,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -34487719.67022901,
      "logits/rejected": -35446177.792,
      "logps/chosen": -481.07776717557255,
      "logps/rejected": -361.6391,
      "loss": 0.5229,
      "loss/base_kto": 3.10184907913208,
      "metrics/advantage_var": 329.1188659667969,
      "regularization/mmd": 1.0813477039337158,
      "regularization/rkhs_norm": 208.35215759277344,
      "rewards/chosen": 0.5546003734792462,
      "rewards/margins": 0.9016446850026837,
      "rewards/rejected": -0.3470443115234375,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.892107009887695,
      "kl": 11.589664459228516,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -35282340.18237082,
      "logits/rejected": -36262250.18649518,
      "logps/chosen": -447.93531534954406,
      "logps/rejected": -357.67041800643085,
      "loss": 0.5278,
      "loss/base_kto": 3.116950035095215,
      "metrics/advantage_var": 351.3929443359375,
      "regularization/mmd": 1.105540156364441,
      "regularization/rkhs_norm": 213.01353454589844,
      "rewards/chosen": 0.5573816270451415,
      "rewards/margins": 0.9172011926004319,
      "rewards/rejected": -0.3598195655552904,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 12.195263862609863,
      "kl": 13.642374038696289,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -35447195.44227886,
      "logits/rejected": -38002390.65579119,
      "logps/chosen": -477.7605416041979,
      "logps/rejected": -384.575397634584,
      "loss": 0.5267,
      "loss/base_kto": 3.0857346057891846,
      "metrics/advantage_var": 371.4261779785156,
      "regularization/mmd": 1.1274933815002441,
      "regularization/rkhs_norm": 217.24343872070312,
      "rewards/chosen": 0.5979615630417213,
      "rewards/margins": 0.9352603561188013,
      "rewards/rejected": -0.33729879307707994,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 9.376483917236328,
      "kl": 15.698315620422363,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35467858.381954886,
      "logits/rejected": -35734383.141463414,
      "logps/chosen": -448.4790883458647,
      "logps/rejected": -394.01725101626016,
      "loss": 0.5296,
      "loss/base_kto": 3.108189105987549,
      "metrics/advantage_var": 376.1659851074219,
      "regularization/mmd": 1.1289169788360596,
      "regularization/rkhs_norm": 217.51773071289062,
      "rewards/chosen": 0.5993606452655075,
      "rewards/margins": 0.9191926050597148,
      "rewards/rejected": -0.3198319597942073,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 9.31547737121582,
      "kl": 13.520224571228027,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34428061.786163524,
      "logits/rejected": -35251276.322981365,
      "logps/chosen": -487.68391312893084,
      "logps/rejected": -390.0930221273292,
      "loss": 0.5323,
      "loss/base_kto": 3.0711276531219482,
      "metrics/advantage_var": 403.23406982421875,
      "regularization/mmd": 1.187195897102356,
      "regularization/rkhs_norm": 228.7467803955078,
      "rewards/chosen": 0.6404232768892492,
      "rewards/margins": 0.9392683666298839,
      "rewards/rejected": -0.29884508974063473,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 9.886720657348633,
      "kl": 13.426684379577637,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -33179677.210776545,
      "logits/rejected": -34135167.8027735,
      "logps/chosen": -485.5435816164818,
      "logps/rejected": -370.60925462249617,
      "loss": 0.5274,
      "loss/base_kto": 3.0824615955352783,
      "metrics/advantage_var": 390.8512878417969,
      "regularization/mmd": 1.136972188949585,
      "regularization/rkhs_norm": 219.0697784423828,
      "rewards/chosen": 0.570498362207186,
      "rewards/margins": 0.9357927378692623,
      "rewards/rejected": -0.3652943756620763,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 17.49335289001465,
      "kl": 14.857207298278809,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -33880664.83280757,
      "logits/rejected": -36222288.0495356,
      "logps/chosen": -486.42291009463725,
      "logps/rejected": -374.14601876934984,
      "loss": 0.5259,
      "loss/base_kto": 3.1070611476898193,
      "metrics/advantage_var": 340.4523620605469,
      "regularization/mmd": 1.1003831624984741,
      "regularization/rkhs_norm": 212.0198974609375,
      "rewards/chosen": 0.570823789771047,
      "rewards/margins": 0.8934752306542464,
      "rewards/rejected": -0.32265144088319947,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 9.30859661102295,
      "kl": 12.42811107635498,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -34244507.17538462,
      "logits/rejected": -35936613.58730159,
      "logps/chosen": -472.44153846153847,
      "logps/rejected": -379.248189484127,
      "loss": 0.5226,
      "loss/base_kto": 3.065871000289917,
      "metrics/advantage_var": 350.88531494140625,
      "regularization/mmd": 1.1149311065673828,
      "regularization/rkhs_norm": 214.8229522705078,
      "rewards/chosen": 0.5845942570612981,
      "rewards/margins": 0.9485169090485253,
      "rewards/rejected": -0.3639226519872272,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 10.817000389099121,
      "kl": 15.556986808776855,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -33087391.427642275,
      "logits/rejected": -35793348.715789475,
      "logps/chosen": -485.847256097561,
      "logps/rejected": -366.6436090225564,
      "loss": 0.5224,
      "loss/base_kto": 3.076166868209839,
      "metrics/advantage_var": 368.3682556152344,
      "regularization/mmd": 1.1028566360473633,
      "regularization/rkhs_norm": 212.4964599609375,
      "rewards/chosen": 0.5766742985423018,
      "rewards/margins": 0.9292531945126965,
      "rewards/rejected": -0.3525788959703947,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.865309715270996,
      "kl": 13.899508476257324,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -35078144.0,
      "logits/rejected": -36999473.15514593,
      "logps/chosen": -464.60597178060414,
      "logps/rejected": -379.12608006912444,
      "loss": 0.5262,
      "loss/base_kto": 3.1125473976135254,
      "metrics/advantage_var": 352.5972595214844,
      "regularization/mmd": 1.0970466136932373,
      "regularization/rkhs_norm": 211.37698364257812,
      "rewards/chosen": 0.5437903957639855,
      "rewards/margins": 0.9147923586396095,
      "rewards/rejected": -0.37100196287562404,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 15.151383399963379,
      "kl": 14.086331367492676,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -36041168.12678289,
      "logits/rejected": -36800209.06009245,
      "logps/chosen": -508.65674524564184,
      "logps/rejected": -352.3438944530046,
      "loss": 0.5273,
      "loss/base_kto": 3.064643621444702,
      "metrics/advantage_var": 382.234619140625,
      "regularization/mmd": 1.153996229171753,
      "regularization/rkhs_norm": 222.34996032714844,
      "rewards/chosen": 0.5957659012542096,
      "rewards/margins": 0.9596543993341884,
      "rewards/rejected": -0.3638884980799788,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 12.178041458129883,
      "kl": 15.788302421569824,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -35979590.77409162,
      "logits/rejected": -36445850.3122102,
      "logps/chosen": -487.91039691943126,
      "logps/rejected": -390.93416731066463,
      "loss": 0.5238,
      "loss/base_kto": 3.0518758296966553,
      "metrics/advantage_var": 377.42388916015625,
      "regularization/mmd": 1.1385143995285034,
      "regularization/rkhs_norm": 219.366943359375,
      "rewards/chosen": 0.5787471150523301,
      "rewards/margins": 0.9418147420054551,
      "rewards/rejected": -0.363067626953125,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.978126922009805e+17,
      "train_loss": 0.5585662964506265,
      "train_runtime": 11062.9611,
      "train_samples_per_second": 13.398,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.978126922009805e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}