{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 24.171916961669922,
      "kl": 17.971378326416016,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35316779.70731708,
      "logits/rejected": -36768866.461538464,
      "logps/chosen": -497.724228277439,
      "logps/rejected": -363.51056690705127,
      "loss": 0.6047,
      "loss/base_kto": 3.8854215145111084,
      "metrics/advantage_var": 204.70074462890625,
      "regularization/lipschitz_norm": 983.5292358398438,
      "regularization/w1": 0.952056348323822,
      "rewards/chosen": 0.28100246336401963,
      "rewards/margins": 0.11808828505372315,
      "rewards/rejected": 0.16291417831029648,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 22.6082820892334,
      "kl": 14.165916442871094,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37087235.13629403,
      "logits/rejected": -37193473.22488038,
      "logps/chosen": -483.79216117917304,
      "logps/rejected": -404.0459778708134,
      "loss": 0.6051,
      "loss/base_kto": 3.9289321899414062,
      "metrics/advantage_var": 192.48802185058594,
      "regularization/lipschitz_norm": 941.6502075195312,
      "regularization/w1": 0.911517322063446,
      "rewards/chosen": 0.13989972849169696,
      "rewards/margins": 0.07407873944570663,
      "rewards/rejected": 0.06582098904599033,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 26.396751403808594,
      "kl": 10.408247947692871,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35812877.32924962,
      "logits/rejected": -37073189.97129186,
      "logps/chosen": -493.8341787901991,
      "logps/rejected": -388.9180123604466,
      "loss": 0.5946,
      "loss/base_kto": 3.8426125049591064,
      "metrics/advantage_var": 201.38356018066406,
      "regularization/lipschitz_norm": 944.7931518554688,
      "regularization/w1": 0.9145596623420715,
      "rewards/chosen": 0.19716486558432236,
      "rewards/margins": 0.1451017434540455,
      "rewards/rejected": 0.05206312213027686,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 21.040809631347656,
      "kl": 3.604231357574463,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37383490.82033898,
      "logits/rejected": -37841154.22608696,
      "logps/chosen": -490.29480932203387,
      "logps/rejected": -394.2195652173913,
      "loss": 0.5971,
      "loss/base_kto": 3.840106248855591,
      "metrics/advantage_var": 192.21023559570312,
      "regularization/lipschitz_norm": 967.9828491210938,
      "regularization/w1": 0.9370073676109314,
      "rewards/chosen": -0.0987275980286679,
      "rewards/margins": 0.12818333310244623,
      "rewards/rejected": -0.22691093113111413,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 22.586702346801758,
      "kl": 7.076615333557129,
      "learning_rate": 9.9e-07,
      "logits/chosen": -35031971.77639752,
      "logits/rejected": -36753507.82389937,
      "logps/chosen": -482.4668575310559,
      "logps/rejected": -354.69388757861634,
      "loss": 0.6005,
      "loss/base_kto": 3.8660409450531006,
      "metrics/advantage_var": 210.2734832763672,
      "regularization/lipschitz_norm": 969.1918334960938,
      "regularization/w1": 0.9381775856018066,
      "rewards/chosen": 0.018050518095123102,
      "rewards/margins": 0.13388869489439345,
      "rewards/rejected": -0.11583817679927035,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 24.313669204711914,
      "kl": 13.761123657226562,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -36796998.33865815,
      "logits/rejected": -36799199.90214067,
      "logps/chosen": -469.5819688498403,
      "logps/rejected": -374.8200974770642,
      "loss": 0.6077,
      "loss/base_kto": 3.8909480571746826,
      "metrics/advantage_var": 223.26220703125,
      "regularization/lipschitz_norm": 1002.9113159179688,
      "regularization/w1": 0.9708181619644165,
      "rewards/chosen": 0.21156050221988568,
      "rewards/margins": 0.09433169383145272,
      "rewards/rejected": 0.11722880838843296,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 21.588844299316406,
      "kl": 9.700366020202637,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36744837.53204173,
      "logits/rejected": -38196094.52873563,
      "logps/chosen": -496.6974664679583,
      "logps/rejected": -368.6950687602627,
      "loss": 0.5943,
      "loss/base_kto": 3.799974203109741,
      "metrics/advantage_var": 212.4312744140625,
      "regularization/lipschitz_norm": 985.76953125,
      "regularization/w1": 0.9542250037193298,
      "rewards/chosen": 0.17170122945361982,
      "rewards/margins": 0.2000509132434039,
      "rewards/rejected": -0.028349683789784096,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 17.44001007080078,
      "kl": 4.787113666534424,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -36035446.15384615,
      "logits/rejected": -36212767.219512194,
      "logps/chosen": -477.42322716346155,
      "logps/rejected": -349.8604706554878,
      "loss": 0.5934,
      "loss/base_kto": 3.8393142223358154,
      "metrics/advantage_var": 205.9862060546875,
      "regularization/lipschitz_norm": 938.0133056640625,
      "regularization/w1": 0.9079969525337219,
      "rewards/chosen": 0.06676110854515663,
      "rewards/margins": 0.1606005620330181,
      "rewards/rejected": -0.09383945348786145,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 18.280359268188477,
      "kl": 1.6990368366241455,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -35199759.52721617,
      "logits/rejected": -36141772.96075353,
      "logps/chosen": -456.05355754276826,
      "logps/rejected": -362.4076236263736,
      "loss": 0.5899,
      "loss/base_kto": 3.8271141052246094,
      "metrics/advantage_var": 199.59747314453125,
      "regularization/lipschitz_norm": 921.5966796875,
      "regularization/w1": 0.8921055197715759,
      "rewards/chosen": -0.10501613587280084,
      "rewards/margins": 0.1806835857225681,
      "rewards/rejected": -0.28569972159536894,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 19.40365219116211,
      "kl": 9.957293510437012,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -34833159.70839936,
      "logits/rejected": -37904980.41294299,
      "logps/chosen": -497.16536251980983,
      "logps/rejected": -373.4144356702619,
      "loss": 0.5977,
      "loss/base_kto": 3.7903313636779785,
      "metrics/advantage_var": 230.1645050048828,
      "regularization/lipschitz_norm": 1023.9474487304688,
      "regularization/w1": 0.9911810159683228,
      "rewards/chosen": 0.16381330535453398,
      "rewards/margins": 0.2036897788739283,
      "rewards/rejected": -0.03987647351939432,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 24.024091720581055,
      "kl": 6.649497985839844,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -36357843.109004736,
      "logits/rejected": -36603995.79598145,
      "logps/chosen": -480.86868088467617,
      "logps/rejected": -375.7693199381762,
      "loss": 0.6005,
      "loss/base_kto": 3.7809770107269287,
      "metrics/advantage_var": 276.2771911621094,
      "regularization/lipschitz_norm": 1056.7017822265625,
      "regularization/w1": 1.0228872299194336,
      "rewards/chosen": 0.07966904240950212,
      "rewards/margins": 0.2167478833226409,
      "rewards/rejected": -0.13707884091313877,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 24.00461196899414,
      "kl": 8.452391624450684,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36556822.22635659,
      "logits/rejected": -36657490.64566929,
      "logps/chosen": -473.9613856589147,
      "logps/rejected": -386.12573818897636,
      "loss": 0.5986,
      "loss/base_kto": 3.76371169090271,
      "metrics/advantage_var": 254.5139923095703,
      "regularization/lipschitz_norm": 1058.958740234375,
      "regularization/w1": 1.0250719785690308,
      "rewards/chosen": 0.09946464124576065,
      "rewards/margins": 0.22167490207253232,
      "rewards/rejected": -0.12221026082677165,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 30.11318588256836,
      "kl": 13.7765531539917,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -35608964.93768546,
      "logits/rejected": -37566044.93729373,
      "logps/chosen": -479.20850333827894,
      "logps/rejected": -371.1122112211221,
      "loss": 0.5963,
      "loss/base_kto": 3.685382843017578,
      "metrics/advantage_var": 293.2428894042969,
      "regularization/lipschitz_norm": 1121.0953369140625,
      "regularization/w1": 1.0852203369140625,
      "rewards/chosen": 0.3413410582952754,
      "rewards/margins": 0.2959683660234542,
      "rewards/rejected": 0.045372692271821176,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 27.58708381652832,
      "kl": 22.29041862487793,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -37715201.9306184,
      "logits/rejected": -39577872.1815235,
      "logps/chosen": -489.99769042232276,
      "logps/rejected": -354.0433549432739,
      "loss": 0.6002,
      "loss/base_kto": 3.7528398036956787,
      "metrics/advantage_var": 287.2712097167969,
      "regularization/lipschitz_norm": 1083.8204345703125,
      "regularization/w1": 1.0491383075714111,
      "rewards/chosen": 0.3891103062694429,
      "rewards/margins": 0.2284970345159988,
      "rewards/rejected": 0.1606132717534441,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 18.10953712463379,
      "kl": 22.626684188842773,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36378617.52922591,
      "logits/rejected": -37791354.65842349,
      "logps/chosen": -491.140304107425,
      "logps/rejected": -381.2754781684699,
      "loss": 0.5995,
      "loss/base_kto": 3.845449924468994,
      "metrics/advantage_var": 213.772216796875,
      "regularization/lipschitz_norm": 982.02880859375,
      "regularization/w1": 0.9506039023399353,
      "rewards/chosen": 0.3383839443019969,
      "rewards/margins": 0.13321049314219688,
      "rewards/rejected": 0.20517345115980004,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 25.496047973632812,
      "kl": 14.383989334106445,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -37659781.835866265,
      "logits/rejected": -38620067.80707396,
      "logps/chosen": -494.45825417933133,
      "logps/rejected": -358.76914188102893,
      "loss": 0.584,
      "loss/base_kto": 3.7954728603363037,
      "metrics/advantage_var": 201.11444091796875,
      "regularization/lipschitz_norm": 905.8062744140625,
      "regularization/w1": 0.8768205046653748,
      "rewards/chosen": 0.24768414250985468,
      "rewards/margins": 0.17627670173554424,
      "rewards/rejected": 0.07140744077431044,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 22.850505828857422,
      "kl": 9.190020561218262,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36118437.97802198,
      "logits/rejected": -37721006.7807154,
      "logps/chosen": -476.0750588697017,
      "logps/rejected": -363.1827614696734,
      "loss": 0.5889,
      "loss/base_kto": 3.759268283843994,
      "metrics/advantage_var": 211.2550048828125,
      "regularization/lipschitz_norm": 983.0006103515625,
      "regularization/w1": 0.951544463634491,
      "rewards/chosen": 0.12163652692522321,
      "rewards/margins": 0.23103632542377733,
      "rewards/rejected": -0.10939979849855413,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 21.94005012512207,
      "kl": 14.6563081741333,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -36833775.43297381,
      "logits/rejected": -37856299.81616482,
      "logps/chosen": -486.2754718798151,
      "logps/rejected": -386.76116778922346,
      "loss": 0.5988,
      "loss/base_kto": 3.7516915798187256,
      "metrics/advantage_var": 250.81607055664062,
      "regularization/lipschitz_norm": 1073.2459716796875,
      "regularization/w1": 1.038901925086975,
      "rewards/chosen": 0.2067007761339561,
      "rewards/margins": 0.20732436276141453,
      "rewards/rejected": -0.0006235866274584303,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 25.312530517578125,
      "kl": 15.028098106384277,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35418189.45839637,
      "logits/rejected": -38815198.08723748,
      "logps/chosen": -501.94752269288955,
      "logps/rejected": -371.9616316639742,
      "loss": 0.5972,
      "loss/base_kto": 3.7215523719787598,
      "metrics/advantage_var": 279.48394775390625,
      "regularization/lipschitz_norm": 1090.6280517578125,
      "regularization/w1": 1.0557279586791992,
      "rewards/chosen": 0.30185509016582357,
      "rewards/margins": 0.23913481142743503,
      "rewards/rejected": 0.06272027873838854,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 21.098722457885742,
      "kl": 15.414970397949219,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36133701.67340591,
      "logits/rejected": -37539753.19309262,
      "logps/chosen": -461.15697900466563,
      "logps/rejected": -361.7773253532182,
      "loss": 0.5973,
      "loss/base_kto": 3.795140504837036,
      "metrics/advantage_var": 254.8007049560547,
      "regularization/lipschitz_norm": 1016.0383911132812,
      "regularization/w1": 0.9835252165794373,
      "rewards/chosen": 0.2212227543998469,
      "rewards/margins": 0.17877199477564715,
      "rewards/rejected": 0.04245075962419974,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 26.532087326049805,
      "kl": 10.874388694763184,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -35524204.45320197,
      "logits/rejected": -37241613.353204176,
      "logps/chosen": -472.52324507389164,
      "logps/rejected": -351.55993852459017,
      "loss": 0.5838,
      "loss/base_kto": 3.7438149452209473,
      "metrics/advantage_var": 227.2717742919922,
      "regularization/lipschitz_norm": 956.939453125,
      "regularization/w1": 0.9263173937797546,
      "rewards/chosen": 0.21551162895114942,
      "rewards/margins": 0.22224590135796485,
      "rewards/rejected": -0.006734272406815417,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 27.539342880249023,
      "kl": 6.9973602294921875,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -37273823.41818182,
      "logits/rejected": -37989006.03870968,
      "logps/chosen": -509.3034564393939,
      "logps/rejected": -372.19339717741934,
      "loss": 0.5934,
      "loss/base_kto": 3.7677838802337646,
      "metrics/advantage_var": 236.2348175048828,
      "regularization/lipschitz_norm": 1011.4757690429688,
      "regularization/w1": 0.979108452796936,
      "rewards/chosen": 0.08697488958185369,
      "rewards/margins": 0.22105295846888745,
      "rewards/rejected": -0.13407806888703377,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 22.417484283447266,
      "kl": 10.195749282836914,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -37403931.01507538,
      "logits/rejected": -38144776.62079063,
      "logps/chosen": -471.87646566164153,
      "logps/rejected": -401.33437042459735,
      "loss": 0.5911,
      "loss/base_kto": 3.7370898723602295,
      "metrics/advantage_var": 287.5673828125,
      "regularization/lipschitz_norm": 1024.3519287109375,
      "regularization/w1": 0.9915726780891418,
      "rewards/chosen": 0.08149675148815366,
      "rewards/margins": 0.22253619628449106,
      "rewards/rejected": -0.1410394447963374,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 20.285343170166016,
      "kl": 8.333754539489746,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36534663.29738059,
      "logits/rejected": -37380453.02060222,
      "logps/chosen": -496.8182781201849,
      "logps/rejected": -372.7764461172742,
      "loss": 0.5937,
      "loss/base_kto": 3.729773759841919,
      "metrics/advantage_var": 263.1363220214844,
      "regularization/lipschitz_norm": 1053.1697998046875,
      "regularization/w1": 1.0194681882858276,
      "rewards/chosen": 0.08392554989948478,
      "rewards/margins": 0.25394270164894,
      "rewards/rejected": -0.17001715174945523,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 22.549976348876953,
      "kl": 7.499153137207031,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -34990649.642384104,
      "logits/rejected": -36608830.10650887,
      "logps/chosen": -490.3294701986755,
      "logps/rejected": -375.35470599112426,
      "loss": 0.5903,
      "loss/base_kto": 3.681213855743408,
      "metrics/advantage_var": 266.59619140625,
      "regularization/lipschitz_norm": 1075.8951416015625,
      "regularization/w1": 1.0414663553237915,
      "rewards/chosen": 0.022774854243196398,
      "rewards/margins": 0.27699002700519254,
      "rewards/rejected": -0.2542151727619961,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 16.535247802734375,
      "kl": 14.017804145812988,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36697649.34939759,
      "logits/rejected": -36691559.064935066,
      "logps/chosen": -508.0226374246988,
      "logps/rejected": -376.5061891233766,
      "loss": 0.5973,
      "loss/base_kto": 3.684291124343872,
      "metrics/advantage_var": 312.0481872558594,
      "regularization/lipschitz_norm": 1130.4091796875,
      "regularization/w1": 1.0942360162734985,
      "rewards/chosen": 0.28289523756647683,
      "rewards/margins": 0.29742380625540205,
      "rewards/rejected": -0.01452856868892521,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 18.890146255493164,
      "kl": 14.805468559265137,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37082898.314785376,
      "logits/rejected": -37032077.56682028,
      "logps/chosen": -495.70160969793324,
      "logps/rejected": -368.2362471198157,
      "loss": 0.5962,
      "loss/base_kto": 3.6946403980255127,
      "metrics/advantage_var": 306.1405334472656,
      "regularization/lipschitz_norm": 1110.30419921875,
      "regularization/w1": 1.0747743844985962,
      "rewards/chosen": 0.2752458241861462,
      "rewards/margins": 0.26083179426635633,
      "rewards/rejected": 0.014414029919789866,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 20.330968856811523,
      "kl": 5.8462653160095215,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35395339.02392344,
      "logits/rejected": -36207197.30474732,
      "logps/chosen": -481.1236044657097,
      "logps/rejected": -363.46815179938744,
      "loss": 0.5917,
      "loss/base_kto": 3.6927356719970703,
      "metrics/advantage_var": 263.9264831542969,
      "regularization/lipschitz_norm": 1075.0716552734375,
      "regularization/w1": 1.040669322013855,
      "rewards/chosen": -0.015467316720284154,
      "rewards/margins": 0.2952021894052901,
      "rewards/rejected": -0.31066950612557426,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 24.9837589263916,
      "kl": 11.190443992614746,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -34349700.20657277,
      "logits/rejected": -36814432.64898596,
      "logps/chosen": -472.0523767605634,
      "logps/rejected": -392.29621684867396,
      "loss": 0.5954,
      "loss/base_kto": 3.7195801734924316,
      "metrics/advantage_var": 276.13665771484375,
      "regularization/lipschitz_norm": 1077.98046875,
      "regularization/w1": 1.0434850454330444,
      "rewards/chosen": 0.1479542333755135,
      "rewards/margins": 0.2727873864916772,
      "rewards/rejected": -0.1248331531161637,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 21.31780242919922,
      "kl": 8.398650169372559,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37238059.8272,
      "logits/rejected": -37278921.673282444,
      "logps/chosen": -469.3322,
      "logps/rejected": -398.4593034351145,
      "loss": 0.5922,
      "loss/base_kto": 3.74658203125,
      "metrics/advantage_var": 238.9662628173828,
      "regularization/lipschitz_norm": 1023.7175903320312,
      "regularization/w1": 0.9909586310386658,
      "rewards/chosen": 0.008236831665039062,
      "rewards/margins": 0.21772090288700946,
      "rewards/rejected": -0.2094840712219704,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 15.567916870117188,
      "kl": 7.363170146942139,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -35003254.249594815,
      "logits/rejected": -36386337.2066365,
      "logps/chosen": -494.2009724473258,
      "logps/rejected": -365.56570512820514,
      "loss": 0.5885,
      "loss/base_kto": 3.7361361980438232,
      "metrics/advantage_var": 224.89883422851562,
      "regularization/lipschitz_norm": 1004.3369140625,
      "regularization/w1": 0.9721981287002563,
      "rewards/chosen": 0.06719711649939539,
      "rewards/margins": 0.237335077126104,
      "rewards/rejected": -0.1701379606267086,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 24.032546997070312,
      "kl": 9.157782554626465,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -34908598.409785934,
      "logits/rejected": -36706009.55910543,
      "logps/chosen": -486.85865825688074,
      "logps/rejected": -368.68033646166134,
      "loss": 0.592,
      "loss/base_kto": 3.7558517456054688,
      "metrics/advantage_var": 231.52017211914062,
      "regularization/lipschitz_norm": 1012.8148803710938,
      "regularization/w1": 0.9804048538208008,
      "rewards/chosen": 0.08251190185546875,
      "rewards/margins": 0.23528511607989716,
      "rewards/rejected": -0.1527732142244284,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 22.605880737304688,
      "kl": 7.248265266418457,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36638179.018867925,
      "logits/rejected": -35327058.68322981,
      "logps/chosen": -476.2138364779874,
      "logps/rejected": -384.40333850931677,
      "loss": 0.5959,
      "loss/base_kto": 3.7408242225646973,
      "metrics/advantage_var": 263.2743835449219,
      "regularization/lipschitz_norm": 1060.4591064453125,
      "regularization/w1": 1.0265244245529175,
      "rewards/chosen": 0.10654787747365124,
      "rewards/margins": 0.25763921023504854,
      "rewards/rejected": -0.15109133276139727,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 27.252864837646484,
      "kl": 11.121315002441406,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35950467.97515528,
      "logits/rejected": -36643794.91823899,
      "logps/chosen": -488.9490489130435,
      "logps/rejected": -365.5615664308176,
      "loss": 0.5998,
      "loss/base_kto": 3.7466704845428467,
      "metrics/advantage_var": 265.4429626464844,
      "regularization/lipschitz_norm": 1086.5301513671875,
      "regularization/w1": 1.0517610311508179,
      "rewards/chosen": 0.152827979615016,
      "rewards/margins": 0.2504669795059972,
      "rewards/rejected": -0.09763899989098122,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 24.75717544555664,
      "kl": 8.752140998840332,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -36627251.2,
      "logits/rejected": -36925485.15275591,
      "logps/chosen": -490.5701550387597,
      "logps/rejected": -367.12942913385825,
      "loss": 0.596,
      "loss/base_kto": 3.7421226501464844,
      "metrics/advantage_var": 273.6818542480469,
      "regularization/lipschitz_norm": 1060.021240234375,
      "regularization/w1": 1.026100516319275,
      "rewards/chosen": 0.15237606551296026,
      "rewards/margins": 0.27113368289269696,
      "rewards/rejected": -0.11875761737973671,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 20.251605987548828,
      "kl": 10.91238784790039,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36909988.71961102,
      "logits/rejected": -37753480.68778281,
      "logps/chosen": -480.24483387358185,
      "logps/rejected": -384.8237415158371,
      "loss": 0.5909,
      "loss/base_kto": 3.7201783657073975,
      "metrics/advantage_var": 244.40771484375,
      "regularization/lipschitz_norm": 1040.3936767578125,
      "regularization/w1": 1.007101058959961,
      "rewards/chosen": 0.1605994473400147,
      "rewards/margins": 0.25788314583481226,
      "rewards/rejected": -0.09728369849479755,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 19.03770637512207,
      "kl": 5.372455596923828,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36305011.9245283,
      "logits/rejected": -36325131.13043478,
      "logps/chosen": -472.3481230345912,
      "logps/rejected": -367.3989955357143,
      "loss": 0.5912,
      "loss/base_kto": 3.7491490840911865,
      "metrics/advantage_var": 242.1458282470703,
      "regularization/lipschitz_norm": 1012.4777221679688,
      "regularization/w1": 0.9800785183906555,
      "rewards/chosen": -0.09079524705994804,
      "rewards/margins": 0.2295552577500772,
      "rewards/rejected": -0.32035050481002525,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 18.146291732788086,
      "kl": 6.706477642059326,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35684746.20150376,
      "logits/rejected": -37200791.10243902,
      "logps/chosen": -505.90216165413534,
      "logps/rejected": -362.4806910569106,
      "loss": 0.5916,
      "loss/base_kto": 3.750601291656494,
      "metrics/advantage_var": 253.0991973876953,
      "regularization/lipschitz_norm": 1014.5658569335938,
      "regularization/w1": 0.9820996522903442,
      "rewards/chosen": 0.04141369583015155,
      "rewards/margins": 0.26068507500634563,
      "rewards/rejected": -0.2192713791761941,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 23.190210342407227,
      "kl": 12.415057182312012,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -36821188.55727554,
      "logits/rejected": -37918976.0,
      "logps/chosen": -470.85255417956654,
      "logps/rejected": -362.6338261471519,
      "loss": 0.599,
      "loss/base_kto": 3.511014938354492,
      "metrics/advantage_var": 425.7567443847656,
      "regularization/lipschitz_norm": 1323.0213623046875,
      "regularization/w1": 1.2806845903396606,
      "rewards/chosen": 0.3479822353681913,
      "rewards/margins": 0.5165544801330746,
      "rewards/rejected": -0.1685722447648833,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 15.094964027404785,
      "kl": 9.411001205444336,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -35588362.98349835,
      "logits/rejected": -37161579.869436204,
      "logps/chosen": -472.8260107260726,
      "logps/rejected": -365.7991237017804,
      "loss": 0.6122,
      "loss/base_kto": 3.365062713623047,
      "metrics/advantage_var": 596.436279296875,
      "regularization/lipschitz_norm": 1583.323486328125,
      "regularization/w1": 1.5326570272445679,
      "rewards/chosen": 0.22851139484065594,
      "rewards/margins": 0.6968778875903314,
      "rewards/rejected": -0.46836649274967546,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 23.973220825195312,
      "kl": 10.743574142456055,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -33417089.819905214,
      "logits/rejected": -35526944.04945904,
      "logps/chosen": -465.03297788309635,
      "logps/rejected": -381.89637268160743,
      "loss": 0.6033,
      "loss/base_kto": 3.2513177394866943,
      "metrics/advantage_var": 620.6433715820312,
      "regularization/lipschitz_norm": 1627.0052490234375,
      "regularization/w1": 1.5749410390853882,
      "rewards/chosen": 0.3973120618582148,
      "rewards/margins": 0.856496381237924,
      "rewards/rejected": -0.4591843193797092,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 21.905805587768555,
      "kl": 9.628497123718262,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35349444.92307692,
      "logits/rejected": -37393514.146341465,
      "logps/chosen": -487.56049679487177,
      "logps/rejected": -378.9127286585366,
      "loss": 0.5981,
      "loss/base_kto": 3.323640823364258,
      "metrics/advantage_var": 539.4470825195312,
      "regularization/lipschitz_norm": 1509.3988037109375,
      "regularization/w1": 1.461098074913025,
      "rewards/chosen": 0.36176212017352766,
      "rewards/margins": 0.7473041756291774,
      "rewards/rejected": -0.3855420554556498,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 20.7901668548584,
      "kl": 13.369658470153809,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -35224843.76545166,
      "logits/rejected": -36602054.80431433,
      "logps/chosen": -473.06418383518223,
      "logps/rejected": -375.352031972265,
      "loss": 0.6177,
      "loss/base_kto": 3.2955589294433594,
      "metrics/advantage_var": 680.991455078125,
      "regularization/lipschitz_norm": 1700.3582763671875,
      "regularization/w1": 1.6459465026855469,
      "rewards/chosen": 0.3789882266955973,
      "rewards/margins": 0.7962333009986116,
      "rewards/rejected": -0.4172450743030143,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 20.343870162963867,
      "kl": 12.414015769958496,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -34090330.73417722,
      "logits/rejected": -35467203.95061728,
      "logps/chosen": -458.1440862341772,
      "logps/rejected": -369.397159529321,
      "loss": 0.6119,
      "loss/base_kto": 3.319322347640991,
      "metrics/advantage_var": 599.7037963867188,
      "regularization/lipschitz_norm": 1628.3851318359375,
      "regularization/w1": 1.5762766599655151,
      "rewards/chosen": 0.3447738840610166,
      "rewards/margins": 0.746129203613968,
      "rewards/rejected": -0.4013553195529514,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 17.768192291259766,
      "kl": 17.229093551635742,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -34604595.34670487,
      "logits/rejected": -34350199.64261168,
      "logps/chosen": -463.49086676217763,
      "logps/rejected": -373.56588273195877,
      "loss": 0.5964,
      "loss/base_kto": 3.3150627613067627,
      "metrics/advantage_var": 555.2005004882812,
      "regularization/lipschitz_norm": 1504.4609375,
      "regularization/w1": 1.4563181400299072,
      "rewards/chosen": 0.5407342774137043,
      "rewards/margins": 0.7110109061504821,
      "rewards/rejected": -0.17027662873677782,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 18.17601203918457,
      "kl": 29.794462203979492,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34898743.119617224,
      "logits/rejected": -36604440.30627871,
      "logps/chosen": -466.8919457735247,
      "logps/rejected": -349.51746745788665,
      "loss": 0.5957,
      "loss/base_kto": 3.277207136154175,
      "metrics/advantage_var": 542.3825073242188,
      "regularization/lipschitz_norm": 1537.3563232421875,
      "regularization/w1": 1.4881608486175537,
      "rewards/chosen": 0.7279997951866527,
      "rewards/margins": 0.7635038544451531,
      "rewards/rejected": -0.035504059258500434,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 25.832965850830078,
      "kl": 25.964340209960938,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35796727.94968554,
      "logits/rejected": -37028294.757763974,
      "logps/chosen": -472.3318592767296,
      "logps/rejected": -367.61379076086956,
      "loss": 0.5976,
      "loss/base_kto": 3.2431321144104004,
      "metrics/advantage_var": 626.7291259765625,
      "regularization/lipschitz_norm": 1588.559814453125,
      "regularization/w1": 1.537725806236267,
      "rewards/chosen": 0.6725802151661999,
      "rewards/margins": 0.8015443727445676,
      "rewards/rejected": -0.12896415757836763,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 17.008806228637695,
      "kl": 18.336414337158203,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -35578185.19938176,
      "logits/rejected": -36221461.83886256,
      "logps/chosen": -487.70580564142193,
      "logps/rejected": -349.38008491311217,
      "loss": 0.6093,
      "loss/base_kto": 3.2575058937072754,
      "metrics/advantage_var": 690.2684936523438,
      "regularization/lipschitz_norm": 1670.5321044921875,
      "regularization/w1": 1.617074966430664,
      "rewards/chosen": 0.562875550165427,
      "rewards/margins": 0.8442507059834563,
      "rewards/rejected": -0.28137515581802924,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 21.019044876098633,
      "kl": 18.322830200195312,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35494652.545727134,
      "logits/rejected": -35744509.07667211,
      "logps/chosen": -497.87378185907045,
      "logps/rejected": -372.2220126427406,
      "loss": 0.6037,
      "loss/base_kto": 3.2976510524749756,
      "metrics/advantage_var": 664.135498046875,
      "regularization/lipschitz_norm": 1582.7314453125,
      "regularization/w1": 1.5320841073989868,
      "rewards/chosen": 0.602982551082857,
      "rewards/margins": 0.7670870563137591,
      "rewards/rejected": -0.16410450523090206,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 21.914852142333984,
      "kl": 18.818378448486328,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -35118670.23298033,
      "logits/rejected": -37378600.52988692,
      "logps/chosen": -499.37414901664147,
      "logps/rejected": -374.53675282714056,
      "loss": 0.6026,
      "loss/base_kto": 3.281919240951538,
      "metrics/advantage_var": 598.3539428710938,
      "regularization/lipschitz_norm": 1589.9345703125,
      "regularization/w1": 1.539056658744812,
      "rewards/chosen": 0.6129649013686649,
      "rewards/margins": 0.7700925528975611,
      "rewards/rejected": -0.15712765152889616,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 19.1385498046875,
      "kl": 18.0656681060791,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -34849660.29581994,
      "logits/rejected": -36409113.67781155,
      "logps/chosen": -456.8292805466238,
      "logps/rejected": -376.95618825987845,
      "loss": 0.6008,
      "loss/base_kto": 3.266716480255127,
      "metrics/advantage_var": 567.34765625,
      "regularization/lipschitz_norm": 1590.2733154296875,
      "regularization/w1": 1.5393844842910767,
      "rewards/chosen": 0.5471190424977391,
      "rewards/margins": 0.810753674921395,
      "rewards/rejected": -0.263634632423656,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 20.76739501953125,
      "kl": 12.531976699829102,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35243240.80127186,
      "logits/rejected": -36533523.2688172,
      "logps/chosen": -490.87112480127183,
      "logps/rejected": -383.5096966205837,
      "loss": 0.6112,
      "loss/base_kto": 3.3029744625091553,
      "metrics/advantage_var": 716.6351928710938,
      "regularization/lipschitz_norm": 1639.134033203125,
      "regularization/w1": 1.5866817235946655,
      "rewards/chosen": 0.4233794341216216,
      "rewards/margins": 0.7739105946783973,
      "rewards/rejected": -0.35053116055677563,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 18.617046356201172,
      "kl": 8.331124305725098,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -33056453.670886077,
      "logits/rejected": -35443857.38271605,
      "logps/chosen": -478.7087618670886,
      "logps/rejected": -396.5805362654321,
      "loss": 0.6065,
      "loss/base_kto": 3.3282151222229004,
      "metrics/advantage_var": 571.9895629882812,
      "regularization/lipschitz_norm": 1573.963134765625,
      "regularization/w1": 1.5235962867736816,
      "rewards/chosen": 0.3222663493096074,
      "rewards/margins": 0.7304141847467698,
      "rewards/rejected": -0.40814783543716243,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 15.352473258972168,
      "kl": 11.78324031829834,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35287961.139430285,
      "logits/rejected": -35840016.70473083,
      "logps/chosen": -483.6323088455772,
      "logps/rejected": -375.938927406199,
      "loss": 0.6006,
      "loss/base_kto": 3.31672739982605,
      "metrics/advantage_var": 587.284423828125,
      "regularization/lipschitz_norm": 1537.381103515625,
      "regularization/w1": 1.4881848096847534,
      "rewards/chosen": 0.42149972987139245,
      "rewards/margins": 0.7596054714375884,
      "rewards/rejected": -0.33810574156619594,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 15.512990951538086,
      "kl": 24.919565200805664,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -36600577.15837104,
      "logits/rejected": -36697033.23176661,
      "logps/chosen": -462.94994343891403,
      "logps/rejected": -381.0310727309562,
      "loss": 0.6,
      "loss/base_kto": 3.2668755054473877,
      "metrics/advantage_var": 563.43408203125,
      "regularization/lipschitz_norm": 1583.4862060546875,
      "regularization/w1": 1.5328145027160645,
      "rewards/chosen": 0.6703406056307739,
      "rewards/margins": 0.7802145086796431,
      "rewards/rejected": -0.10987390304886928,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 21.205745697021484,
      "kl": 10.23928165435791,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34873062.839313574,
      "logits/rejected": -35856525.02034429,
      "logps/chosen": -506.283736349454,
      "logps/rejected": -391.4015062597809,
      "loss": 0.6237,
      "loss/base_kto": 3.296105146408081,
      "metrics/advantage_var": 908.1524658203125,
      "regularization/lipschitz_norm": 1749.1607666015625,
      "regularization/w1": 1.6931877136230469,
      "rewards/chosen": 0.37295617923349744,
      "rewards/margins": 0.7983148634568934,
      "rewards/rejected": -0.42535868422339596,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 17.41119956970215,
      "kl": 13.737591743469238,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -33868203.05631659,
      "logits/rejected": -35198427.0176565,
      "logps/chosen": -511.1039764079148,
      "logps/rejected": -366.57907804975923,
      "loss": 0.6094,
      "loss/base_kto": 3.268270254135132,
      "metrics/advantage_var": 602.8837280273438,
      "regularization/lipschitz_norm": 1659.706298828125,
      "regularization/w1": 1.6065956354141235,
      "rewards/chosen": 0.5461068581591276,
      "rewards/margins": 0.804103467817848,
      "rewards/rejected": -0.2579966096587204,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 19.09857749938965,
      "kl": 13.377432823181152,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -33628585.23358779,
      "logits/rejected": -36098510.0288,
      "logps/chosen": -457.9335400763359,
      "logps/rejected": -372.3904,
      "loss": 0.6032,
      "loss/base_kto": 3.329882860183716,
      "metrics/advantage_var": 557.4445190429688,
      "regularization/lipschitz_norm": 1544.8406982421875,
      "regularization/w1": 1.4954057931900024,
      "rewards/chosen": 0.47822475287750477,
      "rewards/margins": 0.7075744354946922,
      "rewards/rejected": -0.2293496826171875,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 24.2120361328125,
      "kl": 17.392555236816406,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34461819.74622356,
      "logits/rejected": -35917532.37540453,
      "logps/chosen": -482.06353851963746,
      "logps/rejected": -389.41254550970876,
      "loss": 0.5985,
      "loss/base_kto": 3.3604018688201904,
      "metrics/advantage_var": 578.8988037109375,
      "regularization/lipschitz_norm": 1474.77392578125,
      "regularization/w1": 1.4275811910629272,
      "rewards/chosen": 0.48124007948215636,
      "rewards/margins": 0.6917910728737674,
      "rewards/rejected": -0.21055099339161104,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 20.3862247467041,
      "kl": 22.15376091003418,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -35213855.9488,
      "logits/rejected": -35874200.03664122,
      "logps/chosen": -465.654,
      "logps/rejected": -385.3359971374046,
      "loss": 0.617,
      "loss/base_kto": 3.281867742538452,
      "metrics/advantage_var": 818.151123046875,
      "regularization/lipschitz_norm": 1709.1787109375,
      "regularization/w1": 1.6544849872589111,
      "rewards/chosen": 0.568648828125,
      "rewards/margins": 0.7257217777164837,
      "rewards/rejected": -0.15707294959148377,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 15.866477012634277,
      "kl": 16.126449584960938,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -33328958.855284553,
      "logits/rejected": -36385211.47669173,
      "logps/chosen": -500.58958333333334,
      "logps/rejected": -400.2533364661654,
      "loss": 0.5986,
      "loss/base_kto": 3.2381935119628906,
      "metrics/advantage_var": 614.0758666992188,
      "regularization/lipschitz_norm": 1601.5994873046875,
      "regularization/w1": 1.550348162651062,
      "rewards/chosen": 0.5206572741996951,
      "rewards/margins": 0.8061251017886871,
      "rewards/rejected": -0.28546782758899203,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 21.90863037109375,
      "kl": 11.127936363220215,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -34104755.77744361,
      "logits/rejected": -34606497.92520325,
      "logps/chosen": -492.30249060150373,
      "logps/rejected": -373.6853150406504,
      "loss": 0.616,
      "loss/base_kto": 3.3320329189300537,
      "metrics/advantage_var": 577.935302734375,
      "regularization/lipschitz_norm": 1649.1051025390625,
      "regularization/w1": 1.5963337421417236,
      "rewards/chosen": 0.4293969175869361,
      "rewards/margins": 0.742410183553336,
      "rewards/rejected": -0.3130132659663999,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 18.541954040527344,
      "kl": 27.366214752197266,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -35602554.55748031,
      "logits/rejected": -37120523.906976745,
      "logps/chosen": -492.4576771653543,
      "logps/rejected": -383.4197189922481,
      "loss": 0.6149,
      "loss/base_kto": 3.2778451442718506,
      "metrics/advantage_var": 654.3119506835938,
      "regularization/lipschitz_norm": 1695.574462890625,
      "regularization/w1": 1.6413160562515259,
      "rewards/chosen": 0.6180968277097687,
      "rewards/margins": 0.7462049602421694,
      "rewards/rejected": -0.12810813253240066,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 21.975324630737305,
      "kl": 14.015891075134277,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -35352369.00459418,
      "logits/rejected": -37997004.55502392,
      "logps/chosen": -517.374569295559,
      "logps/rejected": -353.6818181818182,
      "loss": 0.6062,
      "loss/base_kto": 3.2623748779296875,
      "metrics/advantage_var": 588.2733764648438,
      "regularization/lipschitz_norm": 1639.5390625,
      "regularization/w1": 1.5870739221572876,
      "rewards/chosen": 0.5089719342966357,
      "rewards/margins": 0.7935869961767443,
      "rewards/rejected": -0.28461506188010866,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 23.730363845825195,
      "kl": 16.976552963256836,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -35217262.8921095,
      "logits/rejected": -37190623.368740514,
      "logps/chosen": -473.2291666666667,
      "logps/rejected": -365.88569328528075,
      "loss": 0.6008,
      "loss/base_kto": 3.291616201400757,
      "metrics/advantage_var": 571.4057006835938,
      "regularization/lipschitz_norm": 1564.6864013671875,
      "regularization/w1": 1.5146163702011108,
      "rewards/chosen": 0.515448135646261,
      "rewards/margins": 0.7447856050453269,
      "rewards/rejected": -0.22933746939906582,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 16.515743255615234,
      "kl": 15.26294994354248,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34693212.510903426,
      "logits/rejected": -35159454.09404389,
      "logps/chosen": -482.8357671339564,
      "logps/rejected": -382.845954153605,
      "loss": 0.597,
      "loss/base_kto": 3.288755178451538,
      "metrics/advantage_var": 622.9114379882812,
      "regularization/lipschitz_norm": 1536.192626953125,
      "regularization/w1": 1.4870343208312988,
      "rewards/chosen": 0.5087190431969189,
      "rewards/margins": 0.7804539206367944,
      "rewards/rejected": -0.27173487743987557,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 14.575602531433105,
      "kl": 15.25003719329834,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34340322.76672105,
      "logits/rejected": -36499314.75862069,
      "logps/chosen": -470.53125,
      "logps/rejected": -367.58302098950526,
      "loss": 0.6105,
      "loss/base_kto": 3.238189458847046,
      "metrics/advantage_var": 650.7476196289062,
      "regularization/lipschitz_norm": 1700.1590576171875,
      "regularization/w1": 1.6457538604736328,
      "rewards/chosen": 0.508925022542185,
      "rewards/margins": 0.8279714777081566,
      "rewards/rejected": -0.3190464551659717,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 18.860393524169922,
      "kl": 8.02838134765625,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -33357676.819571864,
      "logits/rejected": -35763837.955271564,
      "logps/chosen": -473.2672974006116,
      "logps/rejected": -363.9890425319489,
      "loss": 0.6002,
      "loss/base_kto": 3.262939453125,
      "metrics/advantage_var": 584.277099609375,
      "regularization/lipschitz_norm": 1589.353271484375,
      "regularization/w1": 1.5384939908981323,
      "rewards/chosen": 0.45235655431718275,
      "rewards/margins": 0.8391085519022965,
      "rewards/rejected": -0.3867519975851138,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 22.32830047607422,
      "kl": 14.150607109069824,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35828974.33065811,
      "logits/rejected": -35446322.654490106,
      "logps/chosen": -500.97602327447834,
      "logps/rejected": -395.9500095129376,
      "loss": 0.6073,
      "loss/base_kto": 3.2730233669281006,
      "metrics/advantage_var": 650.7073364257812,
      "regularization/lipschitz_norm": 1637.4290771484375,
      "regularization/w1": 1.5850313901901245,
      "rewards/chosen": 0.4652767273243128,
      "rewards/margins": 0.8003541147000501,
      "rewards/rejected": -0.3350773873757373,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 16.482961654663086,
      "kl": 9.742484092712402,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34040332.8,
      "logits/rejected": -36287939.2,
      "logps/chosen": -471.75498046875,
      "logps/rejected": -378.1865234375,
      "loss": 0.5993,
      "loss/base_kto": 3.34765887260437,
      "metrics/advantage_var": 576.4119262695312,
      "regularization/lipschitz_norm": 1494.62353515625,
      "regularization/w1": 1.4467957019805908,
      "rewards/chosen": 0.39304141998291015,
      "rewards/margins": 0.7137337684631347,
      "rewards/rejected": -0.3206923484802246,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 28.27539825439453,
      "kl": 13.098017692565918,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -35510078.56071964,
      "logits/rejected": -35925708.63295269,
      "logps/chosen": -470.10625937031483,
      "logps/rejected": -359.27008564437193,
      "loss": 0.6102,
      "loss/base_kto": 3.3460328578948975,
      "metrics/advantage_var": 1050.8026123046875,
      "regularization/lipschitz_norm": 1585.9912109375,
      "regularization/w1": 1.5352394580841064,
      "rewards/chosen": 0.5042811541006841,
      "rewards/margins": 0.7036321303445666,
      "rewards/rejected": -0.19935097624388254,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 20.633995056152344,
      "kl": 14.561180114746094,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34451084.82034976,
      "logits/rejected": -35866457.26574501,
      "logps/chosen": -497.5389507154213,
      "logps/rejected": -386.1611703149002,
      "loss": 0.5935,
      "loss/base_kto": 3.2682251930236816,
      "metrics/advantage_var": 545.0556640625,
      "regularization/lipschitz_norm": 1528.5404052734375,
      "regularization/w1": 1.479627251625061,
      "rewards/chosen": 0.4862970737281399,
      "rewards/margins": 0.7785655601612717,
      "rewards/rejected": -0.29226848643313175,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 18.701581954956055,
      "kl": 11.452432632446289,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -35697291.78159127,
      "logits/rejected": -36275829.78403756,
      "logps/chosen": -480.68125975039004,
      "logps/rejected": -363.0715473395931,
      "loss": 0.6179,
      "loss/base_kto": 3.37455677986145,
      "metrics/advantage_var": 622.552001953125,
      "regularization/lipschitz_norm": 1620.2760009765625,
      "regularization/w1": 1.568427324295044,
      "rewards/chosen": 0.3718554992199688,
      "rewards/margins": 0.6929975137965778,
      "rewards/rejected": -0.32114201457660896,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 14.728906631469727,
      "kl": 16.546998977661133,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -34478846.36942675,
      "logits/rejected": -36221625.32515337,
      "logps/chosen": -465.9889032643312,
      "logps/rejected": -390.706384202454,
      "loss": 0.5952,
      "loss/base_kto": 3.328676700592041,
      "metrics/advantage_var": 550.82666015625,
      "regularization/lipschitz_norm": 1480.2578125,
      "regularization/w1": 1.4328895807266235,
      "rewards/chosen": 0.3935989817236639,
      "rewards/margins": 0.689255525291532,
      "rewards/rejected": -0.2956565435678681,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 19.20351219177246,
      "kl": 14.74124813079834,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -34962900.06864274,
      "logits/rejected": -37755264.60093897,
      "logps/chosen": -488.10871684867396,
      "logps/rejected": -390.09467918622846,
      "loss": 0.6004,
      "loss/base_kto": 3.341092348098755,
      "metrics/advantage_var": 549.0675659179688,
      "regularization/lipschitz_norm": 1510.6282958984375,
      "regularization/w1": 1.4622882604599,
      "rewards/chosen": 0.4436404604621685,
      "rewards/margins": 0.684837834594113,
      "rewards/rejected": -0.24119737413194445,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 18.116750717163086,
      "kl": 12.19059944152832,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33270414.08992248,
      "logits/rejected": -35000416.75590551,
      "logps/chosen": -506.17713178294576,
      "logps/rejected": -365.162155511811,
      "loss": 0.6061,
      "loss/base_kto": 3.3585686683654785,
      "metrics/advantage_var": 559.7994995117188,
      "regularization/lipschitz_norm": 1539.2572021484375,
      "regularization/w1": 1.4900009632110596,
      "rewards/chosen": 0.38096188094264777,
      "rewards/margins": 0.7030596592629603,
      "rewards/rejected": -0.3220977783203125,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 14.686551094055176,
      "kl": 16.379161834716797,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -33170007.57894737,
      "logits/rejected": -34907986.28571428,
      "logps/chosen": -490.8579358552632,
      "logps/rejected": -368.4364769345238,
      "loss": 0.5992,
      "loss/base_kto": 3.2888245582580566,
      "metrics/advantage_var": 568.0514526367188,
      "regularization/lipschitz_norm": 1554.4371337890625,
      "regularization/w1": 1.504694938659668,
      "rewards/chosen": 0.49511999832956416,
      "rewards/margins": 0.7493636219722586,
      "rewards/rejected": -0.25424362364269437,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 19.832347869873047,
      "kl": 8.920297622680664,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -35746486.567353405,
      "logits/rejected": -35967476.12982999,
      "logps/chosen": -500.636043977813,
      "logps/rejected": -373.3530235703246,
      "loss": 0.5697,
      "loss/base_kto": 3.216679096221924,
      "metrics/advantage_var": 485.2485046386719,
      "regularization/lipschitz_norm": 1385.5474853515625,
      "regularization/w1": 1.3412100076675415,
      "rewards/chosen": 0.3797266456857914,
      "rewards/margins": 0.856157830481,
      "rewards/rejected": -0.47643118479520863,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 21.89859962463379,
      "kl": 7.963223934173584,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34025620.35905512,
      "logits/rejected": -35273955.02635659,
      "logps/chosen": -483.9924212598425,
      "logps/rejected": -391.209980620155,
      "loss": 0.5612,
      "loss/base_kto": 3.1962673664093018,
      "metrics/advantage_var": 484.2899475097656,
      "regularization/lipschitz_norm": 1335.9390869140625,
      "regularization/w1": 1.2931890487670898,
      "rewards/chosen": 0.40625403697096457,
      "rewards/margins": 0.9007795146538619,
      "rewards/rejected": -0.4945254776828973,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 19.94541358947754,
      "kl": 10.798842430114746,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33959290.56969697,
      "logits/rejected": -34563745.85806452,
      "logps/chosen": -480.3489583333333,
      "logps/rejected": -382.5594758064516,
      "loss": 0.5601,
      "loss/base_kto": 3.2194650173187256,
      "metrics/advantage_var": 437.97882080078125,
      "regularization/lipschitz_norm": 1303.2080078125,
      "regularization/w1": 1.261505365371704,
      "rewards/chosen": 0.40321266867897726,
      "rewards/margins": 0.8180123158563966,
      "rewards/rejected": -0.41479964717741935,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 15.935685157775879,
      "kl": 13.427490234375,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34735607.76848874,
      "logits/rejected": -36867442.38297872,
      "logps/chosen": -480.03054662379424,
      "logps/rejected": -407.0298252279635,
      "loss": 0.5543,
      "loss/base_kto": 3.1622695922851562,
      "metrics/advantage_var": 454.5888671875,
      "regularization/lipschitz_norm": 1314.315673828125,
      "regularization/w1": 1.272257685661316,
      "rewards/chosen": 0.48861944560452675,
      "rewards/margins": 0.8846590431361092,
      "rewards/rejected": -0.39603959753158247,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 17.077974319458008,
      "kl": 11.499598503112793,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34062842.391236305,
      "logits/rejected": -35439119.176287055,
      "logps/chosen": -455.6285211267606,
      "logps/rejected": -377.0400741029641,
      "loss": 0.5543,
      "loss/base_kto": 3.197096347808838,
      "metrics/advantage_var": 422.5838928222656,
      "regularization/lipschitz_norm": 1278.277587890625,
      "regularization/w1": 1.237372636795044,
      "rewards/chosen": 0.49088799561693075,
      "rewards/margins": 0.8599527075462282,
      "rewards/rejected": -0.36906471192929746,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 15.541845321655273,
      "kl": 7.3202714920043945,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -35913826.69879518,
      "logits/rejected": -36105568.415584415,
      "logps/chosen": -477.18844126506025,
      "logps/rejected": -377.83081371753246,
      "loss": 0.5543,
      "loss/base_kto": 3.2012290954589844,
      "metrics/advantage_var": 410.8737487792969,
      "regularization/lipschitz_norm": 1274.033203125,
      "regularization/w1": 1.2332642078399658,
      "rewards/chosen": 0.4088220711213997,
      "rewards/margins": 0.8678787767448778,
      "rewards/rejected": -0.4590567056234781,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 19.03652572631836,
      "kl": 16.03883171081543,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34887383.78849144,
      "logits/rejected": -36789831.535321824,
      "logps/chosen": -498.41757387247276,
      "logps/rejected": -364.00431711145995,
      "loss": 0.5556,
      "loss/base_kto": 3.133333206176758,
      "metrics/advantage_var": 471.56182861328125,
      "regularization/lipschitz_norm": 1355.15771484375,
      "regularization/w1": 1.3117927312850952,
      "rewards/chosen": 0.5923158162118488,
      "rewards/margins": 0.9083339132794017,
      "rewards/rejected": -0.31601809706755296,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 16.680809020996094,
      "kl": 14.453997611999512,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -35109820.37735849,
      "logits/rejected": -36396330.93167702,
      "logps/chosen": -477.95735062893084,
      "logps/rejected": -370.841056871118,
      "loss": 0.5515,
      "loss/base_kto": 3.102193593978882,
      "metrics/advantage_var": 468.564453125,
      "regularization/lipschitz_norm": 1352.7255859375,
      "regularization/w1": 1.3094384670257568,
      "rewards/chosen": 0.6048946740492335,
      "rewards/margins": 0.9435668855007935,
      "rewards/rejected": -0.33867221145156007,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 24.052738189697266,
      "kl": 12.60477352142334,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34349540.36825397,
      "logits/rejected": -34623465.94461539,
      "logps/chosen": -466.6015873015873,
      "logps/rejected": -383.36846153846153,
      "loss": 0.564,
      "loss/base_kto": 3.164311170578003,
      "metrics/advantage_var": 476.12921142578125,
      "regularization/lipschitz_norm": 1392.194580078125,
      "regularization/w1": 1.3476442098617554,
      "rewards/chosen": 0.5004607669890873,
      "rewards/margins": 0.896481697352669,
      "rewards/rejected": -0.3960209303635817,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 15.353602409362793,
      "kl": 9.640913963317871,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -35780358.56410257,
      "logits/rejected": -37144394.92682927,
      "logps/chosen": -477.7608173076923,
      "logps/rejected": -368.784775152439,
      "loss": 0.5489,
      "loss/base_kto": 3.1865601539611816,
      "metrics/advantage_var": 413.68359375,
      "regularization/lipschitz_norm": 1244.8675537109375,
      "regularization/w1": 1.2050317525863647,
      "rewards/chosen": 0.45113671131623095,
      "rewards/margins": 0.8710790089028115,
      "rewards/rejected": -0.4199422975865806,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 19.645570755004883,
      "kl": 11.93309211730957,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34557834.44108761,
      "logits/rejected": -35913317.074433655,
      "logps/chosen": -487.8172205438066,
      "logps/rejected": -369.0038683252427,
      "loss": 0.557,
      "loss/base_kto": 3.16609787940979,
      "metrics/advantage_var": 439.7220153808594,
      "regularization/lipschitz_norm": 1332.4222412109375,
      "regularization/w1": 1.2897846698760986,
      "rewards/chosen": 0.48893438149074775,
      "rewards/margins": 0.8944033191392864,
      "rewards/rejected": -0.40546893764853864,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 13.989500999450684,
      "kl": 14.450139045715332,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -34658866.06940063,
      "logits/rejected": -35601557.00309598,
      "logps/chosen": -485.98077681388014,
      "logps/rejected": -389.7189193111455,
      "loss": 0.5648,
      "loss/base_kto": 3.164170980453491,
      "metrics/advantage_var": 473.74200439453125,
      "regularization/lipschitz_norm": 1398.80517578125,
      "regularization/w1": 1.3540433645248413,
      "rewards/chosen": 0.5909651987936219,
      "rewards/margins": 0.8866910182898002,
      "rewards/rejected": -0.29572581949617843,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 16.189327239990234,
      "kl": 11.753397941589355,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34559513.12883436,
      "logits/rejected": -36000852.789808914,
      "logps/chosen": -465.643884202454,
      "logps/rejected": -370.72123805732485,
      "loss": 0.5641,
      "loss/base_kto": 3.1884517669677734,
      "metrics/advantage_var": 502.8970642089844,
      "regularization/lipschitz_norm": 1368.1595458984375,
      "regularization/w1": 1.3243783712387085,
      "rewards/chosen": 0.5534203652223927,
      "rewards/margins": 0.8631156637270703,
      "rewards/rejected": -0.30969529850467753,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 20.154483795166016,
      "kl": 11.416178703308105,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -35384447.424287856,
      "logits/rejected": -36804126.90375204,
      "logps/chosen": -525.9418103448276,
      "logps/rejected": -358.78206566068513,
      "loss": 0.5615,
      "loss/base_kto": 3.1734509468078613,
      "metrics/advantage_var": 462.114013671875,
      "regularization/lipschitz_norm": 1361.947265625,
      "regularization/w1": 1.3183649778366089,
      "rewards/chosen": 0.5241920665643741,
      "rewards/margins": 0.8779498408302795,
      "rewards/rejected": -0.3537577742659054,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 16.525434494018555,
      "kl": 12.25172233581543,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -34165764.777604975,
      "logits/rejected": -36022619.22762951,
      "logps/chosen": -491.60604587869364,
      "logps/rejected": -367.30298273155415,
      "loss": 0.5709,
      "loss/base_kto": 3.1733152866363525,
      "metrics/advantage_var": 476.2198791503906,
      "regularization/lipschitz_norm": 1439.91650390625,
      "regularization/w1": 1.3938392400741577,
      "rewards/chosen": 0.5377036846629568,
      "rewards/margins": 0.8708096133325016,
      "rewards/rejected": -0.33310592866954475,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 15.079082489013672,
      "kl": 12.411714553833008,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -34833135.2599681,
      "logits/rejected": -35935838.87289433,
      "logps/chosen": -451.6981658692185,
      "logps/rejected": -386.6646248085758,
      "loss": 0.561,
      "loss/base_kto": 3.231250047683716,
      "metrics/advantage_var": 405.32867431640625,
      "regularization/lipschitz_norm": 1298.2681884765625,
      "regularization/w1": 1.2567236423492432,
      "rewards/chosen": 0.43814432487913674,
      "rewards/margins": 0.7966030965190438,
      "rewards/rejected": -0.35845877163990714,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 14.774227142333984,
      "kl": 12.383523941040039,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -33579212.8,
      "logits/rejected": -36042826.76825397,
      "logps/chosen": -476.85942307692306,
      "logps/rejected": -371.9820188492063,
      "loss": 0.5626,
      "loss/base_kto": 3.1879355907440186,
      "metrics/advantage_var": 465.83770751953125,
      "regularization/lipschitz_norm": 1356.3656005859375,
      "regularization/w1": 1.3129619359970093,
      "rewards/chosen": 0.46731121356670674,
      "rewards/margins": 0.8773125505272722,
      "rewards/rejected": -0.41000133696056545,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 17.516733169555664,
      "kl": 11.263381004333496,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -34234809.2631579,
      "logits/rejected": -36706703.23809524,
      "logps/chosen": -476.1243832236842,
      "logps/rejected": -377.2783435639881,
      "loss": 0.5597,
      "loss/base_kto": 3.1911723613739014,
      "metrics/advantage_var": 454.2433166503906,
      "regularization/lipschitz_norm": 1328.9354248046875,
      "regularization/w1": 1.2864094972610474,
      "rewards/chosen": 0.46600713227924545,
      "rewards/margins": 0.8416058580976978,
      "rewards/rejected": -0.3755987258184524,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 16.33338165283203,
      "kl": 10.109529495239258,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34452697.403076924,
      "logits/rejected": -36515004.54603174,
      "logps/chosen": -478.72778846153847,
      "logps/rejected": -361.4546130952381,
      "loss": 0.5563,
      "loss/base_kto": 3.1940488815307617,
      "metrics/advantage_var": 450.54400634765625,
      "regularization/lipschitz_norm": 1298.1900634765625,
      "regularization/w1": 1.256648063659668,
      "rewards/chosen": 0.5028693096454326,
      "rewards/margins": 0.8506774451473548,
      "rewards/rejected": -0.34780813550192213,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 17.103260040283203,
      "kl": 11.305916786193848,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34056343.58425197,
      "logits/rejected": -35604441.89767442,
      "logps/chosen": -510.2220964566929,
      "logps/rejected": -384.35329457364344,
      "loss": 0.5573,
      "loss/base_kto": 3.210218906402588,
      "metrics/advantage_var": 398.03277587890625,
      "regularization/lipschitz_norm": 1289.62451171875,
      "regularization/w1": 1.2483564615249634,
      "rewards/chosen": 0.4129806878998524,
      "rewards/margins": 0.8291276567709764,
      "rewards/rejected": -0.416146968871124,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 13.693824768066406,
      "kl": 11.231901168823242,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -34975075.5328,
      "logits/rejected": -36562037.2519084,
      "logps/chosen": -461.2134,
      "logps/rejected": -379.83284828244274,
      "loss": 0.5532,
      "loss/base_kto": 3.1471829414367676,
      "metrics/advantage_var": 453.8047790527344,
      "regularization/lipschitz_norm": 1320.3148193359375,
      "regularization/w1": 1.2780648469924927,
      "rewards/chosen": 0.476273095703125,
      "rewards/margins": 0.8842373915344703,
      "rewards/rejected": -0.4079642958313454,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 21.8228702545166,
      "kl": 10.097284317016602,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35680366.276923075,
      "logits/rejected": -36090766.222222224,
      "logps/chosen": -485.4498076923077,
      "logps/rejected": -395.991691468254,
      "loss": 0.5596,
      "loss/base_kto": 3.2041866779327393,
      "metrics/advantage_var": 447.53546142578125,
      "regularization/lipschitz_norm": 1314.6365966796875,
      "regularization/w1": 1.2725682258605957,
      "rewards/chosen": 0.42717961237980767,
      "rewards/margins": 0.8594246791890834,
      "rewards/rejected": -0.43224506680927577,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 19.715198516845703,
      "kl": 12.224166870117188,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -34828281.2852459,
      "logits/rejected": -36145191.737313434,
      "logps/chosen": -475.6319159836066,
      "logps/rejected": -402.1400186567164,
      "loss": 0.5627,
      "loss/base_kto": 3.2128219604492188,
      "metrics/advantage_var": 442.60357666015625,
      "regularization/lipschitz_norm": 1331.3394775390625,
      "regularization/w1": 1.2887365818023682,
      "rewards/chosen": 0.46946871397925205,
      "rewards/margins": 0.8268080548724424,
      "rewards/rejected": -0.3573393408931903,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 17.652387619018555,
      "kl": 11.485856056213379,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35123821.88871224,
      "logits/rejected": -35522978.40860215,
      "logps/chosen": -507.59807233704294,
      "logps/rejected": -361.008784562212,
      "loss": 0.5576,
      "loss/base_kto": 3.13618803024292,
      "metrics/advantage_var": 487.19732666015625,
      "regularization/lipschitz_norm": 1368.5726318359375,
      "regularization/w1": 1.324778437614441,
      "rewards/chosen": 0.49694688369435613,
      "rewards/margins": 0.9343460491159431,
      "rewards/rejected": -0.43739916542158697,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 20.829431533813477,
      "kl": 9.707969665527344,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35104294.4,
      "logits/rejected": -35396448.0,
      "logps/chosen": -499.63681640625,
      "logps/rejected": -408.16787109375,
      "loss": 0.5597,
      "loss/base_kto": 3.158587694168091,
      "metrics/advantage_var": 461.05224609375,
      "regularization/lipschitz_norm": 1362.4744873046875,
      "regularization/w1": 1.3188751935958862,
      "rewards/chosen": 0.4626021385192871,
      "rewards/margins": 0.8942239284515381,
      "rewards/rejected": -0.431621789932251,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 25.18584442138672,
      "kl": 10.333015441894531,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -34953715.2,
      "logits/rejected": -36164294.4,
      "logps/chosen": -499.62099609375,
      "logps/rejected": -378.3279296875,
      "loss": 0.5629,
      "loss/base_kto": 3.1900126934051514,
      "metrics/advantage_var": 433.4665222167969,
      "regularization/lipschitz_norm": 1356.5330810546875,
      "regularization/w1": 1.3131239414215088,
      "rewards/chosen": 0.4079261779785156,
      "rewards/margins": 0.8461101531982422,
      "rewards/rejected": -0.4381839752197266,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 16.694236755371094,
      "kl": 9.096259117126465,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -33919099.3010279,
      "logits/rejected": -36184412.74123539,
      "logps/chosen": -504.5181718061674,
      "logps/rejected": -352.5863418196995,
      "loss": 0.5737,
      "loss/base_kto": 3.1659293174743652,
      "metrics/advantage_var": 824.1036987304688,
      "regularization/lipschitz_norm": 1470.7811279296875,
      "regularization/w1": 1.4237160682678223,
      "rewards/chosen": 0.4760711714734765,
      "rewards/margins": 0.8588093135347896,
      "rewards/rejected": -0.3827381420613131,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 17.056594848632812,
      "kl": 9.983824729919434,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -32986793.589905363,
      "logits/rejected": -36054085.746130034,
      "logps/chosen": -475.39910291798105,
      "logps/rejected": -352.9758126934984,
      "loss": 0.564,
      "loss/base_kto": 3.222651720046997,
      "metrics/advantage_var": 444.0147399902344,
      "regularization/lipschitz_norm": 1331.890625,
      "regularization/w1": 1.2892701625823975,
      "rewards/chosen": 0.4675080212132788,
      "rewards/margins": 0.823879874217947,
      "rewards/rejected": -0.35637185300466817,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 12.820006370544434,
      "kl": 11.673685073852539,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34266621.588697016,
      "logits/rejected": -36960013.934681185,
      "logps/chosen": -493.0527864992151,
      "logps/rejected": -376.27007192846037,
      "loss": 0.5469,
      "loss/base_kto": 3.1460659503936768,
      "metrics/advantage_var": 466.6276550292969,
      "regularization/lipschitz_norm": 1269.818603515625,
      "regularization/w1": 1.2291842699050903,
      "rewards/chosen": 0.515916761461195,
      "rewards/margins": 0.9079495567992102,
      "rewards/rejected": -0.39203279533801516,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 13.636578559875488,
      "kl": 11.184309959411621,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35773777.13230769,
      "logits/rejected": -37253471.08571429,
      "logps/chosen": -485.21740384615384,
      "logps/rejected": -381.7684027777778,
      "loss": 0.5575,
      "loss/base_kto": 3.125328540802002,
      "metrics/advantage_var": 481.3175964355469,
      "regularization/lipschitz_norm": 1379.0472412109375,
      "regularization/w1": 1.3349177837371826,
      "rewards/chosen": 0.5428059269831731,
      "rewards/margins": 0.9327230644924737,
      "rewards/rejected": -0.38991713750930057,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 18.770830154418945,
      "kl": 11.703639030456543,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34847018.12658228,
      "logits/rejected": -35032209.38271605,
      "logps/chosen": -483.4439280063291,
      "logps/rejected": -353.645881558642,
      "loss": 0.5565,
      "loss/base_kto": 3.1920933723449707,
      "metrics/advantage_var": 452.3001403808594,
      "regularization/lipschitz_norm": 1301.8756103515625,
      "regularization/w1": 1.2602156400680542,
      "rewards/chosen": 0.5000883174847953,
      "rewards/margins": 0.867645866667075,
      "rewards/rejected": -0.3675575491822796,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 15.435226440429688,
      "kl": 13.41156005859375,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35750206.35897436,
      "logits/rejected": -35239427.12195122,
      "logps/chosen": -460.6829427083333,
      "logps/rejected": -404.6037538109756,
      "loss": 0.5564,
      "loss/base_kto": 3.1568715572357178,
      "metrics/advantage_var": 443.94873046875,
      "regularization/lipschitz_norm": 1336.811767578125,
      "regularization/w1": 1.2940337657928467,
      "rewards/chosen": 0.4726253900772486,
      "rewards/margins": 0.8782601538414206,
      "rewards/rejected": -0.40563476376417207,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 20.130613327026367,
      "kl": 9.79339599609375,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -34129891.15492958,
      "logits/rejected": -35186700.780031204,
      "logps/chosen": -465.1637323943662,
      "logps/rejected": -375.63772425897037,
      "loss": 0.5465,
      "loss/base_kto": 3.1563546657562256,
      "metrics/advantage_var": 434.60137939453125,
      "regularization/lipschitz_norm": 1255.7060546875,
      "regularization/w1": 1.2155234813690186,
      "rewards/chosen": 0.5008374902190923,
      "rewards/margins": 0.8796890371032333,
      "rewards/rejected": -0.37885154688414097,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 17.81679916381836,
      "kl": 10.139800071716309,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35804687.900621116,
      "logits/rejected": -36437314.012578614,
      "logps/chosen": -496.7468944099379,
      "logps/rejected": -370.1814809355346,
      "loss": 0.5637,
      "loss/base_kto": 3.197054624557495,
      "metrics/advantage_var": 472.7958984375,
      "regularization/lipschitz_norm": 1356.276611328125,
      "regularization/w1": 1.312875747680664,
      "rewards/chosen": 0.5197515073029891,
      "rewards/margins": 0.87249687534882,
      "rewards/rejected": -0.3527453680458309,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 12.215582847595215,
      "kl": 12.430460929870605,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35006370.6163522,
      "logits/rejected": -36758954.136645965,
      "logps/chosen": -470.7425314465409,
      "logps/rejected": -372.70060170807454,
      "loss": 0.5562,
      "loss/base_kto": 3.1164519786834717,
      "metrics/advantage_var": 489.5458679199219,
      "regularization/lipschitz_norm": 1377.111083984375,
      "regularization/w1": 1.3330434560775757,
      "rewards/chosen": 0.5485211258414406,
      "rewards/margins": 0.9483621501099286,
      "rewards/rejected": -0.39984102426848794,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 16.069026947021484,
      "kl": 10.979535102844238,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -35466033.951219514,
      "logits/rejected": -35640411.897435896,
      "logps/chosen": -470.0374428353659,
      "logps/rejected": -390.1776091746795,
      "loss": 0.5643,
      "loss/base_kto": 3.1961257457733154,
      "metrics/advantage_var": 467.57598876953125,
      "regularization/lipschitz_norm": 1361.852294921875,
      "regularization/w1": 1.3182729482650757,
      "rewards/chosen": 0.4945542870498285,
      "rewards/margins": 0.8762905533571703,
      "rewards/rejected": -0.38173626630734175,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 14.50257396697998,
      "kl": 10.80820369720459,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34907956.17061611,
      "logits/rejected": -36849914.064914994,
      "logps/chosen": -478.53929699842024,
      "logps/rejected": -363.3722952086553,
      "loss": 0.5602,
      "loss/base_kto": 3.1777961254119873,
      "metrics/advantage_var": 454.50732421875,
      "regularization/lipschitz_norm": 1346.4949951171875,
      "regularization/w1": 1.303407073020935,
      "rewards/chosen": 0.4962760840923677,
      "rewards/margins": 0.8795880912830072,
      "rewards/rejected": -0.3833120071906395,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 16.197853088378906,
      "kl": 12.25694751739502,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -33690625.60250391,
      "logits/rejected": -35501629.503900155,
      "logps/chosen": -475.36551251956183,
      "logps/rejected": -351.76489372074883,
      "loss": 0.5538,
      "loss/base_kto": 3.1732850074768066,
      "metrics/advantage_var": 491.0556640625,
      "regularization/lipschitz_norm": 1298.767578125,
      "regularization/w1": 1.2572071552276611,
      "rewards/chosen": 0.5119681918005429,
      "rewards/margins": 0.8778844751803937,
      "rewards/rejected": -0.3659162833798508,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.972016217523487e+17,
      "train_loss": 0.5863274930878213,
      "train_runtime": 11040.2984,
      "train_samples_per_second": 13.425,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.972016217523487e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}