{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 21.025110244750977,
      "kl": 17.884946823120117,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37365357.93865031,
      "logits/rejected": -38282412.84076433,
      "logps/chosen": -458.23054064417175,
      "logps/rejected": -379.5640425955414,
      "loss": 0.6021,
      "loss/base_kto": 3.915940523147583,
      "metrics/advantage_var": 203.65585327148438,
      "regularization/lipschitz_norm": 930.5560913085938,
      "regularization/w1": 0.9007782340049744,
      "rewards/chosen": 0.25468828634250384,
      "rewards/margins": 0.04719344128627376,
      "rewards/rejected": 0.2074948450562301,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 27.312768936157227,
      "kl": 8.006775856018066,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36416431.24290221,
      "logits/rejected": -37550346.303405575,
      "logps/chosen": -479.43838722397476,
      "logps/rejected": -363.0922020123839,
      "loss": 0.613,
      "loss/base_kto": 3.940383195877075,
      "metrics/advantage_var": 288.2891845703125,
      "regularization/lipschitz_norm": 995.1353759765625,
      "regularization/w1": 0.9632909893989563,
      "rewards/chosen": 0.08643815795705898,
      "rewards/margins": 0.0655225330609888,
      "rewards/rejected": 0.020915624896070167,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 23.3543701171875,
      "kl": 16.31055450439453,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35587574.964705884,
      "logits/rejected": -37091164.16,
      "logps/chosen": -462.7930606617647,
      "logps/rejected": -381.9289322916667,
      "loss": 0.5909,
      "loss/base_kto": 3.8472793102264404,
      "metrics/advantage_var": 185.0529022216797,
      "regularization/lipschitz_norm": 908.7521362304688,
      "regularization/w1": 0.8796720504760742,
      "rewards/chosen": 0.26407690609202666,
      "rewards/margins": 0.14243373796051623,
      "rewards/rejected": 0.12164316813151042,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 22.52024269104004,
      "kl": 13.733599662780762,
      "learning_rate": 7.9e-07,
      "logits/chosen": -35642301.03987241,
      "logits/rejected": -36969072.122511484,
      "logps/chosen": -486.08931419457736,
      "logps/rejected": -384.1779287901991,
      "loss": 0.5946,
      "loss/base_kto": 3.894824266433716,
      "metrics/advantage_var": 177.49766540527344,
      "regularization/lipschitz_norm": 890.3466796875,
      "regularization/w1": 0.8618556261062622,
      "rewards/chosen": 0.1831431776712956,
      "rewards/margins": 0.08566634696244282,
      "rewards/rejected": 0.09747683070885277,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 21.335416793823242,
      "kl": 16.760305404663086,
      "learning_rate": 9.9e-07,
      "logits/chosen": -37779421.96978852,
      "logits/rejected": -37832293.074433655,
      "logps/chosen": -480.33128776435046,
      "logps/rejected": -369.65713491100325,
      "loss": 0.5974,
      "loss/base_kto": 3.840963363647461,
      "metrics/advantage_var": 226.6090850830078,
      "regularization/lipschitz_norm": 969.18359375,
      "regularization/w1": 0.9381695985794067,
      "rewards/chosen": 0.33794018149015764,
      "rewards/margins": 0.14155198624244578,
      "rewards/rejected": 0.19638819524771187,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 18.12994956970215,
      "kl": 23.64974594116211,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -36708412.23529412,
      "logits/rejected": -36441805.12302839,
      "logps/chosen": -468.3295762383901,
      "logps/rejected": -342.2988712539432,
      "loss": 0.5903,
      "loss/base_kto": 3.8368747234344482,
      "metrics/advantage_var": 188.5988311767578,
      "regularization/lipschitz_norm": 914.9808959960938,
      "regularization/w1": 0.8857014775276184,
      "rewards/chosen": 0.3696199304917279,
      "rewards/margins": 0.15209650608444283,
      "rewards/rejected": 0.2175234244072851,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 23.3071346282959,
      "kl": 12.652411460876465,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -36364664.49445325,
      "logits/rejected": -37326598.705701075,
      "logps/chosen": -472.49742472266246,
      "logps/rejected": -378.2548632511556,
      "loss": 0.597,
      "loss/base_kto": 3.830272674560547,
      "metrics/advantage_var": 198.0278778076172,
      "regularization/lipschitz_norm": 977.1329956054688,
      "regularization/w1": 0.9458647966384888,
      "rewards/chosen": 0.25103769438391443,
      "rewards/margins": 0.1762750531942687,
      "rewards/rejected": 0.07476264118964573,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 19.439594268798828,
      "kl": 13.954642295837402,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -36075934.784810126,
      "logits/rejected": -37502849.58024691,
      "logps/chosen": -489.05795094936707,
      "logps/rejected": -367.4696662808642,
      "loss": 0.5944,
      "loss/base_kto": 3.816574811935425,
      "metrics/advantage_var": 201.4392852783203,
      "regularization/lipschitz_norm": 970.0056762695312,
      "regularization/w1": 0.9389654994010925,
      "rewards/chosen": 0.25878925564922867,
      "rewards/margins": 0.17822355582464822,
      "rewards/rejected": 0.08056569982458044,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 20.076038360595703,
      "kl": 9.410046577453613,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36244703.594936706,
      "logits/rejected": -38545262.61728395,
      "logps/chosen": -520.6321696993671,
      "logps/rejected": -395.3758921682099,
      "loss": 0.6085,
      "loss/base_kto": 3.821716070175171,
      "metrics/advantage_var": 237.119140625,
      "regularization/lipschitz_norm": 1080.6380615234375,
      "regularization/w1": 1.0460575819015503,
      "rewards/chosen": 0.0688416444802586,
      "rewards/margins": 0.16230471802234725,
      "rewards/rejected": -0.09346307354208863,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 23.015117645263672,
      "kl": 10.640921592712402,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -36794961.52866242,
      "logits/rejected": -38058430.03680982,
      "logps/chosen": -518.9075935509554,
      "logps/rejected": -361.63587998466255,
      "loss": 0.5899,
      "loss/base_kto": 3.789363145828247,
      "metrics/advantage_var": 214.5233612060547,
      "regularization/lipschitz_norm": 960.4099731445312,
      "regularization/w1": 0.9296768307685852,
      "rewards/chosen": 0.1530699517316879,
      "rewards/margins": 0.2038959425138815,
      "rewards/rejected": -0.05082599078219361,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 22.466655731201172,
      "kl": 6.4207682609558105,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37311027.848101266,
      "logits/rejected": -37460992.0,
      "logps/chosen": -471.9447685917722,
      "logps/rejected": -413.5896508487654,
      "loss": 0.5979,
      "loss/base_kto": 3.8489441871643066,
      "metrics/advantage_var": 212.0841827392578,
      "regularization/lipschitz_norm": 964.771484375,
      "regularization/w1": 0.9338987469673157,
      "rewards/chosen": 0.037564694126950036,
      "rewards/margins": 0.13484609713869591,
      "rewards/rejected": -0.09728140301174587,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 17.599266052246094,
      "kl": 15.75390625,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -35155466.581875995,
      "logits/rejected": -34671661.615975425,
      "logps/chosen": -491.6408485691574,
      "logps/rejected": -387.7569604454685,
      "loss": 0.5901,
      "loss/base_kto": 3.7492966651916504,
      "metrics/advantage_var": 238.36729431152344,
      "regularization/lipschitz_norm": 1003.5248413085938,
      "regularization/w1": 0.9714121222496033,
      "rewards/chosen": 0.2973107851935115,
      "rewards/margins": 0.25503296124871655,
      "rewards/rejected": 0.042277823944794964,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 23.92975425720215,
      "kl": 12.21552562713623,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36237630.8502994,
      "logits/rejected": -37203927.84313726,
      "logps/chosen": -476.83004303892216,
      "logps/rejected": -352.4202920751634,
      "loss": 0.5994,
      "loss/base_kto": 3.8031787872314453,
      "metrics/advantage_var": 247.7328643798828,
      "regularization/lipschitz_norm": 1025.1199951171875,
      "regularization/w1": 0.9923161864280701,
      "rewards/chosen": 0.2277440510824055,
      "rewards/margins": 0.17849620341937383,
      "rewards/rejected": 0.04924784766303168,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 19.471908569335938,
      "kl": 15.156867027282715,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -36620967.56363636,
      "logits/rejected": -37121526.090322584,
      "logps/chosen": -485.28669507575756,
      "logps/rejected": -359.53321572580643,
      "loss": 0.5936,
      "loss/base_kto": 3.734309434890747,
      "metrics/advantage_var": 276.2356872558594,
      "regularization/lipschitz_norm": 1047.706787109375,
      "regularization/w1": 1.0141801834106445,
      "rewards/chosen": 0.3124423865116004,
      "rewards/margins": 0.24593759426739675,
      "rewards/rejected": 0.06650479224420362,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 19.602449417114258,
      "kl": 12.539558410644531,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36373920.57050243,
      "logits/rejected": -36801262.62443439,
      "logps/chosen": -494.39723460291737,
      "logps/rejected": -369.23934766214177,
      "loss": 0.5855,
      "loss/base_kto": 3.709517002105713,
      "metrics/advantage_var": 230.6935272216797,
      "regularization/lipschitz_norm": 1006.8311767578125,
      "regularization/w1": 0.9746125340461731,
      "rewards/chosen": 0.1949175520886092,
      "rewards/margins": 0.2636557213487637,
      "rewards/rejected": -0.06873816926015448,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 30.41884422302246,
      "kl": 14.273750305175781,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -36266737.664,
      "logits/rejected": -36792366.900763355,
      "logps/chosen": -467.03355,
      "logps/rejected": -394.8055104961832,
      "loss": 0.6132,
      "loss/base_kto": 3.8142249584198,
      "metrics/advantage_var": 344.7950134277344,
      "regularization/lipschitz_norm": 1127.3157958984375,
      "regularization/w1": 1.0912418365478516,
      "rewards/chosen": 0.196239599609375,
      "rewards/margins": 0.16764388926265805,
      "rewards/rejected": 0.02859571034671696,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.971755981445312,
      "kl": 17.654523849487305,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -36827531.173047476,
      "logits/rejected": -37443430.48165869,
      "logps/chosen": -465.478943338438,
      "logps/rejected": -361.5318480861244,
      "loss": 0.5973,
      "loss/base_kto": 3.7974212169647217,
      "metrics/advantage_var": 233.09898376464844,
      "regularization/lipschitz_norm": 1013.0390625,
      "regularization/w1": 0.9806217551231384,
      "rewards/chosen": 0.2925348208837816,
      "rewards/margins": 0.18011168916310027,
      "rewards/rejected": 0.11242313172068132,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 31.38916015625,
      "kl": 8.756335258483887,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -36462891.02886248,
      "logits/rejected": -37140394.04920405,
      "logps/chosen": -507.6125848896435,
      "logps/rejected": -369.0596960926194,
      "loss": 0.5895,
      "loss/base_kto": 3.728153944015503,
      "metrics/advantage_var": 239.4797821044922,
      "regularization/lipschitz_norm": 1020.1640625,
      "regularization/w1": 0.9875189065933228,
      "rewards/chosen": 0.02916706519134988,
      "rewards/margins": 0.22851897076015867,
      "rewards/rejected": -0.1993519055688088,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 16.97059440612793,
      "kl": 3.486614227294922,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -35607642.35294118,
      "logits/rejected": -35927898.443113774,
      "logps/chosen": -454.1840788398693,
      "logps/rejected": -358.99178985778445,
      "loss": 0.5896,
      "loss/base_kto": 3.7648956775665283,
      "metrics/advantage_var": 245.7963409423828,
      "regularization/lipschitz_norm": 983.54443359375,
      "regularization/w1": 0.9520710110664368,
      "rewards/chosen": -0.143492904363894,
      "rewards/margins": 0.20866043970480452,
      "rewards/rejected": -0.3521533440686985,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 20.872982025146484,
      "kl": 0.5386886596679688,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -34554314.01916933,
      "logits/rejected": -35879431.82874618,
      "logps/chosen": -481.6983326677316,
      "logps/rejected": -379.27814411314984,
      "loss": 0.5953,
      "loss/base_kto": 3.794321060180664,
      "metrics/advantage_var": 223.24380493164062,
      "regularization/lipschitz_norm": 1000.4750366210938,
      "regularization/w1": 0.9684597253799438,
      "rewards/chosen": -0.296756829697484,
      "rewards/margins": 0.20141230226686996,
      "rewards/rejected": -0.498169131964354,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 18.131797790527344,
      "kl": 4.174106121063232,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -36220364.102874435,
      "logits/rejected": -36813531.19224556,
      "logps/chosen": -485.3890885022693,
      "logps/rejected": -375.6084915185784,
      "loss": 0.5884,
      "loss/base_kto": 3.710728406906128,
      "metrics/advantage_var": 244.85513305664062,
      "regularization/lipschitz_norm": 1029.1322021484375,
      "regularization/w1": 0.9962000250816345,
      "rewards/chosen": 0.08689130128061176,
      "rewards/margins": 0.3007636161281481,
      "rewards/rejected": -0.21387231484753635,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 18.12434959411621,
      "kl": 3.3283486366271973,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -35386618.98039216,
      "logits/rejected": -36065979.01796407,
      "logps/chosen": -488.015318627451,
      "logps/rejected": -381.42058851047904,
      "loss": 0.5881,
      "loss/base_kto": 3.782311201095581,
      "metrics/advantage_var": 210.64688110351562,
      "regularization/lipschitz_norm": 953.1384887695312,
      "regularization/w1": 0.9226380586624146,
      "rewards/chosen": -0.1236166361889808,
      "rewards/margins": 0.19476891631441087,
      "rewards/rejected": -0.3183855525033917,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 28.6573486328125,
      "kl": 7.071713447570801,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35926649.135646686,
      "logits/rejected": -37297513.4117647,
      "logps/chosen": -496.1498915615142,
      "logps/rejected": -359.3993566176471,
      "loss": 0.5879,
      "loss/base_kto": 3.6792073249816895,
      "metrics/advantage_var": 269.59552001953125,
      "regularization/lipschitz_norm": 1057.7291259765625,
      "regularization/w1": 1.0238817930221558,
      "rewards/chosen": 0.08306384161819795,
      "rewards/margins": 0.3106822214768231,
      "rewards/rejected": -0.22761837985862518,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 24.90556526184082,
      "kl": 1.9013442993164062,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36500341.44945568,
      "logits/rejected": -36897320.99215071,
      "logps/chosen": -469.07518468118195,
      "logps/rejected": -380.4013196624804,
      "loss": 0.5959,
      "loss/base_kto": 3.786285161972046,
      "metrics/advantage_var": 239.63487243652344,
      "regularization/lipschitz_norm": 1013.4304809570312,
      "regularization/w1": 0.981000542640686,
      "rewards/chosen": -0.10877354742023473,
      "rewards/margins": 0.2145892970978309,
      "rewards/rejected": -0.32336284451806563,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 22.47072410583496,
      "kl": 9.251765251159668,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -36450395.60383387,
      "logits/rejected": -38275497.88379205,
      "logps/chosen": -506.5962959265176,
      "logps/rejected": -361.931312117737,
      "loss": 0.5938,
      "loss/base_kto": 3.7474324703216553,
      "metrics/advantage_var": 252.8069610595703,
      "regularization/lipschitz_norm": 1036.5286865234375,
      "regularization/w1": 1.0033597946166992,
      "rewards/chosen": 0.11604803895797973,
      "rewards/margins": 0.21459335840142868,
      "rewards/rejected": -0.09854531944344896,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 20.507471084594727,
      "kl": 5.03880500793457,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -34750984.95389507,
      "logits/rejected": -35568770.556067586,
      "logps/chosen": -494.1115858505564,
      "logps/rejected": -365.92693932411674,
      "loss": 0.5961,
      "loss/base_kto": 3.7468647956848145,
      "metrics/advantage_var": 250.00405883789062,
      "regularization/lipschitz_norm": 1055.8797607421875,
      "regularization/w1": 1.0220916271209717,
      "rewards/chosen": -0.052434487638488673,
      "rewards/margins": 0.24706647729826725,
      "rewards/rejected": -0.29950096493675593,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 17.546710968017578,
      "kl": 4.73122501373291,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -37730722.687797144,
      "logits/rejected": -38295618.26810478,
      "logps/chosen": -504.9932151347068,
      "logps/rejected": -371.80727561633284,
      "loss": 0.5853,
      "loss/base_kto": 3.669316053390503,
      "metrics/advantage_var": 250.6295928955078,
      "regularization/lipschitz_norm": 1046.5223388671875,
      "regularization/w1": 1.0130335092544556,
      "rewards/chosen": 0.06241330783076218,
      "rewards/margins": 0.3358486447974899,
      "rewards/rejected": -0.27343533696672767,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 26.337894439697266,
      "kl": 6.717900276184082,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -35879189.8044164,
      "logits/rejected": -37414303.306501545,
      "logps/chosen": -473.075660488959,
      "logps/rejected": -363.0831317724458,
      "loss": 0.5945,
      "loss/base_kto": 3.766246795654297,
      "metrics/advantage_var": 261.7023010253906,
      "regularization/lipschitz_norm": 1022.6945190429688,
      "regularization/w1": 0.9899682998657227,
      "rewards/chosen": 0.006618206433317263,
      "rewards/margins": 0.21099697814142485,
      "rewards/rejected": -0.2043787717081076,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 24.658538818359375,
      "kl": 6.590710639953613,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -36786272.452119306,
      "logits/rejected": -37790202.42612753,
      "logps/chosen": -471.8648940345369,
      "logps/rejected": -385.26095937014,
      "loss": 0.5953,
      "loss/base_kto": 3.7582104206085205,
      "metrics/advantage_var": 271.61322021484375,
      "regularization/lipschitz_norm": 1037.139892578125,
      "regularization/w1": 1.0039513111114502,
      "rewards/chosen": 0.08973833380538965,
      "rewards/margins": 0.2203102953936782,
      "rewards/rejected": -0.13057196158828854,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 18.163043975830078,
      "kl": 8.109402656555176,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -36896270.44514106,
      "logits/rejected": -38517629.20872274,
      "logps/chosen": -502.2341300940439,
      "logps/rejected": -381.8467679127726,
      "loss": 0.5978,
      "loss/base_kto": 3.7055413722991943,
      "metrics/advantage_var": 270.1628112792969,
      "regularization/lipschitz_norm": 1112.8507080078125,
      "regularization/w1": 1.0772393941879272,
      "rewards/chosen": 0.1265721575219803,
      "rewards/margins": 0.2647886931436947,
      "rewards/rejected": -0.13821653562171438,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 20.029281616210938,
      "kl": 8.550772666931152,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -36633565.2345679,
      "logits/rejected": -37684962.835443035,
      "logps/chosen": -512.2664930555555,
      "logps/rejected": -355.15822784810126,
      "loss": 0.6043,
      "loss/base_kto": 3.7266175746917725,
      "metrics/advantage_var": 276.9561462402344,
      "regularization/lipschitz_norm": 1144.205322265625,
      "regularization/w1": 1.1075905561447144,
      "rewards/chosen": 0.10596262378457152,
      "rewards/margins": 0.2601121983540805,
      "rewards/rejected": -0.154149574569509,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 23.614953994750977,
      "kl": 10.841440200805664,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -36445322.37837838,
      "logits/rejected": -37434868.3257329,
      "logps/chosen": -477.1833708708709,
      "logps/rejected": -370.58718444625407,
      "loss": 0.5932,
      "loss/base_kto": 3.6958107948303223,
      "metrics/advantage_var": 262.4812316894531,
      "regularization/lipschitz_norm": 1084.862548828125,
      "regularization/w1": 1.0501469373703003,
      "rewards/chosen": 0.2390412110108155,
      "rewards/margins": 0.27143374390618014,
      "rewards/rejected": -0.032392532895364666,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 19.80830955505371,
      "kl": 12.25365924835205,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -36969143.83382789,
      "logits/rejected": -37143487.78877888,
      "logps/chosen": -472.02499072700294,
      "logps/rejected": -382.21176258250824,
      "loss": 0.5975,
      "loss/base_kto": 3.7258071899414062,
      "metrics/advantage_var": 252.5930633544922,
      "regularization/lipschitz_norm": 1089.2669677734375,
      "regularization/w1": 1.0544103384017944,
      "rewards/chosen": 0.252256240618335,
      "rewards/margins": 0.2503932087274155,
      "rewards/rejected": 0.0018630318909194997,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 22.702198028564453,
      "kl": 19.946395874023438,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -36473962.985074624,
      "logits/rejected": -36656322.727868855,
      "logps/chosen": -487.625,
      "logps/rejected": -374.4955430327869,
      "loss": 0.5919,
      "loss/base_kto": 3.726224660873413,
      "metrics/advantage_var": 241.41099548339844,
      "regularization/lipschitz_norm": 1042.0133056640625,
      "regularization/w1": 1.0086688995361328,
      "rewards/chosen": 0.3612498724638526,
      "rewards/margins": 0.21583413800080187,
      "rewards/rejected": 0.14541573446305073,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 18.53467559814453,
      "kl": 16.435373306274414,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -36852501.6677116,
      "logits/rejected": -37943509.732087225,
      "logps/chosen": -503.9807014106583,
      "logps/rejected": -397.40917056074767,
      "loss": 0.6013,
      "loss/base_kto": 3.7544076442718506,
      "metrics/advantage_var": 254.3803253173828,
      "regularization/lipschitz_norm": 1091.273193359375,
      "regularization/w1": 1.0563524961471558,
      "rewards/chosen": 0.28470121282021454,
      "rewards/margins": 0.23722534749476215,
      "rewards/rejected": 0.04747586532545238,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 21.778186798095703,
      "kl": 19.962522506713867,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -35755665.38271605,
      "logits/rejected": -37403887.79746836,
      "logps/chosen": -488.8063753858025,
      "logps/rejected": -359.5591871044304,
      "loss": 0.5913,
      "loss/base_kto": 3.6668612957000732,
      "metrics/advantage_var": 275.1253356933594,
      "regularization/lipschitz_norm": 1098.4267578125,
      "regularization/w1": 1.0632771253585815,
      "rewards/chosen": 0.3463015144253955,
      "rewards/margins": 0.2926325506671441,
      "rewards/rejected": 0.05366896375825134,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 20.468721389770508,
      "kl": 11.175115585327148,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -36549664.820512824,
      "logits/rejected": -36053822.43902439,
      "logps/chosen": -485.66225961538464,
      "logps/rejected": -384.3454649390244,
      "loss": 0.6022,
      "loss/base_kto": 3.7352349758148193,
      "metrics/advantage_var": 276.24713134765625,
      "regularization/lipschitz_norm": 1118.5439453125,
      "regularization/w1": 1.0827505588531494,
      "rewards/chosen": 0.08928127777882111,
      "rewards/margins": 0.2386458422557647,
      "rewards/rejected": -0.1493645644769436,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 26.680635452270508,
      "kl": 22.314111709594727,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -35876984.29368576,
      "logits/rejected": -36651705.48247079,
      "logps/chosen": -486.5537353157122,
      "logps/rejected": -367.5164858096828,
      "loss": 0.592,
      "loss/base_kto": 3.6413025856018066,
      "metrics/advantage_var": 287.0975341796875,
      "regularization/lipschitz_norm": 1130.9725341796875,
      "regularization/w1": 1.0947813987731934,
      "rewards/chosen": 0.4582015386236922,
      "rewards/margins": 0.3339961929221975,
      "rewards/rejected": 0.12420534570149468,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 19.050148010253906,
      "kl": 21.00868034362793,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -36205102.2503962,
      "logits/rejected": -36348233.19938176,
      "logps/chosen": -469.5121830427892,
      "logps/rejected": -360.2412335780526,
      "loss": 0.5923,
      "loss/base_kto": 3.6048240661621094,
      "metrics/advantage_var": 343.6983337402344,
      "regularization/lipschitz_norm": 1171.210205078125,
      "regularization/w1": 1.133731484413147,
      "rewards/chosen": 0.4154094810909023,
      "rewards/margins": 0.3731467095080409,
      "rewards/rejected": 0.0422627715828614,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 20.98898696899414,
      "kl": 15.312861442565918,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36659257.33125972,
      "logits/rejected": -38118092.96075353,
      "logps/chosen": -483.23673211508554,
      "logps/rejected": -379.24774332810046,
      "loss": 0.6152,
      "loss/base_kto": 3.3470497131347656,
      "metrics/advantage_var": 596.2748413085938,
      "regularization/lipschitz_norm": 1626.3515625,
      "regularization/w1": 1.5743082761764526,
      "rewards/chosen": 0.44879553811175643,
      "rewards/margins": 0.7036231123581016,
      "rewards/rejected": -0.25482757424634517,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 28.101476669311523,
      "kl": 22.525440216064453,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -36435304.2962963,
      "logits/rejected": -38330821.67088608,
      "logps/chosen": -492.3283179012346,
      "logps/rejected": -356.2132367484177,
      "loss": 0.5946,
      "loss/base_kto": 3.263479709625244,
      "metrics/advantage_var": 544.399169921875,
      "regularization/lipschitz_norm": 1542.4775390625,
      "regularization/w1": 1.4931182861328125,
      "rewards/chosen": 0.5767975795416185,
      "rewards/margins": 0.7730232649509712,
      "rewards/rejected": -0.19622568540935276,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 20.487337112426758,
      "kl": 10.790139198303223,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -36793257.6,
      "logits/rejected": -37368236.8,
      "logps/chosen": -472.59248046875,
      "logps/rejected": -397.316064453125,
      "loss": 0.6087,
      "loss/base_kto": 3.279811143875122,
      "metrics/advantage_var": 677.769287109375,
      "regularization/lipschitz_norm": 1641.9990234375,
      "regularization/w1": 1.589455008506775,
      "rewards/chosen": 0.4584324359893799,
      "rewards/margins": 0.8360633373260498,
      "rewards/rejected": -0.37763090133666993,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 21.426246643066406,
      "kl": 16.129098892211914,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -35670705.10691824,
      "logits/rejected": -37336127.60248447,
      "logps/chosen": -498.58392295597486,
      "logps/rejected": -377.6365974378882,
      "loss": 0.5985,
      "loss/base_kto": 3.302999973297119,
      "metrics/advantage_var": 613.1333618164062,
      "regularization/lipschitz_norm": 1533.8465576171875,
      "regularization/w1": 1.484763503074646,
      "rewards/chosen": 0.5011485837540537,
      "rewards/margins": 0.7670694004954284,
      "rewards/rejected": -0.2659208167413747,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 16.76197052001953,
      "kl": 15.906763076782227,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35275843.43032159,
      "logits/rejected": -36779630.23923445,
      "logps/chosen": -446.0583843797856,
      "logps/rejected": -381.70255183413076,
      "loss": 0.6058,
      "loss/base_kto": 3.334491491317749,
      "metrics/advantage_var": 602.745361328125,
      "regularization/lipschitz_norm": 1562.248046875,
      "regularization/w1": 1.5122560262680054,
      "rewards/chosen": 0.46790195716285415,
      "rewards/margins": 0.7466924447899475,
      "rewards/rejected": -0.2787904876270933,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 17.927444458007812,
      "kl": 14.64819622039795,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -35588947.99372057,
      "logits/rejected": -36438156.14307932,
      "logps/chosen": -470.8847625588697,
      "logps/rejected": -370.08266912908243,
      "loss": 0.6,
      "loss/base_kto": 3.3011441230773926,
      "metrics/advantage_var": 569.7094116210938,
      "regularization/lipschitz_norm": 1548.290771484375,
      "regularization/w1": 1.4987454414367676,
      "rewards/chosen": 0.5717838592768838,
      "rewards/margins": 0.7820919282954005,
      "rewards/rejected": -0.21030806901851673,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 16.97763442993164,
      "kl": 15.206074714660645,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -33858970.51564828,
      "logits/rejected": -35707821.609195404,
      "logps/chosen": -475.7306259314456,
      "logps/rejected": -367.83566810344826,
      "loss": 0.6073,
      "loss/base_kto": 3.3158607482910156,
      "metrics/advantage_var": 592.8421630859375,
      "regularization/lipschitz_norm": 1593.4368896484375,
      "regularization/w1": 1.5424467325210571,
      "rewards/chosen": 0.5607204749935963,
      "rewards/margins": 0.7207379802572522,
      "rewards/rejected": -0.16001750526365585,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 17.47994613647461,
      "kl": 19.318700790405273,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -36048236.62908012,
      "logits/rejected": -36748389.38613861,
      "logps/chosen": -492.2660886498516,
      "logps/rejected": -364.9637737211221,
      "loss": 0.6295,
      "loss/base_kto": 3.355252981185913,
      "metrics/advantage_var": 805.7520141601562,
      "regularization/lipschitz_norm": 1736.2386474609375,
      "regularization/w1": 1.6806787252426147,
      "rewards/chosen": 0.615691097270841,
      "rewards/margins": 0.6931039826786445,
      "rewards/rejected": -0.07741288540780347,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 19.694547653198242,
      "kl": 13.314984321594238,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -35409134.06369427,
      "logits/rejected": -35681192.04907975,
      "logps/chosen": -495.4216261942675,
      "logps/rejected": -389.0094660659509,
      "loss": 0.5994,
      "loss/base_kto": 3.3425984382629395,
      "metrics/advantage_var": 542.2902221679688,
      "regularization/lipschitz_norm": 1500.9410400390625,
      "regularization/w1": 1.4529107809066772,
      "rewards/chosen": 0.4016315435907643,
      "rewards/margins": 0.6922148206319596,
      "rewards/rejected": -0.29058327704119535,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 16.86559295654297,
      "kl": 13.960928916931152,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35313516.838323355,
      "logits/rejected": -37749077.333333336,
      "logps/chosen": -488.21903068862275,
      "logps/rejected": -356.86611519607845,
      "loss": 0.6047,
      "loss/base_kto": 3.299733877182007,
      "metrics/advantage_var": 598.1213989257812,
      "regularization/lipschitz_norm": 1589.00390625,
      "regularization/w1": 1.5381557941436768,
      "rewards/chosen": 0.46841458075060816,
      "rewards/margins": 0.7895914462722073,
      "rewards/rejected": -0.32117686552159924,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 17.032405853271484,
      "kl": 36.559242248535156,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -35297239.476407915,
      "logits/rejected": -36544018.902086675,
      "logps/chosen": -487.763603500761,
      "logps/rejected": -362.2292335473515,
      "loss": 0.5983,
      "loss/base_kto": 3.1903305053710938,
      "metrics/advantage_var": 623.8605346679688,
      "regularization/lipschitz_norm": 1649.0458984375,
      "regularization/w1": 1.5962764024734497,
      "rewards/chosen": 0.9171306081739916,
      "rewards/margins": 0.8126239823816123,
      "rewards/rejected": 0.10450662579237936,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 21.105398178100586,
      "kl": 25.526243209838867,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35300575.447409734,
      "logits/rejected": -34945192.808709174,
      "logps/chosen": -472.9589874411303,
      "logps/rejected": -354.9222152021773,
      "loss": 0.5874,
      "loss/base_kto": 3.2690811157226562,
      "metrics/advantage_var": 516.9476928710938,
      "regularization/lipschitz_norm": 1477.0047607421875,
      "regularization/w1": 1.42974054813385,
      "rewards/chosen": 0.600369583680828,
      "rewards/margins": 0.7541291855645458,
      "rewards/rejected": -0.15375960188371768,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 13.378235816955566,
      "kl": 21.708932876586914,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -35344169.44761905,
      "logits/rejected": -35967908.627692305,
      "logps/chosen": -478.65694444444443,
      "logps/rejected": -372.5554326923077,
      "loss": 0.5985,
      "loss/base_kto": 3.213732957839966,
      "metrics/advantage_var": 622.2228393554688,
      "regularization/lipschitz_norm": 1626.468994140625,
      "regularization/w1": 1.5744218826293945,
      "rewards/chosen": 0.6992359948536706,
      "rewards/margins": 0.876377202035161,
      "rewards/rejected": -0.17714120718149037,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 18.647127151489258,
      "kl": 10.48285961151123,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -36992763.46537842,
      "logits/rejected": -36654718.640364185,
      "logps/chosen": -521.9863627214171,
      "logps/rejected": -379.23852427921094,
      "loss": 0.6143,
      "loss/base_kto": 3.3263046741485596,
      "metrics/advantage_var": 669.826416015625,
      "regularization/lipschitz_norm": 1640.3973388671875,
      "regularization/w1": 1.5879043340682983,
      "rewards/chosen": 0.29602095009624096,
      "rewards/margins": 0.7560132387349217,
      "rewards/rejected": -0.45999228863868075,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 22.58229637145996,
      "kl": 7.155999660491943,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -35135925.6639478,
      "logits/rejected": -36256035.69415292,
      "logps/chosen": -462.15630097879284,
      "logps/rejected": -379.56561562968517,
      "loss": 0.5991,
      "loss/base_kto": 3.3373146057128906,
      "metrics/advantage_var": 530.9617309570312,
      "regularization/lipschitz_norm": 1503.8431396484375,
      "regularization/w1": 1.4557201862335205,
      "rewards/chosen": 0.2899089271831668,
      "rewards/margins": 0.728012461475591,
      "rewards/rejected": -0.4381035342924241,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 25.21034812927246,
      "kl": 6.888988018035889,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -35157406.784810126,
      "logits/rejected": -36752396.641975306,
      "logps/chosen": -491.6643591772152,
      "logps/rejected": -387.1657986111111,
      "loss": 0.6154,
      "loss/base_kto": 3.2402515411376953,
      "metrics/advantage_var": 743.0656127929688,
      "regularization/lipschitz_norm": 1738.9886474609375,
      "regularization/w1": 1.6833410263061523,
      "rewards/chosen": 0.3672110400622404,
      "rewards/margins": 0.8371742079827651,
      "rewards/rejected": -0.46996316792052467,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 24.018444061279297,
      "kl": 10.88753604888916,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34611270.954330705,
      "logits/rejected": -35785139.00155039,
      "logps/chosen": -454.80063976377954,
      "logps/rejected": -369.83585271317827,
      "loss": 0.6133,
      "loss/base_kto": 3.394625186920166,
      "metrics/advantage_var": 565.0968627929688,
      "regularization/lipschitz_norm": 1561.531494140625,
      "regularization/w1": 1.5115625858306885,
      "rewards/chosen": 0.25888282595656986,
      "rewards/margins": 0.645460438072001,
      "rewards/rejected": -0.3865776121154312,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 22.305767059326172,
      "kl": 17.678312301635742,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34485634.56394453,
      "logits/rejected": -35448685.946117274,
      "logps/chosen": -471.58638289676423,
      "logps/rejected": -359.3486776941363,
      "loss": 0.6086,
      "loss/base_kto": 3.3520774841308594,
      "metrics/advantage_var": 577.2743530273438,
      "regularization/lipschitz_norm": 1566.6103515625,
      "regularization/w1": 1.5164788961410522,
      "rewards/chosen": 0.5585925744384389,
      "rewards/margins": 0.691815598737202,
      "rewards/rejected": -0.13322302429876312,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 17.83325958251953,
      "kl": 20.49886131286621,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -36128266.78768233,
      "logits/rejected": -36684159.0346908,
      "logps/chosen": -479.5831645056726,
      "logps/rejected": -368.16640742835597,
      "loss": 0.5997,
      "loss/base_kto": 3.2682435512542725,
      "metrics/advantage_var": 565.1885986328125,
      "regularization/lipschitz_norm": 1580.0950927734375,
      "regularization/w1": 1.5295318365097046,
      "rewards/chosen": 0.5783309070951935,
      "rewards/margins": 0.7609371280597663,
      "rewards/rejected": -0.18260622096457274,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 56.203582763671875,
      "kl": 14.707659721374512,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -35382577.45904173,
      "logits/rejected": -36586529.97156398,
      "logps/chosen": -485.0324091962906,
      "logps/rejected": -366.7071484992101,
      "loss": 0.6185,
      "loss/base_kto": 3.33923602104187,
      "metrics/advantage_var": 803.1249389648438,
      "regularization/lipschitz_norm": 1662.2376708984375,
      "regularization/w1": 1.609046220779419,
      "rewards/chosen": 0.4047441202485027,
      "rewards/margins": 0.7113397161521855,
      "rewards/rejected": -0.30659559590368285,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 21.07695960998535,
      "kl": 11.48717212677002,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -35615333.76397516,
      "logits/rejected": -37745213.18238994,
      "logps/chosen": -474.88538431677017,
      "logps/rejected": -391.0483244889937,
      "loss": 0.6156,
      "loss/base_kto": 3.283986806869507,
      "metrics/advantage_var": 623.0930786132812,
      "regularization/lipschitz_norm": 1694.774658203125,
      "regularization/w1": 1.640541911125183,
      "rewards/chosen": 0.4490723224900524,
      "rewards/margins": 0.7698123564854829,
      "rewards/rejected": -0.32074003399543044,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 22.005277633666992,
      "kl": 14.696810722351074,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -35796294.87947883,
      "logits/rejected": -36376868.132132135,
      "logps/chosen": -486.74857491856676,
      "logps/rejected": -372.3071509009009,
      "loss": 0.6043,
      "loss/base_kto": 3.3149948120117188,
      "metrics/advantage_var": 617.7034301757812,
      "regularization/lipschitz_norm": 1569.4571533203125,
      "regularization/w1": 1.519234538078308,
      "rewards/chosen": 0.40370426737136095,
      "rewards/margins": 0.7390145532431702,
      "rewards/rejected": -0.33531028587180933,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 22.52359390258789,
      "kl": 9.239676475524902,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -35149287.69372129,
      "logits/rejected": -36877392.02551834,
      "logps/chosen": -473.6223200612557,
      "logps/rejected": -379.8645085725678,
      "loss": 0.6108,
      "loss/base_kto": 3.254795789718628,
      "metrics/advantage_var": 659.5950927734375,
      "regularization/lipschitz_norm": 1685.6546630859375,
      "regularization/w1": 1.6317135095596313,
      "rewards/chosen": 0.4534812097710088,
      "rewards/margins": 0.8147187554841019,
      "rewards/rejected": -0.3612375457130931,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 18.924129486083984,
      "kl": 13.207402229309082,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -35524863.19242902,
      "logits/rejected": -37563724.879256964,
      "logps/chosen": -504.6776912460568,
      "logps/rejected": -369.65992647058823,
      "loss": 0.601,
      "loss/base_kto": 3.313378095626831,
      "metrics/advantage_var": 560.6163330078125,
      "regularization/lipschitz_norm": 1544.231201171875,
      "regularization/w1": 1.494815707206726,
      "rewards/chosen": 0.4816870463759365,
      "rewards/margins": 0.7439189735449332,
      "rewards/rejected": -0.26223192716899674,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 23.38343048095703,
      "kl": 10.662211418151855,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -36036365.39076923,
      "logits/rejected": -35813623.06031746,
      "logps/chosen": -472.31115384615384,
      "logps/rejected": -390.9247271825397,
      "loss": 0.6003,
      "loss/base_kto": 3.3069992065429688,
      "metrics/advantage_var": 582.510009765625,
      "regularization/lipschitz_norm": 1544.6085205078125,
      "regularization/w1": 1.4951810836791992,
      "rewards/chosen": 0.37486323429987983,
      "rewards/margins": 0.7670327628372062,
      "rewards/rejected": -0.3921695285373264,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 16.797740936279297,
      "kl": 11.943312644958496,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -35921441.32544379,
      "logits/rejected": -35954749.033112586,
      "logps/chosen": -481.2622965976331,
      "logps/rejected": -377.4881001655629,
      "loss": 0.6092,
      "loss/base_kto": 3.2473723888397217,
      "metrics/advantage_var": 667.6775512695312,
      "regularization/lipschitz_norm": 1679.6678466796875,
      "regularization/w1": 1.6259183883666992,
      "rewards/chosen": 0.5425694031122874,
      "rewards/margins": 0.8230411796072041,
      "rewards/rejected": -0.2804717764949167,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 14.822357177734375,
      "kl": 14.6751708984375,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -36540168.76998369,
      "logits/rejected": -36430263.07646177,
      "logps/chosen": -481.2759482055465,
      "logps/rejected": -383.56221889055473,
      "loss": 0.5997,
      "loss/base_kto": 3.2612533569335938,
      "metrics/advantage_var": 564.8085327148438,
      "regularization/lipschitz_norm": 1587.0164794921875,
      "regularization/w1": 1.5362318754196167,
      "rewards/chosen": 0.5311997181828609,
      "rewards/margins": 0.7970503743977222,
      "rewards/rejected": -0.2658506562148613,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 18.515363693237305,
      "kl": 8.514366149902344,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34707980.962025315,
      "logits/rejected": -36460354.37037037,
      "logps/chosen": -485.23961629746833,
      "logps/rejected": -362.2927035108025,
      "loss": 0.6115,
      "loss/base_kto": 3.3230254650115967,
      "metrics/advantage_var": 613.64697265625,
      "regularization/lipschitz_norm": 1621.0296630859375,
      "regularization/w1": 1.5691566467285156,
      "rewards/chosen": 0.2536700767806814,
      "rewards/margins": 0.7126278430898779,
      "rewards/rejected": -0.45895776630919655,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 18.386857986450195,
      "kl": 13.531366348266602,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -35775914.136645965,
      "logits/rejected": -36766771.52201258,
      "logps/chosen": -472.79347826086956,
      "logps/rejected": -383.12224842767296,
      "loss": 0.6198,
      "loss/base_kto": 3.2859840393066406,
      "metrics/advantage_var": 736.55517578125,
      "regularization/lipschitz_norm": 1727.5960693359375,
      "regularization/w1": 1.6723130941390991,
      "rewards/chosen": 0.49217295202409256,
      "rewards/margins": 0.8244916478601898,
      "rewards/rejected": -0.3323186958360972,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 14.9951753616333,
      "kl": 13.052634239196777,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -35114431.90108192,
      "logits/rejected": -36866433.01105845,
      "logps/chosen": -500.2502897990726,
      "logps/rejected": -387.1023894154818,
      "loss": 0.6036,
      "loss/base_kto": 3.2885186672210693,
      "metrics/advantage_var": 593.1588134765625,
      "regularization/lipschitz_norm": 1591.5728759765625,
      "regularization/w1": 1.540642499923706,
      "rewards/chosen": 0.4901510294659245,
      "rewards/margins": 0.8024950909091388,
      "rewards/rejected": -0.31234406144321436,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 14.314281463623047,
      "kl": 17.484107971191406,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34972305.46388443,
      "logits/rejected": -35680801.509893455,
      "logps/chosen": -491.9517455858748,
      "logps/rejected": -353.6439783105023,
      "loss": 0.5847,
      "loss/base_kto": 3.2217767238616943,
      "metrics/advantage_var": 534.62353515625,
      "regularization/lipschitz_norm": 1504.163330078125,
      "regularization/w1": 1.4560301303863525,
      "rewards/chosen": 0.5592935969320576,
      "rewards/margins": 0.807646298011776,
      "rewards/rejected": -0.2483527010797184,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 24.737625122070312,
      "kl": 11.298369407653809,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -35272291.803486526,
      "logits/rejected": -36781022.86594761,
      "logps/chosen": -482.4072900158479,
      "logps/rejected": -384.65092931432974,
      "loss": 0.612,
      "loss/base_kto": 3.3351597785949707,
      "metrics/advantage_var": 562.4592895507812,
      "regularization/lipschitz_norm": 1612.666748046875,
      "regularization/w1": 1.5610612630844116,
      "rewards/chosen": 0.4083454113943641,
      "rewards/margins": 0.7207761237959555,
      "rewards/rejected": -0.31243071240159137,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 13.026668548583984,
      "kl": 8.759127616882324,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -34090870.81339713,
      "logits/rejected": -35542930.22970904,
      "logps/chosen": -480.42618620414675,
      "logps/rejected": -383.1708221669219,
      "loss": 0.5866,
      "loss/base_kto": 3.3254687786102295,
      "metrics/advantage_var": 476.3669128417969,
      "regularization/lipschitz_norm": 1412.2210693359375,
      "regularization/w1": 1.3670300245285034,
      "rewards/chosen": 0.3927106355365954,
      "rewards/margins": 0.7149817284640696,
      "rewards/rejected": -0.32227109292747413,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 19.02378273010254,
      "kl": 8.384509086608887,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -35174218.81832544,
      "logits/rejected": -35459534.145285934,
      "logps/chosen": -456.5756319115324,
      "logps/rejected": -403.34041731066463,
      "loss": 0.5969,
      "loss/base_kto": 3.3167827129364014,
      "metrics/advantage_var": 710.2484741210938,
      "regularization/lipschitz_norm": 1506.6376953125,
      "regularization/w1": 1.4584254026412964,
      "rewards/chosen": 0.31521430475075285,
      "rewards/margins": 0.7730084942038778,
      "rewards/rejected": -0.457794189453125,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 29.536518096923828,
      "kl": 7.4209303855896,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -35098285.40609952,
      "logits/rejected": -35436578.2891933,
      "logps/chosen": -499.8775080256822,
      "logps/rejected": -374.57667427701676,
      "loss": 0.6097,
      "loss/base_kto": 3.349459171295166,
      "metrics/advantage_var": 566.67578125,
      "regularization/lipschitz_norm": 1578.5618896484375,
      "regularization/w1": 1.5280479192733765,
      "rewards/chosen": 0.22507051862836075,
      "rewards/margins": 0.6925968427103623,
      "rewards/rejected": -0.4675263240820015,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 14.033746719360352,
      "kl": 7.5047197341918945,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35247297.9029734,
      "logits/rejected": -36115352.16224649,
      "logps/chosen": -480.89666471048514,
      "logps/rejected": -391.7812012480499,
      "loss": 0.6025,
      "loss/base_kto": 3.294116973876953,
      "metrics/advantage_var": 564.1644897460938,
      "regularization/lipschitz_norm": 1576.3626708984375,
      "regularization/w1": 1.525918960571289,
      "rewards/chosen": 0.35076068525956083,
      "rewards/margins": 0.7848183405345218,
      "rewards/rejected": -0.434057655274961,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 14.298561096191406,
      "kl": 13.422953605651855,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -35913955.55555555,
      "logits/rejected": -37833948.35443038,
      "logps/chosen": -491.8377700617284,
      "logps/rejected": -366.3362836234177,
      "loss": 0.6036,
      "loss/base_kto": 3.31005859375,
      "metrics/advantage_var": 584.8604736328125,
      "regularization/lipschitz_norm": 1569.2012939453125,
      "regularization/w1": 1.5189868211746216,
      "rewards/chosen": 0.4509073422278887,
      "rewards/margins": 0.7458736656941294,
      "rewards/rejected": -0.29496632346624063,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 23.15932273864746,
      "kl": 14.832297325134277,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -34271518.14128728,
      "logits/rejected": -35675739.57076205,
      "logps/chosen": -498.0529336734694,
      "logps/rejected": -380.7037811041991,
      "loss": 0.601,
      "loss/base_kto": 3.275827169418335,
      "metrics/advantage_var": 644.1874389648438,
      "regularization/lipschitz_norm": 1582.501220703125,
      "regularization/w1": 1.5318611860275269,
      "rewards/chosen": 0.5628489160462863,
      "rewards/margins": 0.7810239705697603,
      "rewards/rejected": -0.21817505452347394,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 18.470226287841797,
      "kl": 10.67943286895752,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -34660532.522796355,
      "logits/rejected": -36501913.6,
      "logps/chosen": -489.24572568389056,
      "logps/rejected": -350.358341733871,
      "loss": 0.5703,
      "loss/base_kto": 3.2103021144866943,
      "metrics/advantage_var": 453.81402587890625,
      "regularization/lipschitz_norm": 1396.7904052734375,
      "regularization/w1": 1.3520931005477905,
      "rewards/chosen": 0.4466327330986417,
      "rewards/margins": 0.8437312497474572,
      "rewards/rejected": -0.3970985166488155,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 16.989198684692383,
      "kl": 11.994152069091797,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -33972006.54199683,
      "logits/rejected": -35205495.5192604,
      "logps/chosen": -465.40421949286844,
      "logps/rejected": -392.57966583204933,
      "loss": 0.5575,
      "loss/base_kto": 3.219650983810425,
      "metrics/advantage_var": 403.4252624511719,
      "regularization/lipschitz_norm": 1281.2525634765625,
      "regularization/w1": 1.2402524948120117,
      "rewards/chosen": 0.46899085188440964,
      "rewards/margins": 0.8060005110505546,
      "rewards/rejected": -0.337009659166145,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 14.959163665771484,
      "kl": 12.547826766967773,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -35060381.596465394,
      "logits/rejected": -36498508.67221298,
      "logps/chosen": -488.4973766568483,
      "logps/rejected": -364.11168885191347,
      "loss": 0.5623,
      "loss/base_kto": 3.1705167293548584,
      "metrics/advantage_var": 467.0064392089844,
      "regularization/lipschitz_norm": 1371.7470703125,
      "regularization/w1": 1.3278511762619019,
      "rewards/chosen": 0.5463290994051454,
      "rewards/margins": 0.8845043350963111,
      "rewards/rejected": -0.33817523569116575,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 20.10413932800293,
      "kl": 15.856521606445312,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -36529925.286135696,
      "logits/rejected": -36900765.342192695,
      "logps/chosen": -496.71916482300884,
      "logps/rejected": -396.3875103820598,
      "loss": 0.5623,
      "loss/base_kto": 3.1664652824401855,
      "metrics/advantage_var": 468.0096740722656,
      "regularization/lipschitz_norm": 1375.593994140625,
      "regularization/w1": 1.3315749168395996,
      "rewards/chosen": 0.5738953446919939,
      "rewards/margins": 0.8824426339605129,
      "rewards/rejected": -0.308547289268519,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 17.898967742919922,
      "kl": 16.204326629638672,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -35384816.484848484,
      "logits/rejected": -36924283.87096774,
      "logps/chosen": -484.1930871212121,
      "logps/rejected": -385.1054183467742,
      "loss": 0.5616,
      "loss/base_kto": 3.196605682373047,
      "metrics/advantage_var": 449.0083923339844,
      "regularization/lipschitz_norm": 1338.77099609375,
      "regularization/w1": 1.29593026638031,
      "rewards/chosen": 0.6323146473277699,
      "rewards/margins": 0.8422431968174372,
      "rewards/rejected": -0.20992854948966733,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 17.83039665222168,
      "kl": 20.723678588867188,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -35518300.88495575,
      "logits/rejected": -36610558.299003325,
      "logps/chosen": -494.404959439528,
      "logps/rejected": -388.093127076412,
      "loss": 0.5603,
      "loss/base_kto": 3.1352014541625977,
      "metrics/advantage_var": 509.0668640136719,
      "regularization/lipschitz_norm": 1391.6597900390625,
      "regularization/w1": 1.3471266031265259,
      "rewards/chosen": 0.7286337343289085,
      "rewards/margins": 0.9123075296044225,
      "rewards/rejected": -0.18367379527551392,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 18.897979736328125,
      "kl": 22.381803512573242,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -35291014.36918138,
      "logits/rejected": -36207231.02587519,
      "logps/chosen": -474.2518057784912,
      "logps/rejected": -376.01084474885846,
      "loss": 0.5608,
      "loss/base_kto": 3.1498916149139404,
      "metrics/advantage_var": 488.4566345214844,
      "regularization/lipschitz_norm": 1380.9681396484375,
      "regularization/w1": 1.3367770910263062,
      "rewards/chosen": 0.7305796027757574,
      "rewards/margins": 0.9073193462683136,
      "rewards/rejected": -0.17673974349255614,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 16.471464157104492,
      "kl": 13.330780029296875,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34937962.69984448,
      "logits/rejected": -35513027.3155416,
      "logps/chosen": -488.04417768273714,
      "logps/rejected": -349.43153944270017,
      "loss": 0.5527,
      "loss/base_kto": 3.155369520187378,
      "metrics/advantage_var": 420.8874206542969,
      "regularization/lipschitz_norm": 1307.99560546875,
      "regularization/w1": 1.2661398649215698,
      "rewards/chosen": 0.5613595538339571,
      "rewards/margins": 0.8743365047743392,
      "rewards/rejected": -0.31297695094038214,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 14.699928283691406,
      "kl": 16.41225242614746,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34938240.81528662,
      "logits/rejected": -35802017.76687117,
      "logps/chosen": -442.6795382165605,
      "logps/rejected": -391.77993194018404,
      "loss": 0.5536,
      "loss/base_kto": 3.1832523345947266,
      "metrics/advantage_var": 439.9745178222656,
      "regularization/lipschitz_norm": 1287.1234130859375,
      "regularization/w1": 1.2459354400634766,
      "rewards/chosen": 0.5204137814272741,
      "rewards/margins": 0.8519921355899467,
      "rewards/rejected": -0.33157835416267256,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 15.27790641784668,
      "kl": 12.06608772277832,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -35155774.1829653,
      "logits/rejected": -35900771.071207434,
      "logps/chosen": -468.29574132492115,
      "logps/rejected": -375.16036184210526,
      "loss": 0.5639,
      "loss/base_kto": 3.1814515590667725,
      "metrics/advantage_var": 439.0970764160156,
      "regularization/lipschitz_norm": 1373.935546875,
      "regularization/w1": 1.3299696445465088,
      "rewards/chosen": 0.5102838462083498,
      "rewards/margins": 0.8738979078769111,
      "rewards/rejected": -0.36361406166856136,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 19.957759857177734,
      "kl": 13.255197525024414,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -35628739.90996785,
      "logits/rejected": -37262921.14285714,
      "logps/chosen": -494.9699557877814,
      "logps/rejected": -357.8082731762918,
      "loss": 0.5527,
      "loss/base_kto": 3.2057883739471436,
      "metrics/advantage_var": 396.0398254394531,
      "regularization/lipschitz_norm": 1256.2628173828125,
      "regularization/w1": 1.2160624265670776,
      "rewards/chosen": 0.46789781380313,
      "rewards/margins": 0.8238110166164612,
      "rewards/rejected": -0.3559132028133311,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 10.981877326965332,
      "kl": 13.450248718261719,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -36129261.9109063,
      "logits/rejected": -37033179.777424484,
      "logps/chosen": -459.95065284178185,
      "logps/rejected": -370.7652027027027,
      "loss": 0.5522,
      "loss/base_kto": 3.215421438217163,
      "metrics/advantage_var": 434.76611328125,
      "regularization/lipschitz_norm": 1241.7470703125,
      "regularization/w1": 1.2020111083984375,
      "rewards/chosen": 0.4774749802737375,
      "rewards/margins": 0.8330701671713429,
      "rewards/rejected": -0.35559518689760533,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 19.0456600189209,
      "kl": 12.843103408813477,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -34794248.827586204,
      "logits/rejected": -35015555.588785045,
      "logps/chosen": -481.31044278996865,
      "logps/rejected": -356.7027112538941,
      "loss": 0.5563,
      "loss/base_kto": 3.2093822956085205,
      "metrics/advantage_var": 446.06109619140625,
      "regularization/lipschitz_norm": 1282.4368896484375,
      "regularization/w1": 1.2413989305496216,
      "rewards/chosen": 0.5343358628802165,
      "rewards/margins": 0.8298741264091057,
      "rewards/rejected": -0.29553826352888923,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 20.85038948059082,
      "kl": 11.654293060302734,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -34359750.409861326,
      "logits/rejected": -37603863.530903324,
      "logps/chosen": -513.1727657935285,
      "logps/rejected": -380.268868858954,
      "loss": 0.5565,
      "loss/base_kto": 3.168382406234741,
      "metrics/advantage_var": 427.8387756347656,
      "regularization/lipschitz_norm": 1326.203857421875,
      "regularization/w1": 1.2837653160095215,
      "rewards/chosen": 0.523593708620233,
      "rewards/margins": 0.8711015300191782,
      "rewards/rejected": -0.34750782139894515,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 16.17123031616211,
      "kl": 15.288296699523926,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -35400721.41496599,
      "logits/rejected": -36835588.439306356,
      "logps/chosen": -487.6054421768707,
      "logps/rejected": -391.5633128612717,
      "loss": 0.5468,
      "loss/base_kto": 3.1300837993621826,
      "metrics/advantage_var": 433.1132507324219,
      "regularization/lipschitz_norm": 1285.69287109375,
      "regularization/w1": 1.2445507049560547,
      "rewards/chosen": 0.5404657895873193,
      "rewards/margins": 0.8912913253560377,
      "rewards/rejected": -0.3508255357687184,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 15.69742202758789,
      "kl": 12.333353996276855,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35549174.1696,
      "logits/rejected": -36529338.03969466,
      "logps/chosen": -500.6871,
      "logps/rejected": -380.6728530534351,
      "loss": 0.5564,
      "loss/base_kto": 3.196179151535034,
      "metrics/advantage_var": 434.9308776855469,
      "regularization/lipschitz_norm": 1296.4019775390625,
      "regularization/w1": 1.254917025566101,
      "rewards/chosen": 0.44173544921875,
      "rewards/margins": 0.8416032219107825,
      "rewards/rejected": -0.39986777269203244,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 15.341116905212402,
      "kl": 10.772221565246582,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -35202072.97560976,
      "logits/rejected": -36885296.120300755,
      "logps/chosen": -493.02459349593494,
      "logps/rejected": -349.1362546992481,
      "loss": 0.5593,
      "loss/base_kto": 3.1654787063598633,
      "metrics/advantage_var": 463.2176208496094,
      "regularization/lipschitz_norm": 1351.9886474609375,
      "regularization/w1": 1.3087249994277954,
      "rewards/chosen": 0.4563167913173272,
      "rewards/margins": 0.8843819034384504,
      "rewards/rejected": -0.42806511212112314,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 19.67717742919922,
      "kl": 8.95859146118164,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -35181711.58089368,
      "logits/rejected": -37352813.13470682,
      "logps/chosen": -476.40523882896764,
      "logps/rejected": -364.6350287242472,
      "loss": 0.5583,
      "loss/base_kto": 3.2194983959198,
      "metrics/advantage_var": 410.23345947265625,
      "regularization/lipschitz_norm": 1288.4498291015625,
      "regularization/w1": 1.2472193241119385,
      "rewards/chosen": 0.40437935239912365,
      "rewards/margins": 0.8254889463905559,
      "rewards/rejected": -0.4211095939914323,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 16.2672176361084,
      "kl": 11.774325370788574,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -34668303.765395895,
      "logits/rejected": -35742206.28762542,
      "logps/chosen": -478.99780058651027,
      "logps/rejected": -355.7196906354515,
      "loss": 0.5535,
      "loss/base_kto": 3.1635589599609375,
      "metrics/advantage_var": 428.73614501953125,
      "regularization/lipschitz_norm": 1306.254150390625,
      "regularization/w1": 1.2644538879394531,
      "rewards/chosen": 0.5405383068095904,
      "rewards/margins": 0.8783245188249018,
      "rewards/rejected": -0.33778621201531145,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 17.85692596435547,
      "kl": 15.85357666015625,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -34255347.6626506,
      "logits/rejected": -35674630.64935065,
      "logps/chosen": -487.51430722891564,
      "logps/rejected": -378.6520140016234,
      "loss": 0.5591,
      "loss/base_kto": 3.168553590774536,
      "metrics/advantage_var": 461.27398681640625,
      "regularization/lipschitz_norm": 1346.986328125,
      "regularization/w1": 1.3038827180862427,
      "rewards/chosen": 0.6127265102892037,
      "rewards/margins": 0.8764291753457047,
      "rewards/rejected": -0.26370266505650114,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 17.892295837402344,
      "kl": 14.68474292755127,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -36290994.526315786,
      "logits/rejected": -36778870.85714286,
      "logps/chosen": -473.0989412006579,
      "logps/rejected": -372.2460239955357,
      "loss": 0.5504,
      "loss/base_kto": 3.1353840827941895,
      "metrics/advantage_var": 479.9383850097656,
      "regularization/lipschitz_norm": 1309.4774169921875,
      "regularization/w1": 1.2675741910934448,
      "rewards/chosen": 0.5927888970626028,
      "rewards/margins": 0.9044809986774187,
      "rewards/rejected": -0.31169210161481586,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 17.130151748657227,
      "kl": 13.543744087219238,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35352974.83771252,
      "logits/rejected": -36581376.0,
      "logps/chosen": -472.7634756568779,
      "logps/rejected": -390.00819510268565,
      "loss": 0.559,
      "loss/base_kto": 3.152021884918213,
      "metrics/advantage_var": 429.50201416015625,
      "regularization/lipschitz_norm": 1363.3504638671875,
      "regularization/w1": 1.319723129272461,
      "rewards/chosen": 0.5528380498635529,
      "rewards/margins": 0.8701875244405302,
      "rewards/rejected": -0.3173494745769772,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 20.55853843688965,
      "kl": 14.852238655090332,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -35541704.62695925,
      "logits/rejected": -36462493.10903427,
      "logps/chosen": -484.4791340125392,
      "logps/rejected": -367.09642718068534,
      "loss": 0.564,
      "loss/base_kto": 3.1748039722442627,
      "metrics/advantage_var": 455.13946533203125,
      "regularization/lipschitz_norm": 1381.0677490234375,
      "regularization/w1": 1.3368734121322632,
      "rewards/chosen": 0.5463944677275177,
      "rewards/margins": 0.8718686215254635,
      "rewards/rejected": -0.32547415379794586,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 18.252906799316406,
      "kl": 13.813323974609375,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -35506461.164556965,
      "logits/rejected": -36864281.28395062,
      "logps/chosen": -489.61367681962025,
      "logps/rejected": -373.4073591820988,
      "loss": 0.571,
      "loss/base_kto": 3.1924209594726562,
      "metrics/advantage_var": 476.4051818847656,
      "regularization/lipschitz_norm": 1420.7030029296875,
      "regularization/w1": 1.375240445137024,
      "rewards/chosen": 0.5333185316641119,
      "rewards/margins": 0.853571456453729,
      "rewards/rejected": -0.3202529247896171,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 15.302053451538086,
      "kl": 13.113249778747559,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -36436406.176744185,
      "logits/rejected": -36537416.56692913,
      "logps/chosen": -489.64287790697676,
      "logps/rejected": -368.50002460629923,
      "loss": 0.5556,
      "loss/base_kto": 3.1588809490203857,
      "metrics/advantage_var": 470.2917175292969,
      "regularization/lipschitz_norm": 1328.6754150390625,
      "regularization/w1": 1.2861578464508057,
      "rewards/chosen": 0.5469550554142442,
      "rewards/margins": 0.8917398656381614,
      "rewards/rejected": -0.3447848102239173,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 16.81578254699707,
      "kl": 14.144734382629395,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -33565854.9248,
      "logits/rejected": -35864763.603053436,
      "logps/chosen": -451.1166,
      "logps/rejected": -388.69527671755725,
      "loss": 0.5461,
      "loss/base_kto": 3.16017484664917,
      "metrics/advantage_var": 395.6061096191406,
      "regularization/lipschitz_norm": 1248.55224609375,
      "regularization/w1": 1.2085986137390137,
      "rewards/chosen": 0.4966119140625,
      "rewards/margins": 0.8574853038525763,
      "rewards/rejected": -0.3608733897900763,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 19.078401565551758,
      "kl": 13.644217491149902,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -35438682.73417722,
      "logits/rejected": -36835305.87654321,
      "logps/chosen": -458.0954806170886,
      "logps/rejected": -364.69150270061726,
      "loss": 0.5505,
      "loss/base_kto": 3.154127597808838,
      "metrics/advantage_var": 464.5173034667969,
      "regularization/lipschitz_norm": 1291.393798828125,
      "regularization/w1": 1.25006902217865,
      "rewards/chosen": 0.5648512055602255,
      "rewards/margins": 0.8932003778933659,
      "rewards/rejected": -0.3283491723331404,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 17.28318977355957,
      "kl": 12.8956880569458,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -35547772.54054054,
      "logits/rejected": -37020988.872964166,
      "logps/chosen": -502.237987987988,
      "logps/rejected": -373.9553898615635,
      "loss": 0.5497,
      "loss/base_kto": 3.1797616481781006,
      "metrics/advantage_var": 415.0672912597656,
      "regularization/lipschitz_norm": 1258.2498779296875,
      "regularization/w1": 1.217985987663269,
      "rewards/chosen": 0.5235647023977102,
      "rewards/margins": 0.860215737172242,
      "rewards/rejected": -0.33665103477453173,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 17.75486946105957,
      "kl": 13.473390579223633,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34530838.26086956,
      "logits/rejected": -35509151.39622641,
      "logps/chosen": -475.0430900621118,
      "logps/rejected": -383.69025157232704,
      "loss": 0.5525,
      "loss/base_kto": 3.2205536365509033,
      "metrics/advantage_var": 390.6640625,
      "regularization/lipschitz_norm": 1239.4990234375,
      "regularization/w1": 1.1998350620269775,
      "rewards/chosen": 0.5065290557671778,
      "rewards/margins": 0.7972339148949662,
      "rewards/rejected": -0.29070485912778843,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 19.26053237915039,
      "kl": 14.291238784790039,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -35028423.470031545,
      "logits/rejected": -34800960.19814242,
      "logps/chosen": -477.7087933753943,
      "logps/rejected": -374.2738003095975,
      "loss": 0.5604,
      "loss/base_kto": 3.202681303024292,
      "metrics/advantage_var": 425.21026611328125,
      "regularization/lipschitz_norm": 1323.0657958984375,
      "regularization/w1": 1.280727744102478,
      "rewards/chosen": 0.5387579018385252,
      "rewards/margins": 0.834407238426059,
      "rewards/rejected": -0.29564933658753384,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 20.083316802978516,
      "kl": 12.89837646484375,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -35416430.17721519,
      "logits/rejected": -37350216.69135802,
      "logps/chosen": -478.3454806170886,
      "logps/rejected": -386.7404513888889,
      "loss": 0.558,
      "loss/base_kto": 3.1874847412109375,
      "metrics/advantage_var": 456.527587890625,
      "regularization/lipschitz_norm": 1318.8660888671875,
      "regularization/w1": 1.2766623497009277,
      "rewards/chosen": 0.5069576215140427,
      "rewards/margins": 0.8588423456954181,
      "rewards/rejected": -0.3518847241813754,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 19.203149795532227,
      "kl": 14.212717056274414,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35609565.703349285,
      "logits/rejected": -37047460.65543645,
      "logps/chosen": -476.80746610845296,
      "logps/rejected": -378.27455015313933,
      "loss": 0.5658,
      "loss/base_kto": 3.2434043884277344,
      "metrics/advantage_var": 446.1128845214844,
      "regularization/lipschitz_norm": 1325.3538818359375,
      "regularization/w1": 1.2829424142837524,
      "rewards/chosen": 0.47803357381379585,
      "rewards/margins": 0.7959637866087088,
      "rewards/rejected": -0.3179302127949129,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 20.670875549316406,
      "kl": 11.36286449432373,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -36989975.59447005,
      "logits/rejected": -36957013.06200318,
      "logps/chosen": -482.69378840245776,
      "logps/rejected": -391.89586645469,
      "loss": 0.5605,
      "loss/base_kto": 3.207538604736328,
      "metrics/advantage_var": 426.26690673828125,
      "regularization/lipschitz_norm": 1318.4716796875,
      "regularization/w1": 1.2762806415557861,
      "rewards/chosen": 0.48256902504260274,
      "rewards/margins": 0.8347832136107665,
      "rewards/rejected": -0.3522141885681638,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 20.096817016601562,
      "kl": 12.562037467956543,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -36984000.972644374,
      "logits/rejected": -36085634.881028935,
      "logps/chosen": -502.9935885258359,
      "logps/rejected": -382.9482013665595,
      "loss": 0.5626,
      "loss/base_kto": 3.1672394275665283,
      "metrics/advantage_var": 472.66845703125,
      "regularization/lipschitz_norm": 1377.91748046875,
      "regularization/w1": 1.3338241577148438,
      "rewards/chosen": 0.5462447053393332,
      "rewards/margins": 0.8777947147202994,
      "rewards/rejected": -0.3315500093809661,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 14.879459381103516,
      "kl": 11.950034141540527,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35631020.14488189,
      "logits/rejected": -35859317.87906977,
      "logps/chosen": -462.8613188976378,
      "logps/rejected": -388.47257751937985,
      "loss": 0.5478,
      "loss/base_kto": 3.1338982582092285,
      "metrics/advantage_var": 419.29840087890625,
      "regularization/lipschitz_norm": 1289.6092529296875,
      "regularization/w1": 1.2483415603637695,
      "rewards/chosen": 0.5252166507750984,
      "rewards/margins": 0.872691782497482,
      "rewards/rejected": -0.3474751317223837,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 15.49474048614502,
      "kl": 13.836501121520996,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -36151189.437398374,
      "logits/rejected": -37945337.840601504,
      "logps/chosen": -489.70701219512193,
      "logps/rejected": -376.35352443609025,
      "loss": 0.5572,
      "loss/base_kto": 3.1607234477996826,
      "metrics/advantage_var": 425.92974853515625,
      "regularization/lipschitz_norm": 1339.749755859375,
      "regularization/w1": 1.2968777418136597,
      "rewards/chosen": 0.5414697166380843,
      "rewards/margins": 0.8892156837286623,
      "rewards/rejected": -0.34774596709057803,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 17.457658767700195,
      "kl": 14.280056953430176,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -34176784.37441497,
      "logits/rejected": -36227716.20657277,
      "logps/chosen": -481.05669851794073,
      "logps/rejected": -364.6233372456964,
      "loss": 0.5574,
      "loss/base_kto": 3.2173683643341064,
      "metrics/advantage_var": 414.7599182128906,
      "regularization/lipschitz_norm": 1282.919189453125,
      "regularization/w1": 1.2418657541275024,
      "rewards/chosen": 0.5074455273132801,
      "rewards/margins": 0.7996735454874291,
      "rewards/rejected": -0.29222801817414906,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 14.278672218322754,
      "kl": 13.657382011413574,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34881162.75118859,
      "logits/rejected": -36222960.22187982,
      "logps/chosen": -480.0768621236133,
      "logps/rejected": -370.05342353620955,
      "loss": 0.5551,
      "loss/base_kto": 3.135535955429077,
      "metrics/advantage_var": 473.9613342285156,
      "regularization/lipschitz_norm": 1348.775146484375,
      "regularization/w1": 1.3056143522262573,
      "rewards/chosen": 0.572680489952704,
      "rewards/margins": 0.9162802061627386,
      "rewards/rejected": -0.34359971621003466,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.978042558275912e+17,
      "train_loss": 0.5857396982288525,
      "train_runtime": 11128.919,
      "train_samples_per_second": 13.318,
      "train_steps_per_second": 0.208
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.978042558275912e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}