{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 11.605793952941895,
      "kl": 38.56059265136719,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37154164.36363637,
      "logits/rejected": -37756561.341935486,
      "logps/chosen": -471.05530303030304,
      "logps/rejected": -359.22752016129033,
      "loss": 0.4618,
      "loss/base_kto": 3.694525957107544,
      "metrics/advantage_var": 740.17822265625,
      "rewards/chosen": 0.8659122351444128,
      "rewards/margins": 0.3264073287054008,
      "rewards/rejected": 0.539504906439012,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.088749885559082,
      "kl": 30.996259689331055,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -38410347.78947368,
      "logits/rejected": -39412351.375609756,
      "logps/chosen": -490.34473684210525,
      "logps/rejected": -391.3777947154472,
      "loss": 0.4519,
      "loss/base_kto": 3.6148488521575928,
      "metrics/advantage_var": 912.3471069335938,
      "rewards/chosen": 0.8235496578359962,
      "rewards/margins": 0.35158057437563034,
      "rewards/rejected": 0.47196908346036587,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 10.01998233795166,
      "kl": 30.216571807861328,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35869769.86229508,
      "logits/rejected": -37921590.256716415,
      "logps/chosen": -471.6993852459016,
      "logps/rejected": -353.15212220149255,
      "loss": 0.4291,
      "loss/base_kto": 3.4325599670410156,
      "metrics/advantage_var": 872.025390625,
      "rewards/chosen": 0.8260520059554304,
      "rewards/margins": 0.6611124444247051,
      "rewards/rejected": 0.16493956153072528,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 8.345759391784668,
      "kl": 5.388225078582764,
      "learning_rate": 7.9e-07,
      "logits/chosen": -36489209.467304625,
      "logits/rejected": -38324117.36600307,
      "logps/chosen": -461.50956937799043,
      "logps/rejected": -364.3822741194487,
      "loss": 0.4193,
      "loss/base_kto": 3.354257345199585,
      "metrics/advantage_var": 1499.1781005859375,
      "rewards/chosen": 0.2019310910165595,
      "rewards/margins": 0.8813837972761307,
      "rewards/rejected": -0.6794527062595712,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 8.309015274047852,
      "kl": 13.195367813110352,
      "learning_rate": 9.9e-07,
      "logits/chosen": -38952421.558685444,
      "logits/rejected": -39376905.5850234,
      "logps/chosen": -480.94776995305165,
      "logps/rejected": -377.9949297971919,
      "loss": 0.4222,
      "loss/base_kto": 3.3775384426116943,
      "metrics/advantage_var": 2635.339599609375,
      "rewards/chosen": 0.5415884862669943,
      "rewards/margins": 0.8321847921260646,
      "rewards/rejected": -0.2905963058590703,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 10.182024002075195,
      "kl": 18.867149353027344,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -39828811.294117644,
      "logits/rejected": -40465155.413333334,
      "logps/chosen": -488.21875,
      "logps/rejected": -362.84151041666667,
      "loss": 0.3905,
      "loss/base_kto": 3.1243832111358643,
      "metrics/advantage_var": 2918.470458984375,
      "rewards/chosen": 1.039443790211397,
      "rewards/margins": 1.222436903411267,
      "rewards/rejected": -0.1829931131998698,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 8.444199562072754,
      "kl": 30.96662712097168,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -39468761.98019802,
      "logits/rejected": -39347196.961424336,
      "logps/chosen": -481.0873556105611,
      "logps/rejected": -369.1377271884273,
      "loss": 0.4128,
      "loss/base_kto": 3.3024284839630127,
      "metrics/advantage_var": 3886.193115234375,
      "rewards/chosen": 0.9114466500360974,
      "rewards/margins": 1.037914289721482,
      "rewards/rejected": -0.1264676396853846,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 7.57260799407959,
      "kl": 19.63751983642578,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -39679900.75770925,
      "logits/rejected": -40483971.6327212,
      "logps/chosen": -465.3614629221733,
      "logps/rejected": -387.42336185308847,
      "loss": 0.3935,
      "loss/base_kto": 3.1482582092285156,
      "metrics/advantage_var": 4115.23583984375,
      "rewards/chosen": 1.1309302689920613,
      "rewards/margins": 1.4825149821207133,
      "rewards/rejected": -0.3515847131286519,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 7.263091564178467,
      "kl": 27.852466583251953,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -38640721.2192846,
      "logits/rejected": -39732961.85871272,
      "logps/chosen": -457.130783437014,
      "logps/rejected": -364.8397272370487,
      "loss": 0.3727,
      "loss/base_kto": 2.9812400341033936,
      "metrics/advantage_var": 4626.76513671875,
      "rewards/chosen": 1.418452664815805,
      "rewards/margins": 1.6763056952256852,
      "rewards/rejected": -0.2578530304098803,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 9.630581855773926,
      "kl": 23.25460433959961,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -39927047.7824,
      "logits/rejected": -40623030.5221374,
      "logps/chosen": -479.1677,
      "logps/rejected": -378.52118320610685,
      "loss": 0.3835,
      "loss/base_kto": 3.0679073333740234,
      "metrics/advantage_var": 4547.76513671875,
      "rewards/chosen": 1.1745533203125,
      "rewards/margins": 1.6070852449606394,
      "rewards/rejected": -0.43253192464813933,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 9.11620044708252,
      "kl": 10.112515449523926,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -41367302.24390244,
      "logits/rejected": -39530563.75338346,
      "logps/chosen": -471.4418699186992,
      "logps/rejected": -374.84586466165416,
      "loss": 0.3788,
      "loss/base_kto": 3.0303921699523926,
      "metrics/advantage_var": 4945.99169921875,
      "rewards/chosen": 0.36794150747903964,
      "rewards/margins": 1.698686062959538,
      "rewards/rejected": -1.3307445554804982,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 7.173008918762207,
      "kl": 11.464367866516113,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -40763076.18691589,
      "logits/rejected": -41477347.91222571,
      "logps/chosen": -462.00783683800626,
      "logps/rejected": -361.60555446708463,
      "loss": 0.3659,
      "loss/base_kto": 2.9271907806396484,
      "metrics/advantage_var": 5027.3837890625,
      "rewards/chosen": 0.8695794696748442,
      "rewards/margins": 1.992290364316302,
      "rewards/rejected": -1.1227108946414577,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 8.966463088989258,
      "kl": 21.945667266845703,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -42089467.266563945,
      "logits/rejected": -42016605.717908084,
      "logps/chosen": -477.8919491525424,
      "logps/rejected": -397.86717511885894,
      "loss": 0.3576,
      "loss/base_kto": 2.860548257827759,
      "metrics/advantage_var": 5913.638671875,
      "rewards/chosen": 1.2006039376986228,
      "rewards/margins": 2.0745871364902233,
      "rewards/rejected": -0.8739831987916006,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.794249534606934,
      "kl": 19.930723190307617,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -41426398.53594771,
      "logits/rejected": -41161758.65868264,
      "logps/chosen": -449.8214358660131,
      "logps/rejected": -375.6300056137725,
      "loss": 0.3565,
      "loss/base_kto": 2.852098226547241,
      "metrics/advantage_var": 6078.57275390625,
      "rewards/chosen": 0.9442857729843239,
      "rewards/margins": 2.0589253293208696,
      "rewards/rejected": -1.1146395563365457,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 7.703754901885986,
      "kl": 38.4437370300293,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -41185273.68875193,
      "logits/rejected": -40902857.22979398,
      "logps/chosen": -444.9513193374422,
      "logps/rejected": -354.3673236925515,
      "loss": 0.3444,
      "loss/base_kto": 2.75480318069458,
      "metrics/advantage_var": 5763.5478515625,
      "rewards/chosen": 1.9258929753707628,
      "rewards/margins": 2.2856331464858703,
      "rewards/rejected": -0.3597401711151075,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 9.472077369689941,
      "kl": 20.64752197265625,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -40008365.798165135,
      "logits/rejected": -42599021.59744409,
      "logps/chosen": -482.6004873853211,
      "logps/rejected": -369.16458666134184,
      "loss": 0.3551,
      "loss/base_kto": 2.8407864570617676,
      "metrics/advantage_var": 6040.44482421875,
      "rewards/chosen": 1.5773067182721712,
      "rewards/margins": 2.260341206840462,
      "rewards/rejected": -0.6830344885682907,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 10.506298065185547,
      "kl": 13.261344909667969,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -42238106.0815047,
      "logits/rejected": -42170939.015576325,
      "logps/chosen": -505.4585619122257,
      "logps/rejected": -399.87091121495325,
      "loss": 0.3522,
      "loss/base_kto": 2.8176186084747314,
      "metrics/advantage_var": 6590.22998046875,
      "rewards/chosen": 1.1125284511841202,
      "rewards/margins": 2.436326514943612,
      "rewards/rejected": -1.3237980637594917,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 10.129120826721191,
      "kl": 21.043310165405273,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -42286177.60258482,
      "logits/rejected": -43014422.85022693,
      "logps/chosen": -467.0582592891761,
      "logps/rejected": -390.2524111195159,
      "loss": 0.3483,
      "loss/base_kto": 2.7863080501556396,
      "metrics/advantage_var": 7763.60009765625,
      "rewards/chosen": 1.3456361736735158,
      "rewards/margins": 2.490973715631723,
      "rewards/rejected": -1.1453375419582073,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 8.79063606262207,
      "kl": 13.356651306152344,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -44623424.59076923,
      "logits/rejected": -43544953.092063494,
      "logps/chosen": -497.1355288461538,
      "logps/rejected": -378.4875248015873,
      "loss": 0.3515,
      "loss/base_kto": 2.812063217163086,
      "metrics/advantage_var": 8117.02294921875,
      "rewards/chosen": 1.206925330528846,
      "rewards/margins": 2.5688414275340543,
      "rewards/rejected": -1.3619160970052084,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 6.636849403381348,
      "kl": 31.743444442749023,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -42751870.06646526,
      "logits/rejected": -43807303.24919094,
      "logps/chosen": -484.7029833836858,
      "logps/rejected": -392.5766332928803,
      "loss": 0.3452,
      "loss/base_kto": 2.7615013122558594,
      "metrics/advantage_var": 7189.68896484375,
      "rewards/chosen": 2.0912890477483006,
      "rewards/margins": 2.4713890881223652,
      "rewards/rejected": -0.38010004037406453,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 8.101085662841797,
      "kl": 31.998029708862305,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44528687.362041466,
      "logits/rejected": -43389398.444104135,
      "logps/chosen": -480.5172448165869,
      "logps/rejected": -392.3862940275651,
      "loss": 0.3334,
      "loss/base_kto": 2.666898488998413,
      "metrics/advantage_var": 7660.3828125,
      "rewards/chosen": 1.7667409601774322,
      "rewards/margins": 2.757692241224044,
      "rewards/rejected": -0.9909512810466118,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 11.130863189697266,
      "kl": 12.283018112182617,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -44233614.58227848,
      "logits/rejected": -44120013.43209877,
      "logps/chosen": -465.97962816455697,
      "logps/rejected": -396.02430555555554,
      "loss": 0.3415,
      "loss/base_kto": 2.7321603298187256,
      "metrics/advantage_var": 8213.9638671875,
      "rewards/chosen": 1.2179762441900712,
      "rewards/margins": 2.7348437994080497,
      "rewards/rejected": -1.5168675552179784,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 10.0189208984375,
      "kl": 22.358829498291016,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -45440364.307692304,
      "logits/rejected": -45431271.02439024,
      "logps/chosen": -463.77383814102564,
      "logps/rejected": -389.2611471036585,
      "loss": 0.329,
      "loss/base_kto": 2.632397174835205,
      "metrics/advantage_var": 8176.68212890625,
      "rewards/chosen": 1.6049115107609675,
      "rewards/margins": 2.86886492902745,
      "rewards/rejected": -1.2639534182664824,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 9.430086135864258,
      "kl": 11.885652542114258,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -45028388.218649514,
      "logits/rejected": -45278267.136778116,
      "logps/chosen": -477.4973874598071,
      "logps/rejected": -379.5108045212766,
      "loss": 0.3388,
      "loss/base_kto": 2.7101292610168457,
      "metrics/advantage_var": 9187.2099609375,
      "rewards/chosen": 1.0469594875715937,
      "rewards/margins": 2.83426569631257,
      "rewards/rejected": -1.7873062087409763,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 9.097721099853516,
      "kl": 13.10331916809082,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -45285132.962025315,
      "logits/rejected": -45354138.86419753,
      "logps/chosen": -491.37193433544303,
      "logps/rejected": -402.21766493055554,
      "loss": 0.324,
      "loss/base_kto": 2.592097043991089,
      "metrics/advantage_var": 9418.1435546875,
      "rewards/chosen": 1.265615921986254,
      "rewards/margins": 3.227010432137681,
      "rewards/rejected": -1.9613945101514274,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.278778076171875,
      "kl": 24.34718894958496,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -45963127.23287671,
      "logits/rejected": -46308840.45977011,
      "logps/chosen": -502.2434717465753,
      "logps/rejected": -399.21125179597703,
      "loss": 0.327,
      "loss/base_kto": 2.616192579269409,
      "metrics/advantage_var": 9953.5107421875,
      "rewards/chosen": 1.5700516374143836,
      "rewards/margins": 3.2479978815774584,
      "rewards/rejected": -1.6779462441630748,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 7.010895252227783,
      "kl": 13.371597290039062,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -43210697.06110283,
      "logits/rejected": -44957669.09688013,
      "logps/chosen": -475.28055141579733,
      "logps/rejected": -381.8127052545156,
      "loss": 0.3057,
      "loss/base_kto": 2.4459750652313232,
      "metrics/advantage_var": 10097.4755859375,
      "rewards/chosen": 1.9163489078800298,
      "rewards/margins": 3.5759380019699822,
      "rewards/rejected": -1.6595890940899527,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 7.648643970489502,
      "kl": 42.0706787109375,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -45007123.15114504,
      "logits/rejected": -44185783.5008,
      "logps/chosen": -448.98234732824426,
      "logps/rejected": -368.3568,
      "loss": 0.3176,
      "loss/base_kto": 2.5408151149749756,
      "metrics/advantage_var": 9049.8310546875,
      "rewards/chosen": 2.615083045085878,
      "rewards/margins": 3.058985632976503,
      "rewards/rejected": -0.443902587890625,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 7.628378391265869,
      "kl": 39.29732894897461,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -45669481.82370821,
      "logits/rejected": -44295375.4340836,
      "logps/chosen": -457.95146276595744,
      "logps/rejected": -362.8493770096463,
      "loss": 0.3251,
      "loss/base_kto": 2.600756883621216,
      "metrics/advantage_var": 9290.9599609375,
      "rewards/chosen": 2.090501222929331,
      "rewards/margins": 2.8993852333669317,
      "rewards/rejected": -0.8088840104376005,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.913888454437256,
      "kl": 23.4282283782959,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -44830290.13493253,
      "logits/rejected": -43444036.90701468,
      "logps/chosen": -460.875,
      "logps/rejected": -383.3435715742251,
      "loss": 0.3012,
      "loss/base_kto": 2.409473180770874,
      "metrics/advantage_var": 11249.1708984375,
      "rewards/chosen": 2.4155743783381745,
      "rewards/margins": 3.6628631429308536,
      "rewards/rejected": -1.2472887645926793,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 6.445258140563965,
      "kl": 29.516122817993164,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -46369467.09172932,
      "logits/rejected": -47186552.71544716,
      "logps/chosen": -422.66809210526316,
      "logps/rejected": -364.4777693089431,
      "loss": 0.3262,
      "loss/base_kto": 2.609204053878784,
      "metrics/advantage_var": 10683.3701171875,
      "rewards/chosen": 1.9281864940671993,
      "rewards/margins": 3.128518763503175,
      "rewards/rejected": -1.2003322694359757,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 9.627001762390137,
      "kl": 23.947446823120117,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -46729614.133748055,
      "logits/rejected": -47353798.12872841,
      "logps/chosen": -487.88039463452566,
      "logps/rejected": -393.484693877551,
      "loss": 0.3246,
      "loss/base_kto": 2.596665382385254,
      "metrics/advantage_var": 11245.66796875,
      "rewards/chosen": 2.035088475347492,
      "rewards/margins": 3.350969458900769,
      "rewards/rejected": -1.315880983553277,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 5.452735900878906,
      "kl": 20.10293197631836,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -45706389.36541353,
      "logits/rejected": -44925420.85203252,
      "logps/chosen": -478.0436090225564,
      "logps/rejected": -420.4835619918699,
      "loss": 0.3199,
      "loss/base_kto": 2.5589563846588135,
      "metrics/advantage_var": 12959.5068359375,
      "rewards/chosen": 2.0530989338580827,
      "rewards/margins": 3.7097156410163654,
      "rewards/rejected": -1.6566167071582825,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 11.135366439819336,
      "kl": 31.90445327758789,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -45910517.47945205,
      "logits/rejected": -46397469.42528736,
      "logps/chosen": -491.58063998287673,
      "logps/rejected": -392.5621408045977,
      "loss": 0.3016,
      "loss/base_kto": 2.4128031730651855,
      "metrics/advantage_var": 11997.140625,
      "rewards/chosen": 2.2658833542915238,
      "rewards/margins": 3.9077030999362794,
      "rewards/rejected": -1.6418197456447559,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 6.5614848136901855,
      "kl": 30.111474990844727,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -45139300.37004405,
      "logits/rejected": -45934034.697829716,
      "logps/chosen": -459.43658223201174,
      "logps/rejected": -384.7371139398998,
      "loss": 0.3333,
      "loss/base_kto": 2.6663873195648193,
      "metrics/advantage_var": 11191.8955078125,
      "rewards/chosen": 2.1196790966868577,
      "rewards/margins": 3.0193248686114194,
      "rewards/rejected": -0.8996457719245617,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 9.026055335998535,
      "kl": 38.45805740356445,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -45909858.461538464,
      "logits/rejected": -46176119.46666667,
      "logps/chosen": -476.51875,
      "logps/rejected": -382.14977678571427,
      "loss": 0.3181,
      "loss/base_kto": 2.544774055480957,
      "metrics/advantage_var": 11464.9521484375,
      "rewards/chosen": 2.5570633638822113,
      "rewards/margins": 3.5192292206220426,
      "rewards/rejected": -0.9621658567398313,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 9.854619979858398,
      "kl": 32.282257080078125,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -48232252.631578945,
      "logits/rejected": -47521709.71428572,
      "logps/chosen": -477.9053248355263,
      "logps/rejected": -400.0725911458333,
      "loss": 0.3232,
      "loss/base_kto": 2.585928440093994,
      "metrics/advantage_var": 12165.7529296875,
      "rewards/chosen": 1.7689299332468134,
      "rewards/margins": 3.367724052945474,
      "rewards/rejected": -1.5987941196986608,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 9.233141899108887,
      "kl": 37.69820785522461,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -46655029.39877301,
      "logits/rejected": -47521289.78343949,
      "logps/chosen": -435.96841449386505,
      "logps/rejected": -400.7941132563694,
      "loss": 0.3245,
      "loss/base_kto": 2.595954418182373,
      "metrics/advantage_var": 10885.6875,
      "rewards/chosen": 2.3360288655099692,
      "rewards/margins": 3.291977616194185,
      "rewards/rejected": -0.9559487506842158,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 6.974427700042725,
      "kl": 27.17978858947754,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47464819.11409396,
      "logits/rejected": -48631063.27272727,
      "logps/chosen": -476.66013003355704,
      "logps/rejected": -390.7624175219941,
      "loss": 0.2932,
      "loss/base_kto": 2.345959424972534,
      "metrics/advantage_var": 13561.8515625,
      "rewards/chosen": 2.032148322803062,
      "rewards/margins": 4.105728262032809,
      "rewards/rejected": -2.073579939229747,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 8.006534576416016,
      "kl": 12.020552635192871,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -47671432.53333333,
      "logits/rejected": -47465428.45984252,
      "logps/chosen": -442.7193798449612,
      "logps/rejected": -396.0220964566929,
      "loss": 0.2713,
      "loss/base_kto": 2.1700031757354736,
      "metrics/advantage_var": 14014.9755859375,
      "rewards/chosen": 2.201185879966085,
      "rewards/margins": 4.629551175856833,
      "rewards/rejected": -2.4283652958907482,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 7.6743483543396,
      "kl": 24.583898544311523,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -49097158.05271318,
      "logits/rejected": -47500434.74645669,
      "logps/chosen": -466.3345930232558,
      "logps/rejected": -376.74753937007875,
      "loss": 0.2633,
      "loss/base_kto": 2.1062209606170654,
      "metrics/advantage_var": 13870.4033203125,
      "rewards/chosen": 2.9003382010053294,
      "rewards/margins": 4.841445715153951,
      "rewards/rejected": -1.941107514148622,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 8.210561752319336,
      "kl": 15.757326126098633,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -46189912.631239936,
      "logits/rejected": -45744180.83156297,
      "logps/chosen": -455.30495169082127,
      "logps/rejected": -347.65063069044004,
      "loss": 0.2604,
      "loss/base_kto": 2.0829906463623047,
      "metrics/advantage_var": 13050.2421875,
      "rewards/chosen": 2.3093481877767714,
      "rewards/margins": 4.568288032119526,
      "rewards/rejected": -2.258939844342754,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 8.713674545288086,
      "kl": 13.271387100219727,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -45598088.13643411,
      "logits/rejected": -45550296.894488186,
      "logps/chosen": -445.2339631782946,
      "logps/rejected": -374.74773622047246,
      "loss": 0.2738,
      "loss/base_kto": 2.1901986598968506,
      "metrics/advantage_var": 13676.1171875,
      "rewards/chosen": 2.0202163578003876,
      "rewards/margins": 4.501684930942612,
      "rewards/rejected": -2.4814685731422244,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.964659690856934,
      "kl": 18.402326583862305,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -45798747.37463127,
      "logits/rejected": -47414003.242524914,
      "logps/chosen": -449.796552359882,
      "logps/rejected": -371.562032807309,
      "loss": 0.2526,
      "loss/base_kto": 2.020669937133789,
      "metrics/advantage_var": 14040.15625,
      "rewards/chosen": 2.557764464071718,
      "rewards/margins": 4.88743947315602,
      "rewards/rejected": -2.329675009084302,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 7.827051162719727,
      "kl": 17.168731689453125,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -47639918.85358255,
      "logits/rejected": -47987541.868338555,
      "logps/chosen": -488.3009151090343,
      "logps/rejected": -416.904927507837,
      "loss": 0.2435,
      "loss/base_kto": 1.9479866027832031,
      "metrics/advantage_var": 14400.1591796875,
      "rewards/chosen": 2.3399764681902258,
      "rewards/margins": 5.139182284094419,
      "rewards/rejected": -2.799205815904193,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 7.2465667724609375,
      "kl": 25.497581481933594,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -48523415.194630876,
      "logits/rejected": -48706350.40935673,
      "logps/chosen": -467.25388003355704,
      "logps/rejected": -402.67114400584796,
      "loss": 0.2546,
      "loss/base_kto": 2.0367116928100586,
      "metrics/advantage_var": 14426.1474609375,
      "rewards/chosen": 2.1353075680316693,
      "rewards/margins": 4.538409260454458,
      "rewards/rejected": -2.403101692422789,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 6.00705099105835,
      "kl": 21.413923263549805,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -47991043.65134707,
      "logits/rejected": -48706687.8027735,
      "logps/chosen": -492.87371236133123,
      "logps/rejected": -417.9573382126348,
      "loss": 0.2415,
      "loss/base_kto": 1.9320722818374634,
      "metrics/advantage_var": 14509.7939453125,
      "rewards/chosen": 2.263974826044968,
      "rewards/margins": 5.147939564464652,
      "rewards/rejected": -2.8839647384196843,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 6.559169769287109,
      "kl": 21.99832534790039,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -47288352.05007825,
      "logits/rejected": -48029686.4149766,
      "logps/chosen": -456.8280516431925,
      "logps/rejected": -395.6653666146646,
      "loss": 0.2501,
      "loss/base_kto": 2.000561475753784,
      "metrics/advantage_var": 12681.5341796875,
      "rewards/chosen": 2.232018603592038,
      "rewards/margins": 4.566518682813957,
      "rewards/rejected": -2.334500079221919,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 7.573378562927246,
      "kl": 24.434301376342773,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -48100737.87928222,
      "logits/rejected": -46305408.95952024,
      "logps/chosen": -460.87418433931487,
      "logps/rejected": -382.82650862068965,
      "loss": 0.2611,
      "loss/base_kto": 2.0884008407592773,
      "metrics/advantage_var": 12721.8154296875,
      "rewards/chosen": 2.5049300077105423,
      "rewards/margins": 4.615244235362342,
      "rewards/rejected": -2.110314227651799,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 9.15713882446289,
      "kl": 9.266432762145996,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -50749891.62633997,
      "logits/rejected": -49394278.073365234,
      "logps/chosen": -472.1126052833078,
      "logps/rejected": -413.897826953748,
      "loss": 0.2668,
      "loss/base_kto": 2.1340107917785645,
      "metrics/advantage_var": 12453.4423828125,
      "rewards/chosen": 1.885936976574464,
      "rewards/margins": 4.484020959391948,
      "rewards/rejected": -2.598083982817484,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 6.648115634918213,
      "kl": 19.018064498901367,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -48953454.3518225,
      "logits/rejected": -49181296.81355932,
      "logps/chosen": -481.27902139461173,
      "logps/rejected": -391.85891756548534,
      "loss": 0.2595,
      "loss/base_kto": 2.075740098953247,
      "metrics/advantage_var": 12773.25390625,
      "rewards/chosen": 2.3078029046156896,
      "rewards/margins": 4.5615917865494335,
      "rewards/rejected": -2.2537888819337444,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.432963371276855,
      "kl": 15.786555290222168,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -50043944.09638554,
      "logits/rejected": -49340396.051948056,
      "logps/chosen": -471.32958396084337,
      "logps/rejected": -410.0531655844156,
      "loss": 0.2574,
      "loss/base_kto": 2.0592541694641113,
      "metrics/advantage_var": 12785.0068359375,
      "rewards/chosen": 2.132786578442677,
      "rewards/margins": 4.651474433556313,
      "rewards/rejected": -2.5186878551136362,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 7.463342666625977,
      "kl": 18.011547088623047,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -50799278.9068323,
      "logits/rejected": -50387240.251572326,
      "logps/chosen": -484.422748447205,
      "logps/rejected": -387.8544123427673,
      "loss": 0.2459,
      "loss/base_kto": 1.9671169519424438,
      "metrics/advantage_var": 13662.9599609375,
      "rewards/chosen": 2.752448230056289,
      "rewards/margins": 4.902260748481996,
      "rewards/rejected": -2.1498125184257075,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 7.461705207824707,
      "kl": 19.686004638671875,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -48579801.8031746,
      "logits/rejected": -48981808.04923077,
      "logps/chosen": -444.07172619047617,
      "logps/rejected": -402.4366105769231,
      "loss": 0.2576,
      "loss/base_kto": 2.0607800483703613,
      "metrics/advantage_var": 15960.1123046875,
      "rewards/chosen": 2.096119326636905,
      "rewards/margins": 4.8182899245936355,
      "rewards/rejected": -2.7221705979567306,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 5.755481719970703,
      "kl": 23.16046714782715,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -50075429.79604262,
      "logits/rejected": -49548141.71428572,
      "logps/chosen": -451.0539383561644,
      "logps/rejected": -415.2268007624398,
      "loss": 0.2499,
      "loss/base_kto": 1.999585747718811,
      "metrics/advantage_var": 14338.078125,
      "rewards/chosen": 2.7706828356877855,
      "rewards/margins": 4.9420303665365015,
      "rewards/rejected": -2.171347530848716,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 7.263795375823975,
      "kl": 20.749311447143555,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -51008243.731629394,
      "logits/rejected": -50280040.905198775,
      "logps/chosen": -457.93106030351436,
      "logps/rejected": -404.38876146788994,
      "loss": 0.2574,
      "loss/base_kto": 2.0591797828674316,
      "metrics/advantage_var": 14394.5234375,
      "rewards/chosen": 2.1054878600489215,
      "rewards/margins": 4.947445068991486,
      "rewards/rejected": -2.841957208942565,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 8.185635566711426,
      "kl": 20.86648941040039,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -50875951.299837925,
      "logits/rejected": -50029618.968325794,
      "logps/chosen": -462.942868719611,
      "logps/rejected": -397.45456259426845,
      "loss": 0.2628,
      "loss/base_kto": 2.102466344833374,
      "metrics/advantage_var": 14819.9130859375,
      "rewards/chosen": 2.3890321505900527,
      "rewards/margins": 4.7555759736481225,
      "rewards/rejected": -2.3665438230580693,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 7.5429463386535645,
      "kl": 18.411264419555664,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -51579456.402515724,
      "logits/rejected": -49579478.65838509,
      "logps/chosen": -472.2454304245283,
      "logps/rejected": -389.61934685559004,
      "loss": 0.2593,
      "loss/base_kto": 2.0742015838623047,
      "metrics/advantage_var": 13896.4990234375,
      "rewards/chosen": 2.2443824624115565,
      "rewards/margins": 4.678256367569747,
      "rewards/rejected": -2.433873905158191,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 7.785284042358398,
      "kl": 21.28897476196289,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -49997334.6835443,
      "logits/rejected": -50460479.209876545,
      "logps/chosen": -468.59083267405066,
      "logps/rejected": -376.14158950617286,
      "loss": 0.2469,
      "loss/base_kto": 1.9749466180801392,
      "metrics/advantage_var": 13357.8466796875,
      "rewards/chosen": 2.6586543215981013,
      "rewards/margins": 4.838992298124433,
      "rewards/rejected": -2.180337976526331,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 12.808547973632812,
      "kl": 12.648378372192383,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -50732009.80804954,
      "logits/rejected": -51210214.15772871,
      "logps/chosen": -462.1233552631579,
      "logps/rejected": -389.2183063880126,
      "loss": 0.2536,
      "loss/base_kto": 2.0289154052734375,
      "metrics/advantage_var": 14579.5732421875,
      "rewards/chosen": 2.117305791045859,
      "rewards/margins": 4.87371756309091,
      "rewards/rejected": -2.7564117720450514,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 9.873778343200684,
      "kl": 26.791122436523438,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -49176047.06771654,
      "logits/rejected": -49040658.65426356,
      "logps/chosen": -465.3425688976378,
      "logps/rejected": -400.659253875969,
      "loss": 0.2663,
      "loss/base_kto": 2.130403518676758,
      "metrics/advantage_var": 14569.2568359375,
      "rewards/chosen": 2.50025817390502,
      "rewards/margins": 4.679842339966067,
      "rewards/rejected": -2.1795841660610464,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 11.098130226135254,
      "kl": 17.67889404296875,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -50631510.12713178,
      "logits/rejected": -49922744.64251968,
      "logps/chosen": -460.33478682170545,
      "logps/rejected": -392.7937992125984,
      "loss": 0.2534,
      "loss/base_kto": 2.027090072631836,
      "metrics/advantage_var": 12999.9736328125,
      "rewards/chosen": 2.4095879133357556,
      "rewards/margins": 4.742966358217645,
      "rewards/rejected": -2.3333784448818897,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 10.580224990844727,
      "kl": 17.041147232055664,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -50606938.11838006,
      "logits/rejected": -50023391.89968652,
      "logps/chosen": -484.9173481308411,
      "logps/rejected": -402.11265673981194,
      "loss": 0.2528,
      "loss/base_kto": 2.0220911502838135,
      "metrics/advantage_var": 13346.0146484375,
      "rewards/chosen": 2.3297069704049846,
      "rewards/margins": 4.816585751140682,
      "rewards/rejected": -2.4868787807356973,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 9.04914379119873,
      "kl": 25.278024673461914,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -50172554.938530736,
      "logits/rejected": -50815196.50244698,
      "logps/chosen": -453.6979947526237,
      "logps/rejected": -378.65576570146817,
      "loss": 0.2627,
      "loss/base_kto": 2.1016016006469727,
      "metrics/advantage_var": 13008.0537109375,
      "rewards/chosen": 2.8642087647582457,
      "rewards/margins": 4.541724488529657,
      "rewards/rejected": -1.6775157237714111,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 8.332275390625,
      "kl": 34.81646728515625,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -49631567.01234568,
      "logits/rejected": -49031887.39240506,
      "logps/chosen": -466.037037037037,
      "logps/rejected": -399.32730913765823,
      "loss": 0.2689,
      "loss/base_kto": 2.1509957313537598,
      "metrics/advantage_var": 13046.8349609375,
      "rewards/chosen": 2.996243323808835,
      "rewards/margins": 4.272110447955888,
      "rewards/rejected": -1.275867124147053,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 9.706599235534668,
      "kl": 21.504140853881836,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -50742327.97427653,
      "logits/rejected": -52272604.20668693,
      "logps/chosen": -450.47432676848877,
      "logps/rejected": -411.2093939969605,
      "loss": 0.2669,
      "loss/base_kto": 2.1349241733551025,
      "metrics/advantage_var": 13198.6015625,
      "rewards/chosen": 2.2765420502788385,
      "rewards/margins": 4.696065805384272,
      "rewards/rejected": -2.419523755105433,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 9.024945259094238,
      "kl": 20.595943450927734,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -50928485.52410576,
      "logits/rejected": -51772176.477237046,
      "logps/chosen": -463.4435264385692,
      "logps/rejected": -410.42481848508635,
      "loss": 0.2558,
      "loss/base_kto": 2.0465526580810547,
      "metrics/advantage_var": 15188.572265625,
      "rewards/chosen": 2.496919955287714,
      "rewards/margins": 4.9222173316000175,
      "rewards/rejected": -2.4252973763123036,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 8.380735397338867,
      "kl": 22.653018951416016,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -50397374.511627905,
      "logits/rejected": -50354895.21889764,
      "logps/chosen": -444.8803294573643,
      "logps/rejected": -395.8338336614173,
      "loss": 0.2498,
      "loss/base_kto": 1.998310923576355,
      "metrics/advantage_var": 13499.3720703125,
      "rewards/chosen": 2.6756109193313953,
      "rewards/margins": 4.87968326185108,
      "rewards/rejected": -2.204072342519685,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 12.051867485046387,
      "kl": 19.444766998291016,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -52307059.86544342,
      "logits/rejected": -52765738.53035144,
      "logps/chosen": -442.72348050458714,
      "logps/rejected": -422.0098841853035,
      "loss": 0.2538,
      "loss/base_kto": 2.0302066802978516,
      "metrics/advantage_var": 14113.2998046875,
      "rewards/chosen": 2.5834022078674503,
      "rewards/margins": 4.8570046782897744,
      "rewards/rejected": -2.273602470422324,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 6.080167293548584,
      "kl": 7.740408420562744,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -52097834.53776435,
      "logits/rejected": -52385685.95469256,
      "logps/chosen": -476.92942787009065,
      "logps/rejected": -413.64504955501616,
      "loss": 0.2637,
      "loss/base_kto": 2.1094841957092285,
      "metrics/advantage_var": 14161.9091796875,
      "rewards/chosen": 1.8997557487372545,
      "rewards/margins": 4.792029290530337,
      "rewards/rejected": -2.8922735417930827,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 7.666107654571533,
      "kl": 29.72540283203125,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -50770696.827586204,
      "logits/rejected": -52469633.04404568,
      "logps/chosen": -465.53841829085457,
      "logps/rejected": -401.9165477161501,
      "loss": 0.246,
      "loss/base_kto": 1.9683994054794312,
      "metrics/advantage_var": 13659.8037109375,
      "rewards/chosen": 2.673970509862256,
      "rewards/margins": 4.902071932170576,
      "rewards/rejected": -2.2281014223083195,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 8.589899063110352,
      "kl": 30.167007446289062,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -51137649.777777776,
      "logits/rejected": -51158989.587692305,
      "logps/chosen": -476.984375,
      "logps/rejected": -403.0909375,
      "loss": 0.2636,
      "loss/base_kto": 2.1090667247772217,
      "metrics/advantage_var": 12947.015625,
      "rewards/chosen": 2.508792937748016,
      "rewards/margins": 4.470145289010035,
      "rewards/rejected": -1.9613523512620192,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 9.947898864746094,
      "kl": 24.149383544921875,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -50633531.65781711,
      "logits/rejected": -50599163.7475083,
      "logps/chosen": -453.01069321533924,
      "logps/rejected": -398.06753529900334,
      "loss": 0.2631,
      "loss/base_kto": 2.1051042079925537,
      "metrics/advantage_var": 14002.7099609375,
      "rewards/chosen": 2.5346550055309733,
      "rewards/margins": 4.710280727408569,
      "rewards/rejected": -2.1756257218775956,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.734704494476318,
      "kl": 35.1544303894043,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -51763093.56850394,
      "logits/rejected": -51378245.85426357,
      "logps/chosen": -463.19566929133856,
      "logps/rejected": -394.7056201550388,
      "loss": 0.24,
      "loss/base_kto": 1.9199455976486206,
      "metrics/advantage_var": 13500.6142578125,
      "rewards/chosen": 2.862628414124016,
      "rewards/margins": 4.883309963905992,
      "rewards/rejected": -2.0206815497819766,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 6.12605619430542,
      "kl": 18.320152282714844,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -49078500.644338116,
      "logits/rejected": -50882374.95865238,
      "logps/chosen": -414.313446969697,
      "logps/rejected": -398.36633805513014,
      "loss": 0.2708,
      "loss/base_kto": 2.166231155395508,
      "metrics/advantage_var": 12967.21875,
      "rewards/chosen": 1.996106015438098,
      "rewards/margins": 4.480647016108083,
      "rewards/rejected": -2.4845410006699846,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 5.955198287963867,
      "kl": 15.756006240844727,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -50458820.608,
      "logits/rejected": -51340717.92366412,
      "logps/chosen": -462.9027,
      "logps/rejected": -390.4223282442748,
      "loss": 0.2493,
      "loss/base_kto": 1.9940131902694702,
      "metrics/advantage_var": 14228.1376953125,
      "rewards/chosen": 2.121536328125,
      "rewards/margins": 5.070816392831584,
      "rewards/rejected": -2.949280064706584,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 7.135191440582275,
      "kl": 24.769649505615234,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -51178060.88178914,
      "logits/rejected": -52667457.76146789,
      "logps/chosen": -469.5570087859425,
      "logps/rejected": -387.4057482798165,
      "loss": 0.2443,
      "loss/base_kto": 1.9541826248168945,
      "metrics/advantage_var": 13435.5263671875,
      "rewards/chosen": 2.4769127208965656,
      "rewards/margins": 4.854215258167208,
      "rewards/rejected": -2.377302537270642,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 7.013718605041504,
      "kl": 30.34478187561035,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -51477564.23529412,
      "logits/rejected": -52709886.46246246,
      "logps/chosen": -484.40109272875816,
      "logps/rejected": -392.9382272897898,
      "loss": 0.2,
      "loss/base_kto": 1.5998648405075073,
      "metrics/advantage_var": 14826.869140625,
      "rewards/chosen": 3.1217575571895426,
      "rewards/margins": 5.812387762590256,
      "rewards/rejected": -2.6906302054007134,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 5.8692803382873535,
      "kl": 12.55923080444336,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -51684074.43797856,
      "logits/rejected": -52261406.21371611,
      "logps/chosen": -428.73631316998467,
      "logps/rejected": -383.90814393939394,
      "loss": 0.2111,
      "loss/base_kto": 1.6889790296554565,
      "metrics/advantage_var": 14125.8115234375,
      "rewards/chosen": 2.474229143735643,
      "rewards/margins": 5.531732632571369,
      "rewards/rejected": -3.0575034888357258,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 5.495901584625244,
      "kl": 18.747608184814453,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -51583710.500807755,
      "logits/rejected": -52984393.58547655,
      "logps/chosen": -455.1819466882068,
      "logps/rejected": -395.01366301059,
      "loss": 0.2167,
      "loss/base_kto": 1.7334028482437134,
      "metrics/advantage_var": 14336.228515625,
      "rewards/chosen": 2.454308036841175,
      "rewards/margins": 5.4658964061443145,
      "rewards/rejected": -3.0115883693031393,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 9.811450958251953,
      "kl": 15.564459800720215,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -51759881.68229955,
      "logits/rejected": -52201898.80452342,
      "logps/chosen": -459.4629349470499,
      "logps/rejected": -398.67245557350566,
      "loss": 0.2064,
      "loss/base_kto": 1.6514919996261597,
      "metrics/advantage_var": 14630.4482421875,
      "rewards/chosen": 3.066073649950359,
      "rewards/margins": 5.556259906765787,
      "rewards/rejected": -2.490186256815428,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 5.7088398933410645,
      "kl": 17.640766143798828,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -50637759.902290076,
      "logits/rejected": -52251328.512,
      "logps/chosen": -458.0880248091603,
      "logps/rejected": -429.6307,
      "loss": 0.2117,
      "loss/base_kto": 1.6933727264404297,
      "metrics/advantage_var": 15645.4248046875,
      "rewards/chosen": 2.766318098461355,
      "rewards/margins": 5.662018098461355,
      "rewards/rejected": -2.8957,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 9.062532424926758,
      "kl": 18.433141708374023,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -51133872.56528418,
      "logits/rejected": -49807802.81081081,
      "logps/chosen": -442.15101766513055,
      "logps/rejected": -396.4219495230525,
      "loss": 0.2287,
      "loss/base_kto": 1.8292301893234253,
      "metrics/advantage_var": 13636.6064453125,
      "rewards/chosen": 2.8211182390673004,
      "rewards/margins": 5.241429419388247,
      "rewards/rejected": -2.420311180320946,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 8.166321754455566,
      "kl": 22.650665283203125,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -52599490.43037975,
      "logits/rejected": -51828483.16049383,
      "logps/chosen": -466.92306170886076,
      "logps/rejected": -387.64144483024694,
      "loss": 0.2115,
      "loss/base_kto": 1.6923620700836182,
      "metrics/advantage_var": 14835.9453125,
      "rewards/chosen": 2.4578896051720727,
      "rewards/margins": 5.504484295565591,
      "rewards/rejected": -3.0465946903935186,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 8.062451362609863,
      "kl": 12.976425170898438,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -52049936.07535322,
      "logits/rejected": -53151976.510108866,
      "logps/chosen": -446.8129905808477,
      "logps/rejected": -401.06497861586314,
      "loss": 0.2076,
      "loss/base_kto": 1.660894751548767,
      "metrics/advantage_var": 13747.203125,
      "rewards/chosen": 2.508091517857143,
      "rewards/margins": 5.53510038133887,
      "rewards/rejected": -3.0270088634817265,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.792516231536865,
      "kl": 20.18115234375,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -53751346.09618574,
      "logits/rejected": -53658386.52880354,
      "logps/chosen": -464.70885157545604,
      "logps/rejected": -407.9797359675037,
      "loss": 0.2025,
      "loss/base_kto": 1.6203575134277344,
      "metrics/advantage_var": 15640.3388671875,
      "rewards/chosen": 2.468496547211857,
      "rewards/margins": 5.659715995374707,
      "rewards/rejected": -3.1912194481628506,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 6.621832847595215,
      "kl": 16.571104049682617,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -54200557.00890208,
      "logits/rejected": -53209642.24422442,
      "logps/chosen": -462.1706695103858,
      "logps/rejected": -402.0129950495049,
      "loss": 0.2237,
      "loss/base_kto": 1.7895901203155518,
      "metrics/advantage_var": 14439.8701171875,
      "rewards/chosen": 2.625101061178598,
      "rewards/margins": 5.253301478876618,
      "rewards/rejected": -2.62820041769802,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 8.105810165405273,
      "kl": 21.278722763061523,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -50443359.44391785,
      "logits/rejected": -49563052.90880989,
      "logps/chosen": -468.43725315955766,
      "logps/rejected": -405.7663736476043,
      "loss": 0.2185,
      "loss/base_kto": 1.7482216358184814,
      "metrics/advantage_var": 15048.6787109375,
      "rewards/chosen": 2.739806068263724,
      "rewards/margins": 5.470672537303523,
      "rewards/rejected": -2.7308664690397992,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.075442314147949,
      "kl": 19.994430541992188,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -52378354.526315786,
      "logits/rejected": -51290253.13323124,
      "logps/chosen": -460.5764553429027,
      "logps/rejected": -422.2306661562021,
      "loss": 0.214,
      "loss/base_kto": 1.7116119861602783,
      "metrics/advantage_var": 16243.4814453125,
      "rewards/chosen": 2.764156651839115,
      "rewards/margins": 5.658843060122997,
      "rewards/rejected": -2.894686408283882,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 7.0332512855529785,
      "kl": 19.325130462646484,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -51644251.14551084,
      "logits/rejected": -52120175.44479495,
      "logps/chosen": -464.09950657894734,
      "logps/rejected": -402.872535488959,
      "loss": 0.2107,
      "loss/base_kto": 1.6855430603027344,
      "metrics/advantage_var": 13973.416015625,
      "rewards/chosen": 2.7761113311484134,
      "rewards/margins": 5.570257077710421,
      "rewards/rejected": -2.794145746562007,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 7.986169815063477,
      "kl": 16.947446823120117,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -54452428.475435816,
      "logits/rejected": -54813858.5146379,
      "logps/chosen": -476.34974247226626,
      "logps/rejected": -396.2220242681048,
      "loss": 0.1982,
      "loss/base_kto": 1.5859140157699585,
      "metrics/advantage_var": 14368.3193359375,
      "rewards/chosen": 2.7504950521865097,
      "rewards/margins": 5.752678587252476,
      "rewards/rejected": -3.0021835350659667,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 7.128055572509766,
      "kl": 20.3778133392334,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -50613619.450980395,
      "logits/rejected": -53207653.17365269,
      "logps/chosen": -435.92953431372547,
      "logps/rejected": -404.85291916167665,
      "loss": 0.2143,
      "loss/base_kto": 1.7144848108291626,
      "metrics/advantage_var": 14200.7333984375,
      "rewards/chosen": 2.464855717677696,
      "rewards/margins": 5.405907452099477,
      "rewards/rejected": -2.9410517344217815,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 8.116265296936035,
      "kl": 16.64581298828125,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -52379571.53246753,
      "logits/rejected": -51955798.361445785,
      "logps/chosen": -464.8143262987013,
      "logps/rejected": -407.6978539156627,
      "loss": 0.2168,
      "loss/base_kto": 1.734110713005066,
      "metrics/advantage_var": 14600.6748046875,
      "rewards/chosen": 2.50759253563819,
      "rewards/margins": 5.451606935522885,
      "rewards/rejected": -2.944014399884695,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 9.05835247039795,
      "kl": 18.716712951660156,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -52313060.8424337,
      "logits/rejected": -52677460.53208138,
      "logps/chosen": -454.5990639625585,
      "logps/rejected": -390.97911776212834,
      "loss": 0.2116,
      "loss/base_kto": 1.6926889419555664,
      "metrics/advantage_var": 14277.234375,
      "rewards/chosen": 2.606424554833756,
      "rewards/margins": 5.508694756992892,
      "rewards/rejected": -2.9022702021591353,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 5.412971019744873,
      "kl": 21.103687286376953,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -52296739.194762684,
      "logits/rejected": -52193589.18983558,
      "logps/chosen": -446.3010433715221,
      "logps/rejected": -400.5539985052317,
      "loss": 0.2085,
      "loss/base_kto": 1.6677101850509644,
      "metrics/advantage_var": 15533.890625,
      "rewards/chosen": 2.562060866535393,
      "rewards/margins": 5.68401731973046,
      "rewards/rejected": -3.121956453195067,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.837034702301025,
      "kl": 12.999594688415527,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -52166433.04510109,
      "logits/rejected": -52585486.467817895,
      "logps/chosen": -468.9892107309487,
      "logps/rejected": -407.98569956828885,
      "loss": 0.2153,
      "loss/base_kto": 1.7224851846694946,
      "metrics/advantage_var": 14251.125,
      "rewards/chosen": 2.384062439249611,
      "rewards/margins": 5.407675475209187,
      "rewards/rejected": -3.0236130359595763,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 7.596892356872559,
      "kl": 17.91208839416504,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -52285226.17944535,
      "logits/rejected": -52072183.940029986,
      "logps/chosen": -451.25785073409463,
      "logps/rejected": -392.07093328335833,
      "loss": 0.2083,
      "loss/base_kto": 1.6665111780166626,
      "metrics/advantage_var": 14959.923828125,
      "rewards/chosen": 2.6295397013279977,
      "rewards/margins": 5.66366997505832,
      "rewards/rejected": -3.034130273730322,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 8.364079475402832,
      "kl": 9.80460262298584,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -52477008.22119816,
      "logits/rejected": -53476327.58028617,
      "logps/chosen": -435.32574884792626,
      "logps/rejected": -392.80703497615264,
      "loss": 0.226,
      "loss/base_kto": 1.8082932233810425,
      "metrics/advantage_var": 14568.2607421875,
      "rewards/chosen": 2.3125780049923197,
      "rewards/margins": 5.337056110740531,
      "rewards/rejected": -3.0244781057482113,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 11.61428165435791,
      "kl": 17.37249755859375,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -53554920.149068326,
      "logits/rejected": -52665005.88679245,
      "logps/chosen": -458.90052406832297,
      "logps/rejected": -417.7639544025157,
      "loss": 0.2136,
      "loss/base_kto": 1.7086185216903687,
      "metrics/advantage_var": 13907.859375,
      "rewards/chosen": 2.4842216539086763,
      "rewards/margins": 5.431352540922237,
      "rewards/rejected": -2.9471308870135613,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.054996490478516,
      "kl": 9.886092185974121,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -52787035.56204379,
      "logits/rejected": -52475398.02352941,
      "logps/chosen": -484.2533759124088,
      "logps/rejected": -387.08613445378154,
      "loss": 0.2097,
      "loss/base_kto": 1.6778475046157837,
      "metrics/advantage_var": 13511.6064453125,
      "rewards/chosen": 2.6910379006044707,
      "rewards/margins": 5.576327709887559,
      "rewards/rejected": -2.8852898092830883,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 7.358473777770996,
      "kl": 18.090463638305664,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -50325776.0,
      "logits/rejected": -50889488.0,
      "logps/chosen": -437.87919921875,
      "logps/rejected": -388.3118896484375,
      "loss": 0.208,
      "loss/base_kto": 1.6639856100082397,
      "metrics/advantage_var": 13564.1123046875,
      "rewards/chosen": 2.6645410537719725,
      "rewards/margins": 5.443995475769043,
      "rewards/rejected": -2.7794544219970705,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 9.689790725708008,
      "kl": 18.853412628173828,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -53538429.46727549,
      "logits/rejected": -54475780.93097913,
      "logps/chosen": -470.3403729071537,
      "logps/rejected": -404.3622592295345,
      "loss": 0.2138,
      "loss/base_kto": 1.7103630304336548,
      "metrics/advantage_var": 14769.6220703125,
      "rewards/chosen": 2.7466041413979263,
      "rewards/margins": 5.500908540474972,
      "rewards/rejected": -2.7543043990770464,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 7.575942516326904,
      "kl": 18.5459041595459,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -52107129.6,
      "logits/rejected": -52743897.6,
      "logps/chosen": -443.063037109375,
      "logps/rejected": -400.88740234375,
      "loss": 0.2097,
      "loss/base_kto": 1.677259087562561,
      "metrics/advantage_var": 13134.6123046875,
      "rewards/chosen": 2.7101999282836915,
      "rewards/margins": 5.353364753723145,
      "rewards/rejected": -2.6431648254394533,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 7.75359582901001,
      "kl": 19.799182891845703,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -49708657.50558214,
      "logits/rejected": -51144722.81776416,
      "logps/chosen": -456.9265350877193,
      "logps/rejected": -399.776440467075,
      "loss": 0.2149,
      "loss/base_kto": 1.7193249464035034,
      "metrics/advantage_var": 13961.0068359375,
      "rewards/chosen": 2.6514213111792264,
      "rewards/margins": 5.558279980302504,
      "rewards/rejected": -2.9068586691232774,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 5.188368320465088,
      "kl": 15.995924949645996,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -52745606.757164404,
      "logits/rejected": -52081904.64829822,
      "logps/chosen": -503.8714177978884,
      "logps/rejected": -408.71950972447326,
      "loss": 0.2177,
      "loss/base_kto": 1.7417219877243042,
      "metrics/advantage_var": 14489.0693359375,
      "rewards/chosen": 2.7078634638951735,
      "rewards/margins": 5.375946963945822,
      "rewards/rejected": -2.668083500050648,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 6.56077241897583,
      "kl": 16.229629516601562,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -51885137.2192846,
      "logits/rejected": -52316156.78492936,
      "logps/chosen": -480.2950038880249,
      "logps/rejected": -415.40894819466246,
      "loss": 0.2057,
      "loss/base_kto": 1.6456931829452515,
      "metrics/advantage_var": 14930.6376953125,
      "rewards/chosen": 2.668261490936042,
      "rewards/margins": 5.4684896876834905,
      "rewards/rejected": -2.800228196747449,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 7.278511047363281,
      "kl": 17.13551139831543,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -53035779.32467532,
      "logits/rejected": -52839140.240963854,
      "logps/chosen": -471.993455762987,
      "logps/rejected": -399.45218373493975,
      "loss": 0.2193,
      "loss/base_kto": 1.7541418075561523,
      "metrics/advantage_var": 14101.2490234375,
      "rewards/chosen": 2.4638329047661323,
      "rewards/margins": 5.324497915884826,
      "rewards/rejected": -2.8606650111186935,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 7.59853458404541,
      "kl": 16.550039291381836,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -51503687.3114241,
      "logits/rejected": -50366601.385335416,
      "logps/chosen": -455.4739827856025,
      "logps/rejected": -414.71514235569424,
      "loss": 0.2146,
      "loss/base_kto": 1.7168140411376953,
      "metrics/advantage_var": 15569.7490234375,
      "rewards/chosen": 2.7565891224080596,
      "rewards/margins": 5.4543376964135195,
      "rewards/rejected": -2.6977485740054603,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 6.622338771820068,
      "kl": 15.1036376953125,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -52360348.09756097,
      "logits/rejected": -53833321.025641024,
      "logps/chosen": -446.0986566310976,
      "logps/rejected": -419.4545272435897,
      "loss": 0.2128,
      "loss/base_kto": 1.7021082639694214,
      "metrics/advantage_var": 13941.78125,
      "rewards/chosen": 2.581070504537443,
      "rewards/margins": 5.448736504512403,
      "rewards/rejected": -2.86766599997496,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 7.87120246887207,
      "kl": 16.005062103271484,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -54459303.96380091,
      "logits/rejected": -53815627.92868719,
      "logps/chosen": -476.5891308446455,
      "logps/rejected": -403.3553737844408,
      "loss": 0.2189,
      "loss/base_kto": 1.7509472370147705,
      "metrics/advantage_var": 14989.7529296875,
      "rewards/chosen": 2.553857863758484,
      "rewards/margins": 5.529682006536037,
      "rewards/rejected": -2.9758241427775527,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 8.618688583374023,
      "kl": 20.370731353759766,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -51821293.021416806,
      "logits/rejected": -51338192.832095094,
      "logps/chosen": -461.3043657331137,
      "logps/rejected": -396.511794205052,
      "loss": 0.2111,
      "loss/base_kto": 1.6885474920272827,
      "metrics/advantage_var": 15565.265625,
      "rewards/chosen": 2.3771085786269563,
      "rewards/margins": 5.516415291804891,
      "rewards/rejected": -3.1393067131779344,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 10.641010284423828,
      "kl": 15.09222412109375,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -51623777.486068115,
      "logits/rejected": -51611396.03785489,
      "logps/chosen": -458.0194465944272,
      "logps/rejected": -377.268089511041,
      "loss": 0.227,
      "loss/base_kto": 1.8159443140029907,
      "metrics/advantage_var": 14810.2236328125,
      "rewards/chosen": 2.501199917158475,
      "rewards/margins": 5.438771927090455,
      "rewards/rejected": -2.9375720099319795,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 7.686944484710693,
      "kl": 10.616701126098633,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -52331420.31858407,
      "logits/rejected": -52707416.45182724,
      "logps/chosen": -463.57586651917404,
      "logps/rejected": -415.9006955980066,
      "loss": 0.21,
      "loss/base_kto": 1.6796455383300781,
      "metrics/advantage_var": 14238.4287109375,
      "rewards/chosen": 2.7552421817385695,
      "rewards/margins": 5.631798071286431,
      "rewards/rejected": -2.8765558895478613,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 8.286816596984863,
      "kl": 21.288183212280273,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -50794508.94154818,
      "logits/rejected": -50980807.02318393,
      "logps/chosen": -460.2579976303318,
      "logps/rejected": -390.54897604327664,
      "loss": 0.2197,
      "loss/base_kto": 1.7577959299087524,
      "metrics/advantage_var": 13955.8486328125,
      "rewards/chosen": 2.697458854783768,
      "rewards/margins": 5.36005537900614,
      "rewards/rejected": -2.6625965242223724,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 8.0601806640625,
      "kl": 20.204782485961914,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -53413062.298412696,
      "logits/rejected": -53059010.56,
      "logps/chosen": -463.35287698412696,
      "logps/rejected": -399.71653846153845,
      "loss": 0.2183,
      "loss/base_kto": 1.746720314025879,
      "metrics/advantage_var": 15232.736328125,
      "rewards/chosen": 2.51821773468502,
      "rewards/margins": 5.441813362689828,
      "rewards/rejected": -2.923595628004808,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.989338484496138e+17,
      "train_loss": 0.27494916788242846,
      "train_runtime": 11062.9074,
      "train_samples_per_second": 13.398,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.989338484496138e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}