{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 24.32085418701172,
      "kl": 6.620049953460693,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -35861141.06329114,
      "logits/rejected": -37501882.469135806,
      "logps/chosen": -501.94417523734177,
      "logps/rejected": -375.9259741512346,
      "loss": 0.5316,
      "loss/base_kto": 3.8595731258392334,
      "loss/total_with_kl": 4.253028869628906,
      "metrics/advantage_var": 236.3098907470703,
      "regularization/advantage_var": 236.3098907470703,
      "regularization/kl": 0.39345595240592957,
      "rewards/chosen": 0.02552956569043896,
      "rewards/margins": 0.14411608236509293,
      "rewards/rejected": -0.11858651667465399,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 12.490849494934082,
      "kl": 26.332868576049805,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -35957793.13405239,
      "logits/rejected": -38035605.29952456,
      "logps/chosen": -489.97972842835134,
      "logps/rejected": -359.01057349445324,
      "loss": 0.5181,
      "loss/base_kto": 3.813135862350464,
      "loss/total_with_kl": 4.144712924957275,
      "metrics/advantage_var": 199.14524841308594,
      "regularization/advantage_var": 199.14524841308594,
      "regularization/kl": 0.3315768241882324,
      "rewards/chosen": 0.4595373591583205,
      "rewards/margins": 0.16364866697051095,
      "rewards/rejected": 0.29588869218780955,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 16.590961456298828,
      "kl": 3.329580068588257,
      "learning_rate": 5.9e-07,
      "logits/chosen": -35121857.64102564,
      "logits/rejected": -36009749.853658535,
      "logps/chosen": -470.4701522435897,
      "logps/rejected": -365.0965129573171,
      "loss": 0.5163,
      "loss/base_kto": 3.8311352729797363,
      "loss/total_with_kl": 4.130624294281006,
      "metrics/advantage_var": 179.87359619140625,
      "regularization/advantage_var": 179.87359619140625,
      "regularization/kl": 0.2994895279407501,
      "rewards/chosen": 0.015784152043171417,
      "rewards/margins": 0.15219200421602297,
      "rewards/rejected": -0.13640785217285156,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 15.667641639709473,
      "kl": 12.654009819030762,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37766268.17007874,
      "logits/rejected": -38015642.79069767,
      "logps/chosen": -504.93292322834645,
      "logps/rejected": -384.18226744186046,
      "loss": 0.5236,
      "loss/base_kto": 3.8299171924591064,
      "loss/total_with_kl": 4.188797473907471,
      "metrics/advantage_var": 215.5437774658203,
      "regularization/advantage_var": 215.5437774658203,
      "regularization/kl": 0.3588803708553314,
      "rewards/chosen": 0.2038393035648376,
      "rewards/margins": 0.16062911575634983,
      "rewards/rejected": 0.04321018780848777,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 14.052574157714844,
      "kl": 4.885131359100342,
      "learning_rate": 9.9e-07,
      "logits/chosen": -36457730.74425727,
      "logits/rejected": -37892599.01754386,
      "logps/chosen": -491.5156489280245,
      "logps/rejected": -374.7734250398724,
      "loss": 0.5269,
      "loss/base_kto": 3.8030097484588623,
      "loss/total_with_kl": 4.215572357177734,
      "metrics/advantage_var": 247.7853240966797,
      "regularization/advantage_var": 247.7853240966797,
      "regularization/kl": 0.41256260871887207,
      "rewards/chosen": -0.02621486833230642,
      "rewards/margins": 0.2226151938281551,
      "rewards/rejected": -0.24883006216046152,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 16.11071014404297,
      "kl": 18.43494415283203,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -37538017.123664126,
      "logits/rejected": -37439799.296,
      "logps/chosen": -490.54790076335877,
      "logps/rejected": -360.4264,
      "loss": 0.515,
      "loss/base_kto": 3.7947540283203125,
      "loss/total_with_kl": 4.120279312133789,
      "metrics/advantage_var": 195.51060485839844,
      "regularization/advantage_var": 195.51060485839844,
      "regularization/kl": 0.3255251348018646,
      "rewards/chosen": 0.34407502385496186,
      "rewards/margins": 0.18177883244871185,
      "rewards/rejected": 0.16229619140625,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 13.922213554382324,
      "kl": 11.039660453796387,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -35708098.74846626,
      "logits/rejected": -38608390.522292994,
      "logps/chosen": -484.8937404141104,
      "logps/rejected": -383.91650079617835,
      "loss": 0.5165,
      "loss/base_kto": 3.730560302734375,
      "loss/total_with_kl": 4.132201671600342,
      "metrics/advantage_var": 241.22573852539062,
      "regularization/advantage_var": 241.22573852539062,
      "regularization/kl": 0.4016408622264862,
      "rewards/chosen": 0.25796843452687646,
      "rewards/margins": 0.23204063272649486,
      "rewards/rejected": 0.025927801800381605,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 15.479750633239746,
      "kl": 24.18060874938965,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -37649381.90104947,
      "logits/rejected": -37157214.799347475,
      "logps/chosen": -493.6107102698651,
      "logps/rejected": -372.21951468189235,
      "loss": 0.5144,
      "loss/base_kto": 3.7350997924804688,
      "loss/total_with_kl": 4.114933967590332,
      "metrics/advantage_var": 228.1287078857422,
      "regularization/advantage_var": 228.1287078857422,
      "regularization/kl": 0.3798343241214752,
      "rewards/chosen": 0.4730671620976621,
      "rewards/margins": 0.2011343828135318,
      "rewards/rejected": 0.2719327792841303,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 18.767160415649414,
      "kl": 36.3804817199707,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -36200173.30551416,
      "logits/rejected": -37414057.82594417,
      "logps/chosen": -463.64535208643815,
      "logps/rejected": -356.4462746305419,
      "loss": 0.5124,
      "loss/base_kto": 3.7325439453125,
      "loss/total_with_kl": 4.099050521850586,
      "metrics/advantage_var": 220.1240692138672,
      "regularization/advantage_var": 220.1240692138672,
      "regularization/kl": 0.3665066361427307,
      "rewards/chosen": 0.6073920396330105,
      "rewards/margins": 0.22740391893854467,
      "rewards/rejected": 0.3799881206944658,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 11.879533767700195,
      "kl": 32.49602127075195,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -37876213.36498516,
      "logits/rejected": -39788665.66336634,
      "logps/chosen": -486.05471068249255,
      "logps/rejected": -380.03282281353137,
      "loss": 0.525,
      "loss/base_kto": 3.7456014156341553,
      "loss/total_with_kl": 4.20007848739624,
      "metrics/advantage_var": 272.9594421386719,
      "regularization/advantage_var": 272.9594421386719,
      "regularization/kl": 0.4544773995876312,
      "rewards/chosen": 0.5189340744244946,
      "rewards/margins": 0.19561730518653564,
      "rewards/rejected": 0.32331676923795893,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 16.6059513092041,
      "kl": 30.7705078125,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -37962922.9544688,
      "logits/rejected": -38796179.19068413,
      "logps/chosen": -475.97112141652616,
      "logps/rejected": -378.00206968704515,
      "loss": 0.5136,
      "loss/base_kto": 3.6783127784729004,
      "loss/total_with_kl": 4.108422756195068,
      "metrics/advantage_var": 258.3245544433594,
      "regularization/advantage_var": 258.3245544433594,
      "regularization/kl": 0.430110365152359,
      "rewards/chosen": 0.48481614111245785,
      "rewards/margins": 0.2808650342705501,
      "rewards/rejected": 0.20395110684190776,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 13.229418754577637,
      "kl": 13.234227180480957,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -36810953.22979398,
      "logits/rejected": -37116424.6779661,
      "logps/chosen": -465.34152139461173,
      "logps/rejected": -377.6193663328197,
      "loss": 0.5197,
      "loss/base_kto": 3.7048676013946533,
      "loss/total_with_kl": 4.157932758331299,
      "metrics/advantage_var": 272.11114501953125,
      "regularization/advantage_var": 272.11114501953125,
      "regularization/kl": 0.45306500792503357,
      "rewards/chosen": 0.14031385128547197,
      "rewards/margins": 0.3018961750151884,
      "rewards/rejected": -0.1615823237297164,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 20.17181968688965,
      "kl": 11.351752281188965,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -36860006.080996886,
      "logits/rejected": -36849118.29467085,
      "logps/chosen": -478.508761682243,
      "logps/rejected": -378.24250587774293,
      "loss": 0.5292,
      "loss/base_kto": 3.760735034942627,
      "loss/total_with_kl": 4.233994960784912,
      "metrics/advantage_var": 284.2402038574219,
      "regularization/advantage_var": 284.2402038574219,
      "regularization/kl": 0.473259836435318,
      "rewards/chosen": 0.1905547168767341,
      "rewards/margins": 0.20015269176158046,
      "rewards/rejected": -0.009597974884846367,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 14.432672500610352,
      "kl": 12.176385879516602,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -36651500.97576737,
      "logits/rejected": -39384570.577912256,
      "logps/chosen": -504.3191639741519,
      "logps/rejected": -364.26059001512857,
      "loss": 0.5152,
      "loss/base_kto": 3.6998088359832764,
      "loss/total_with_kl": 4.121967315673828,
      "metrics/advantage_var": 253.5484161376953,
      "regularization/advantage_var": 253.5484161376953,
      "regularization/kl": 0.4221580922603607,
      "rewards/chosen": 0.2182309546647819,
      "rewards/margins": 0.27326805060466053,
      "rewards/rejected": -0.055037095939878614,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 14.449652671813965,
      "kl": 8.657655715942383,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -36689699.25748503,
      "logits/rejected": -38683975.94771242,
      "logps/chosen": -462.6022642215569,
      "logps/rejected": -374.1436376633987,
      "loss": 0.5214,
      "loss/base_kto": 3.7633156776428223,
      "loss/total_with_kl": 4.171494483947754,
      "metrics/advantage_var": 245.15245056152344,
      "regularization/advantage_var": 245.15245056152344,
      "regularization/kl": 0.40817880630493164,
      "rewards/chosen": 0.12144154988363118,
      "rewards/margins": 0.23151773113331203,
      "rewards/rejected": -0.11007618124968087,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 13.811630249023438,
      "kl": 21.14801597595215,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -38208566.59127625,
      "logits/rejected": -38600158.692889564,
      "logps/chosen": -520.9845516962844,
      "logps/rejected": -393.59805219364597,
      "loss": 0.5152,
      "loss/base_kto": 3.6663787364959717,
      "loss/total_with_kl": 4.121258735656738,
      "metrics/advantage_var": 273.2014465332031,
      "regularization/advantage_var": 273.2014465332031,
      "regularization/kl": 0.45488038659095764,
      "rewards/chosen": 0.4207693170846375,
      "rewards/margins": 0.2904753714027871,
      "rewards/rejected": 0.1302939456818504,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 17.25345802307129,
      "kl": 4.230582237243652,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -34972668.7284345,
      "logits/rejected": -36581563.889908254,
      "logps/chosen": -471.2091154153355,
      "logps/rejected": -365.0969036697248,
      "loss": 0.5219,
      "loss/base_kto": 3.714958906173706,
      "loss/total_with_kl": 4.175113201141357,
      "metrics/advantage_var": 276.3685607910156,
      "regularization/advantage_var": 276.3685607910156,
      "regularization/kl": 0.4601536691188812,
      "rewards/chosen": 0.03428346432816868,
      "rewards/margins": 0.2866421838068575,
      "rewards/rejected": -0.2523587194786888,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 13.907388687133789,
      "kl": 2.6898460388183594,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -34957206.126009695,
      "logits/rejected": -36393083.93343419,
      "logps/chosen": -497.07017366720515,
      "logps/rejected": -366.5627600226929,
      "loss": 0.508,
      "loss/base_kto": 3.656599760055542,
      "loss/total_with_kl": 4.064292907714844,
      "metrics/advantage_var": 244.8606414794922,
      "regularization/advantage_var": 244.8606414794922,
      "regularization/kl": 0.40769296884536743,
      "rewards/chosen": -0.03925414077684837,
      "rewards/margins": 0.337245006393041,
      "rewards/rejected": -0.37649914716988936,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 13.988927841186523,
      "kl": 2.5878779888153076,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -36559256.31397174,
      "logits/rejected": -36681259.79471228,
      "logps/chosen": -482.8315345368917,
      "logps/rejected": -356.6305647356143,
      "loss": 0.5296,
      "loss/base_kto": 3.727613925933838,
      "loss/total_with_kl": 4.237107753753662,
      "metrics/advantage_var": 306.0021057128906,
      "regularization/advantage_var": 306.0021057128906,
      "regularization/kl": 0.5094935297966003,
      "rewards/chosen": -0.0574733788027681,
      "rewards/margins": 0.26809830134035983,
      "rewards/rejected": -0.32557168014312793,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 12.955965995788574,
      "kl": 3.9791793823242188,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -36830659.86031746,
      "logits/rejected": -36714439.286153845,
      "logps/chosen": -482.76468253968255,
      "logps/rejected": -390.1410576923077,
      "loss": 0.5134,
      "loss/base_kto": 3.6922378540039062,
      "loss/total_with_kl": 4.107511520385742,
      "metrics/advantage_var": 249.4137420654297,
      "regularization/advantage_var": 249.4137420654297,
      "regularization/kl": 0.41527387499809265,
      "rewards/chosen": -0.03859777601938399,
      "rewards/margins": 0.3019872929033925,
      "rewards/rejected": -0.34058506892277646,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 11.858966827392578,
      "kl": 3.107635259628296,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -37624990.283911675,
      "logits/rejected": -38347227.54179566,
      "logps/chosen": -479.9210370662461,
      "logps/rejected": -360.61368034055727,
      "loss": 0.5196,
      "loss/base_kto": 3.744265079498291,
      "loss/total_with_kl": 4.1571197509765625,
      "metrics/advantage_var": 247.960693359375,
      "regularization/advantage_var": 247.960693359375,
      "regularization/kl": 0.4128545820713043,
      "rewards/chosen": -0.07257702222757911,
      "rewards/margins": 0.239034141499322,
      "rewards/rejected": -0.3116111637269011,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 12.195089340209961,
      "kl": 4.067375183105469,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -35495411.358024694,
      "logits/rejected": -36405591.49367089,
      "logps/chosen": -455.54841820987656,
      "logps/rejected": -378.18814280063293,
      "loss": 0.5203,
      "loss/base_kto": 3.71710205078125,
      "loss/total_with_kl": 4.162749767303467,
      "metrics/advantage_var": 267.65625,
      "regularization/advantage_var": 267.65625,
      "regularization/kl": 0.44564762711524963,
      "rewards/chosen": 0.07983982415846837,
      "rewards/margins": 0.2812370198055028,
      "rewards/rejected": -0.20139719564703445,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 15.630626678466797,
      "kl": 10.41035270690918,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -35388355.029007636,
      "logits/rejected": -35857553.8176,
      "logps/chosen": -465.7159828244275,
      "logps/rejected": -357.2461,
      "loss": 0.5108,
      "loss/base_kto": 3.6266372203826904,
      "loss/total_with_kl": 4.086459636688232,
      "metrics/advantage_var": 276.1694030761719,
      "regularization/advantage_var": 276.1694030761719,
      "regularization/kl": 0.4598220884799957,
      "rewards/chosen": 0.22740483174797232,
      "rewards/margins": 0.3562315895604723,
      "rewards/rejected": -0.1288267578125,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 15.51076602935791,
      "kl": 20.242170333862305,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -36996972.14132105,
      "logits/rejected": -37566604.0063593,
      "logps/chosen": -493.4741743471582,
      "logps/rejected": -387.02091613672496,
      "loss": 0.5283,
      "loss/base_kto": 3.663759708404541,
      "loss/total_with_kl": 4.226189136505127,
      "metrics/advantage_var": 337.7952575683594,
      "regularization/advantage_var": 337.7952575683594,
      "regularization/kl": 0.5624291300773621,
      "rewards/chosen": 0.4211090134768625,
      "rewards/margins": 0.3138429770475104,
      "rewards/rejected": 0.10726603642935215,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 16.41870880126953,
      "kl": 27.8725528717041,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -35428084.59600615,
      "logits/rejected": -37382629.13831478,
      "logps/chosen": -462.9358198924731,
      "logps/rejected": -382.7552662957075,
      "loss": 0.5079,
      "loss/base_kto": 3.6638245582580566,
      "loss/total_with_kl": 4.063495635986328,
      "metrics/advantage_var": 240.042724609375,
      "regularization/advantage_var": 240.042724609375,
      "regularization/kl": 0.39967110753059387,
      "rewards/chosen": 0.4316009662118376,
      "rewards/margins": 0.24892917213921487,
      "rewards/rejected": 0.1826717940726227,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 15.007033348083496,
      "kl": 13.093035697937012,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -36440083.38170347,
      "logits/rejected": -37480203.888544895,
      "logps/chosen": -464.44829455835963,
      "logps/rejected": -378.70474554953563,
      "loss": 0.5196,
      "loss/base_kto": 3.7181496620178223,
      "loss/total_with_kl": 4.156769752502441,
      "metrics/advantage_var": 263.4352722167969,
      "regularization/advantage_var": 263.4352722167969,
      "regularization/kl": 0.4386197030544281,
      "rewards/chosen": 0.21653328784256703,
      "rewards/margins": 0.24704505239255464,
      "rewards/rejected": -0.030511764549987604,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 21.0418701171875,
      "kl": 16.60923957824707,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -36879919.537290715,
      "logits/rejected": -36814105.06581059,
      "logps/chosen": -499.548848934551,
      "logps/rejected": -359.74197431781704,
      "loss": 0.5134,
      "loss/base_kto": 3.6128647327423096,
      "loss/total_with_kl": 4.107340335845947,
      "metrics/advantage_var": 296.9823913574219,
      "regularization/advantage_var": 296.9823913574219,
      "regularization/kl": 0.49447569251060486,
      "rewards/chosen": 0.3630597072467775,
      "rewards/margins": 0.37933509169968305,
      "rewards/rejected": -0.016275384452905547,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 20.375349044799805,
      "kl": 25.070850372314453,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -36709987.921331316,
      "logits/rejected": -37055876.75605816,
      "logps/chosen": -494.6037254160363,
      "logps/rejected": -383.87136510500807,
      "loss": 0.5273,
      "loss/base_kto": 3.655336856842041,
      "loss/total_with_kl": 4.21851110458374,
      "metrics/advantage_var": 338.24273681640625,
      "regularization/advantage_var": 338.24273681640625,
      "regularization/kl": 0.5631741285324097,
      "rewards/chosen": 0.4840269334016878,
      "rewards/margins": 0.2985697623328646,
      "rewards/rejected": 0.1854571710688232,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 11.905738830566406,
      "kl": 26.71291160583496,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -37440047.40740741,
      "logits/rejected": -38528165.26582278,
      "logps/chosen": -502.99652777777777,
      "logps/rejected": -366.58603639240505,
      "loss": 0.5069,
      "loss/base_kto": 3.6144073009490967,
      "loss/total_with_kl": 4.054908275604248,
      "metrics/advantage_var": 264.5651550292969,
      "regularization/advantage_var": 264.5651550292969,
      "regularization/kl": 0.440500944852829,
      "rewards/chosen": 0.5046551551347898,
      "rewards/margins": 0.2937068629216395,
      "rewards/rejected": 0.21094829221315023,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 15.570670127868652,
      "kl": 15.21961498260498,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -37499581.62962963,
      "logits/rejected": -38416870.07594936,
      "logps/chosen": -523.3862847222222,
      "logps/rejected": -365.23763844936707,
      "loss": 0.5129,
      "loss/base_kto": 3.680920124053955,
      "loss/total_with_kl": 4.1031928062438965,
      "metrics/advantage_var": 253.6173858642578,
      "regularization/advantage_var": 253.6173858642578,
      "regularization/kl": 0.4222729206085205,
      "rewards/chosen": 0.2762401722095631,
      "rewards/margins": 0.2778101491218322,
      "rewards/rejected": -0.0015699769122691095,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 18.33673667907715,
      "kl": 13.349814414978027,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -34919093.41732284,
      "logits/rejected": -37028608.39689922,
      "logps/chosen": -475.5461614173228,
      "logps/rejected": -383.7982315891473,
      "loss": 0.5224,
      "loss/base_kto": 3.6636300086975098,
      "loss/total_with_kl": 4.178959369659424,
      "metrics/advantage_var": 309.5070495605469,
      "regularization/advantage_var": 309.5070495605469,
      "regularization/kl": 0.5153291821479797,
      "rewards/chosen": 0.21472533218503936,
      "rewards/margins": 0.28177217926973536,
      "rewards/rejected": -0.06704684708469598,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 17.798927307128906,
      "kl": 10.555925369262695,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -35837511.461412154,
      "logits/rejected": -38261688.27421758,
      "logps/chosen": -480.0981116584565,
      "logps/rejected": -368.63720193740687,
      "loss": 0.5162,
      "loss/base_kto": 3.6703248023986816,
      "loss/total_with_kl": 4.1295952796936035,
      "metrics/advantage_var": 275.83819580078125,
      "regularization/advantage_var": 275.83819580078125,
      "regularization/kl": 0.45927056670188904,
      "rewards/chosen": 0.162047525540557,
      "rewards/margins": 0.29109999612725135,
      "rewards/rejected": -0.1290524705866943,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 15.719472885131836,
      "kl": 10.699854850769043,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -35092709.68224299,
      "logits/rejected": -35829111.57366771,
      "logps/chosen": -454.16856503115264,
      "logps/rejected": -385.9563822492163,
      "loss": 0.5147,
      "loss/base_kto": 3.670102834701538,
      "loss/total_with_kl": 4.117818355560303,
      "metrics/advantage_var": 268.8981018066406,
      "regularization/advantage_var": 268.8981018066406,
      "regularization/kl": 0.4477153718471527,
      "rewards/chosen": 0.16595170208226853,
      "rewards/margins": 0.2956248616905415,
      "rewards/rejected": -0.12967315960827291,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 16.269304275512695,
      "kl": 19.776601791381836,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -35707229.893416926,
      "logits/rejected": -37422351.15264797,
      "logps/chosen": -493.3255779780564,
      "logps/rejected": -376.56371690031153,
      "loss": 0.5158,
      "loss/base_kto": 3.6863350868225098,
      "loss/total_with_kl": 4.1266865730285645,
      "metrics/advantage_var": 264.475341796875,
      "regularization/advantage_var": 264.475341796875,
      "regularization/kl": 0.4403514564037323,
      "rewards/chosen": 0.36896426401168203,
      "rewards/margins": 0.26900655129750756,
      "rewards/rejected": 0.09995771271417446,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 12.744359016418457,
      "kl": 11.336798667907715,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -35560088.39055118,
      "logits/rejected": -36831657.475968994,
      "logps/chosen": -487.2536417322835,
      "logps/rejected": -345.7433139534884,
      "loss": 0.5088,
      "loss/base_kto": 3.6728012561798096,
      "loss/total_with_kl": 4.070383548736572,
      "metrics/advantage_var": 238.7881622314453,
      "regularization/advantage_var": 238.7881622314453,
      "regularization/kl": 0.3975822627544403,
      "rewards/chosen": 0.18450591320127951,
      "rewards/margins": 0.2887475253621322,
      "rewards/rejected": -0.10424161216085272,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 14.963236808776855,
      "kl": 10.810988426208496,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -36938595.313755795,
      "logits/rejected": -38018664.341232225,
      "logps/chosen": -500.0477202472952,
      "logps/rejected": -379.4044727488152,
      "loss": 0.5245,
      "loss/base_kto": 3.689709424972534,
      "loss/total_with_kl": 4.195741176605225,
      "metrics/advantage_var": 303.9228210449219,
      "regularization/advantage_var": 303.9228210449219,
      "regularization/kl": 0.5060314536094666,
      "rewards/chosen": 0.2027928678108699,
      "rewards/margins": 0.28093195748260297,
      "rewards/rejected": -0.07813908967173307,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 15.082501411437988,
      "kl": 16.48919677734375,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -37628381.20388349,
      "logits/rejected": -38292254.163141996,
      "logps/chosen": -491.7256270226537,
      "logps/rejected": -385.6554003021148,
      "loss": 0.5114,
      "loss/base_kto": 3.6569862365722656,
      "loss/total_with_kl": 4.09153938293457,
      "metrics/advantage_var": 260.99267578125,
      "regularization/advantage_var": 260.99267578125,
      "regularization/kl": 0.4345528185367584,
      "rewards/chosen": 0.252661053802589,
      "rewards/margins": 0.2780839135707218,
      "rewards/rejected": -0.02542285976813279,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 12.365095138549805,
      "kl": 11.425230979919434,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -36415946.36190476,
      "logits/rejected": -38076844.50461538,
      "logps/chosen": -476.50565476190474,
      "logps/rejected": -360.44697115384616,
      "loss": 0.5075,
      "loss/base_kto": 3.651010274887085,
      "loss/total_with_kl": 4.060315132141113,
      "metrics/advantage_var": 245.828369140625,
      "regularization/advantage_var": 245.828369140625,
      "regularization/kl": 0.4093042314052582,
      "rewards/chosen": 0.12858787415519593,
      "rewards/margins": 0.3083314795488257,
      "rewards/rejected": -0.1797436053936298,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 9.462163925170898,
      "kl": 7.94988489151001,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -35250398.03236246,
      "logits/rejected": -35756571.92727273,
      "logps/chosen": -466.686286407767,
      "logps/rejected": -388.2438446969697,
      "loss": 0.525,
      "loss/base_kto": 3.443466901779175,
      "loss/total_with_kl": 4.199981212615967,
      "metrics/advantage_var": 454.363037109375,
      "regularization/advantage_var": 454.363037109375,
      "regularization/kl": 0.7565144896507263,
      "rewards/chosen": 0.2011941952998584,
      "rewards/margins": 0.590251192888045,
      "rewards/rejected": -0.38905699758818657,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 15.779438972473145,
      "kl": 8.964886665344238,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -36306812.275650844,
      "logits/rejected": -37220629.63955343,
      "logps/chosen": -463.8027852220521,
      "logps/rejected": -387.6121162280702,
      "loss": 0.5334,
      "loss/base_kto": 3.242673635482788,
      "loss/total_with_kl": 4.267380237579346,
      "metrics/advantage_var": 615.4395751953125,
      "regularization/advantage_var": 615.4395751953125,
      "regularization/kl": 1.0247067213058472,
      "rewards/chosen": 0.43634005162471284,
      "rewards/margins": 0.843648992116701,
      "rewards/rejected": -0.4073089404919881,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 16.526897430419922,
      "kl": 18.56119155883789,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -35259160.1509434,
      "logits/rejected": -36981903.10559006,
      "logps/chosen": -442.0420597484277,
      "logps/rejected": -392.12567934782606,
      "loss": 0.5062,
      "loss/base_kto": 3.1583974361419678,
      "loss/total_with_kl": 4.049962997436523,
      "metrics/advantage_var": 535.4747924804688,
      "regularization/advantage_var": 535.4747924804688,
      "regularization/kl": 0.8915656208992004,
      "rewards/chosen": 0.5975148902749116,
      "rewards/margins": 0.886257548048895,
      "rewards/rejected": -0.2887426577739834,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 13.164573669433594,
      "kl": 5.3090996742248535,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -35334127.29526917,
      "logits/rejected": -35689172.629685156,
      "logps/chosen": -512.6254078303426,
      "logps/rejected": -366.82208114692656,
      "loss": 0.5442,
      "loss/base_kto": 3.19159197807312,
      "loss/total_with_kl": 4.353483200073242,
      "metrics/advantage_var": 697.8326416015625,
      "regularization/advantage_var": 697.8326416015625,
      "regularization/kl": 1.1618913412094116,
      "rewards/chosen": 0.2475855268817547,
      "rewards/margins": 0.8847968132063658,
      "rewards/rejected": -0.6372112863246111,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 14.01774787902832,
      "kl": 12.346087455749512,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -36277728.84867395,
      "logits/rejected": -36667039.44913928,
      "logps/chosen": -460.0459243369735,
      "logps/rejected": -373.7971439749609,
      "loss": 0.5197,
      "loss/base_kto": 3.16581392288208,
      "loss/total_with_kl": 4.157769680023193,
      "metrics/advantage_var": 595.7692260742188,
      "regularization/advantage_var": 595.7692260742188,
      "regularization/kl": 0.9919557571411133,
      "rewards/chosen": 0.4603787569471529,
      "rewards/margins": 0.8959217157634185,
      "rewards/rejected": -0.43554295881626565,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 17.604755401611328,
      "kl": 10.787038803100586,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -35393958.10687023,
      "logits/rejected": -36645922.4064,
      "logps/chosen": -461.3892175572519,
      "logps/rejected": -370.68335,
      "loss": 0.5349,
      "loss/base_kto": 3.1907060146331787,
      "loss/total_with_kl": 4.279114723205566,
      "metrics/advantage_var": 653.69873046875,
      "regularization/advantage_var": 653.69873046875,
      "regularization/kl": 1.088408350944519,
      "rewards/chosen": 0.46098511674021947,
      "rewards/margins": 0.8803063081464695,
      "rewards/rejected": -0.41932119140625,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 28.774839401245117,
      "kl": 15.302284240722656,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -37184880.145015106,
      "logits/rejected": -37203610.097087376,
      "logps/chosen": -506.91814577039275,
      "logps/rejected": -382.5256877022654,
      "loss": 0.5375,
      "loss/base_kto": 3.16581130027771,
      "loss/total_with_kl": 4.299928188323975,
      "metrics/advantage_var": 681.1513061523438,
      "regularization/advantage_var": 681.1513061523438,
      "regularization/kl": 1.1341170072555542,
      "rewards/chosen": 0.599076101066843,
      "rewards/margins": 0.8863653470673613,
      "rewards/rejected": -0.2872892460005183,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 12.60871696472168,
      "kl": 12.3494873046875,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -34259159.578947365,
      "logits/rejected": -35080630.85714286,
      "logps/chosen": -488.53196957236844,
      "logps/rejected": -376.5957961309524,
      "loss": 0.5604,
      "loss/base_kto": 3.1485283374786377,
      "loss/total_with_kl": 4.483153343200684,
      "metrics/advantage_var": 801.5763549804688,
      "regularization/advantage_var": 801.5763549804688,
      "regularization/kl": 1.3346246480941772,
      "rewards/chosen": 0.4776663529245477,
      "rewards/margins": 0.952251386522948,
      "rewards/rejected": -0.4745850335984003,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 17.281497955322266,
      "kl": 10.597274780273438,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -35714048.0,
      "logits/rejected": -37265164.487804875,
      "logps/chosen": -488.4378004807692,
      "logps/rejected": -372.6516053734756,
      "loss": 0.5289,
      "loss/base_kto": 3.154386281967163,
      "loss/total_with_kl": 4.231297969818115,
      "metrics/advantage_var": 646.7941284179688,
      "regularization/advantage_var": 646.7941284179688,
      "regularization/kl": 1.0769122838974,
      "rewards/chosen": 0.5121666345840845,
      "rewards/margins": 0.8904215715466774,
      "rewards/rejected": -0.3782549369625929,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 18.517337799072266,
      "kl": 11.049582481384277,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -36722954.71627907,
      "logits/rejected": -38406360.088188976,
      "logps/chosen": -491.9407945736434,
      "logps/rejected": -379.6351131889764,
      "loss": 0.5082,
      "loss/base_kto": 3.207235336303711,
      "loss/total_with_kl": 4.065227031707764,
      "metrics/advantage_var": 515.3104248046875,
      "regularization/advantage_var": 515.3104248046875,
      "regularization/kl": 0.8579918146133423,
      "rewards/chosen": 0.46582092758296995,
      "rewards/margins": 0.8286288907965527,
      "rewards/rejected": -0.36280796321358266,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 12.599120140075684,
      "kl": 15.788786888122559,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -35283677.70716511,
      "logits/rejected": -36716287.19749216,
      "logps/chosen": -474.21928543613706,
      "logps/rejected": -393.32949647335425,
      "loss": 0.5195,
      "loss/base_kto": 3.2040252685546875,
      "loss/total_with_kl": 4.156173229217529,
      "metrics/advantage_var": 571.8604736328125,
      "regularization/advantage_var": 571.8604736328125,
      "regularization/kl": 0.9521476030349731,
      "rewards/chosen": 0.5738645179249416,
      "rewards/margins": 0.8790741172434908,
      "rewards/rejected": -0.3052095993185492,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 14.024765014648438,
      "kl": 18.037206649780273,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -34794381.14375988,
      "logits/rejected": -34839822.63987635,
      "logps/chosen": -468.80677330173773,
      "logps/rejected": -405.15938948995364,
      "loss": 0.5548,
      "loss/base_kto": 3.163182497024536,
      "loss/total_with_kl": 4.438502788543701,
      "metrics/advantage_var": 765.958251953125,
      "regularization/advantage_var": 765.958251953125,
      "regularization/kl": 1.2753204107284546,
      "rewards/chosen": 0.5771619365866903,
      "rewards/margins": 0.9175937749391827,
      "rewards/rejected": -0.3404318383524923,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 17.971912384033203,
      "kl": 13.64586353302002,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -35411977.19760479,
      "logits/rejected": -35166495.79084967,
      "logps/chosen": -476.7110778443114,
      "logps/rejected": -385.3743617238562,
      "loss": 0.5063,
      "loss/base_kto": 3.1725013256073,
      "loss/total_with_kl": 4.050698280334473,
      "metrics/advantage_var": 527.4454956054688,
      "regularization/advantage_var": 527.4454956054688,
      "regularization/kl": 0.8781967163085938,
      "rewards/chosen": 0.5798879383566851,
      "rewards/margins": 0.8787531631575429,
      "rewards/rejected": -0.29886522480085786,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 13.165546417236328,
      "kl": 20.489919662475586,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -34604458.538345866,
      "logits/rejected": -36270602.822764225,
      "logps/chosen": -509.2143327067669,
      "logps/rejected": -364.005081300813,
      "loss": 0.5053,
      "loss/base_kto": 3.206397771835327,
      "loss/total_with_kl": 4.042151927947998,
      "metrics/advantage_var": 501.9543151855469,
      "regularization/advantage_var": 501.9543151855469,
      "regularization/kl": 0.8357540369033813,
      "rewards/chosen": 0.6431686516094924,
      "rewards/margins": 0.8166307889713446,
      "rewards/rejected": -0.17346213736185215,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 12.91707706451416,
      "kl": 21.993675231933594,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -35183648.61146497,
      "logits/rejected": -35695361.57055215,
      "logps/chosen": -490.61962579617835,
      "logps/rejected": -376.44924271472394,
      "loss": 0.5275,
      "loss/base_kto": 3.1577212810516357,
      "loss/total_with_kl": 4.2201361656188965,
      "metrics/advantage_var": 638.0870361328125,
      "regularization/advantage_var": 638.0870361328125,
      "regularization/kl": 1.0624150037765503,
      "rewards/chosen": 0.7239522751729199,
      "rewards/margins": 0.9027313857596363,
      "rewards/rejected": -0.17877911058671636,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 15.724650382995605,
      "kl": 16.417760848999023,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -36217919.49125596,
      "logits/rejected": -37836051.2688172,
      "logps/chosen": -503.1971383147854,
      "logps/rejected": -366.30568356374806,
      "loss": 0.4982,
      "loss/base_kto": 3.1603543758392334,
      "loss/total_with_kl": 3.985971450805664,
      "metrics/advantage_var": 495.8663024902344,
      "regularization/advantage_var": 495.8663024902344,
      "regularization/kl": 0.8256174325942993,
      "rewards/chosen": 0.5560551629726004,
      "rewards/margins": 0.8365336448454402,
      "rewards/rejected": -0.28047848187283986,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 8.864245414733887,
      "kl": 18.243162155151367,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -34416254.00623053,
      "logits/rejected": -35867651.210031345,
      "logps/chosen": -467.94256230529595,
      "logps/rejected": -376.60295846394985,
      "loss": 0.5065,
      "loss/base_kto": 3.1706998348236084,
      "loss/total_with_kl": 4.052399158477783,
      "metrics/advantage_var": 529.5491943359375,
      "regularization/advantage_var": 529.5491943359375,
      "regularization/kl": 0.8816993832588196,
      "rewards/chosen": 0.6203484936295268,
      "rewards/margins": 0.8738607416059914,
      "rewards/rejected": -0.25351224797646454,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 18.818096160888672,
      "kl": 15.484634399414062,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -34946939.65751211,
      "logits/rejected": -35976376.35098336,
      "logps/chosen": -469.2755452342488,
      "logps/rejected": -377.5408472012103,
      "loss": 0.4996,
      "loss/base_kto": 3.171642780303955,
      "loss/total_with_kl": 3.996892213821411,
      "metrics/advantage_var": 495.6452331542969,
      "regularization/advantage_var": 495.6452331542969,
      "regularization/kl": 0.8252493143081665,
      "rewards/chosen": 0.5336291878597031,
      "rewards/margins": 0.8582243059336441,
      "rewards/rejected": -0.324595118073941,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 37.44119644165039,
      "kl": 11.217509269714355,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -34135380.24801272,
      "logits/rejected": -35630465.376344085,
      "logps/chosen": -484.7997813990461,
      "logps/rejected": -376.2759216589862,
      "loss": 0.5282,
      "loss/base_kto": 3.1395866870880127,
      "loss/total_with_kl": 4.225513458251953,
      "metrics/advantage_var": 652.2083129882812,
      "regularization/advantage_var": 652.2083129882812,
      "regularization/kl": 1.08592689037323,
      "rewards/chosen": 0.5123951825504273,
      "rewards/margins": 0.926046468732743,
      "rewards/rejected": -0.41365128618231567,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 39.47532653808594,
      "kl": 12.592890739440918,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -34648123.55411955,
      "logits/rejected": -36117406.40242057,
      "logps/chosen": -482.23046243941843,
      "logps/rejected": -373.59852496217854,
      "loss": 0.5135,
      "loss/base_kto": 3.1341800689697266,
      "loss/total_with_kl": 4.107840061187744,
      "metrics/advantage_var": 584.7808227539062,
      "regularization/advantage_var": 584.7808227539062,
      "regularization/kl": 0.9736601114273071,
      "rewards/chosen": 0.5263789212376313,
      "rewards/margins": 0.9207766510622342,
      "rewards/rejected": -0.39439772982460286,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 12.804630279541016,
      "kl": 9.709197044372559,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -34007461.453376204,
      "logits/rejected": -35523036.20668693,
      "logps/chosen": -498.3818830385852,
      "logps/rejected": -345.47091090425533,
      "loss": 0.4968,
      "loss/base_kto": 3.1976964473724365,
      "loss/total_with_kl": 3.974379777908325,
      "metrics/advantage_var": 466.4764709472656,
      "regularization/advantage_var": 466.4764709472656,
      "regularization/kl": 0.7766832709312439,
      "rewards/chosen": 0.4751230712105607,
      "rewards/margins": 0.8474625227772163,
      "rewards/rejected": -0.3723394515666556,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 22.580102920532227,
      "kl": 13.041857719421387,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -34118773.1589896,
      "logits/rejected": -37289167.49917628,
      "logps/chosen": -502.77693164933135,
      "logps/rejected": -363.8554880560132,
      "loss": 0.5379,
      "loss/base_kto": 3.1549394130706787,
      "loss/total_with_kl": 4.303187847137451,
      "metrics/advantage_var": 689.6389770507812,
      "regularization/advantage_var": 689.6389770507812,
      "regularization/kl": 1.1482489109039307,
      "rewards/chosen": 0.6239117059806835,
      "rewards/margins": 0.9157528138723381,
      "rewards/rejected": -0.2918411078916546,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 294.7341003417969,
      "kl": 13.90600872039795,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -34551348.63551402,
      "logits/rejected": -36423211.335423194,
      "logps/chosen": -462.7169003115265,
      "logps/rejected": -376.18399784482756,
      "loss": 0.6041,
      "loss/base_kto": 3.23050856590271,
      "loss/total_with_kl": 4.832540035247803,
      "metrics/advantage_var": 962.1810913085938,
      "regularization/advantage_var": 962.1810913085938,
      "regularization/kl": 1.6020317077636719,
      "rewards/chosen": 0.561777560510368,
      "rewards/margins": 0.869574050740151,
      "rewards/rejected": -0.307796490229783,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 20.57028579711914,
      "kl": 13.543728828430176,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -33880727.16190476,
      "logits/rejected": -36298474.732307695,
      "logps/chosen": -507.5727182539683,
      "logps/rejected": -376.46466346153846,
      "loss": 0.5345,
      "loss/base_kto": 3.1752588748931885,
      "loss/total_with_kl": 4.276364326477051,
      "metrics/advantage_var": 661.3244018554688,
      "regularization/advantage_var": 661.3244018554688,
      "regularization/kl": 1.1011050939559937,
      "rewards/chosen": 0.578230697389633,
      "rewards/margins": 0.9060270700233468,
      "rewards/rejected": -0.3277963726337139,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 22.362438201904297,
      "kl": 13.31640625,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -33722781.50715421,
      "logits/rejected": -35482087.61904762,
      "logps/chosen": -482.0680147058824,
      "logps/rejected": -355.771001344086,
      "loss": 0.498,
      "loss/base_kto": 3.190498113632202,
      "loss/total_with_kl": 3.9841275215148926,
      "metrics/advantage_var": 476.6542663574219,
      "regularization/advantage_var": 476.6542663574219,
      "regularization/kl": 0.7936292886734009,
      "rewards/chosen": 0.532119823752981,
      "rewards/margins": 0.8617982669233438,
      "rewards/rejected": -0.3296784431703629,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 10.12584114074707,
      "kl": 9.132817268371582,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -34739407.372056514,
      "logits/rejected": -35731210.7496112,
      "logps/chosen": -473.11548273155415,
      "logps/rejected": -369.72633164852255,
      "loss": 0.513,
      "loss/base_kto": 3.213224411010742,
      "loss/total_with_kl": 4.104243755340576,
      "metrics/advantage_var": 535.1466674804688,
      "regularization/advantage_var": 535.1466674804688,
      "regularization/kl": 0.8910191655158997,
      "rewards/chosen": 0.41781879706510006,
      "rewards/margins": 0.8382309466872326,
      "rewards/rejected": -0.4204121496221326,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 11.648301124572754,
      "kl": 13.205253601074219,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -33042906.80680062,
      "logits/rejected": -34789291.87993681,
      "logps/chosen": -469.2457496136012,
      "logps/rejected": -368.17145537124804,
      "loss": 0.5159,
      "loss/base_kto": 3.186049222946167,
      "loss/total_with_kl": 4.12746524810791,
      "metrics/advantage_var": 565.4149780273438,
      "regularization/advantage_var": 565.4149780273438,
      "regularization/kl": 0.9414157867431641,
      "rewards/chosen": 0.5633167579332496,
      "rewards/margins": 0.8688969814626977,
      "rewards/rejected": -0.3055802235294481,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 9.580194473266602,
      "kl": 14.520026206970215,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -34545826.90909091,
      "logits/rejected": -35603428.240963854,
      "logps/chosen": -483.2869318181818,
      "logps/rejected": -386.94423004518075,
      "loss": 0.5089,
      "loss/base_kto": 3.1537270545959473,
      "loss/total_with_kl": 4.070878505706787,
      "metrics/advantage_var": 550.841796875,
      "regularization/advantage_var": 550.841796875,
      "regularization/kl": 0.9171515703201294,
      "rewards/chosen": 0.5488103395932681,
      "rewards/margins": 0.909023332066089,
      "rewards/rejected": -0.36021299247282096,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 34.79518127441406,
      "kl": 6.8231730461120605,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -34463994.90352221,
      "logits/rejected": -35986306.24561404,
      "logps/chosen": -483.4157733537519,
      "logps/rejected": -375.83243620414675,
      "loss": 0.521,
      "loss/base_kto": 3.204749822616577,
      "loss/total_with_kl": 4.167901515960693,
      "metrics/advantage_var": 578.4695434570312,
      "regularization/advantage_var": 578.4695434570312,
      "regularization/kl": 0.963151752948761,
      "rewards/chosen": 0.3909355969743013,
      "rewards/margins": 0.9083980277427173,
      "rewards/rejected": -0.517462430768416,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 13.085524559020996,
      "kl": 15.920722961425781,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -34013177.71779141,
      "logits/rejected": -35884775.541401275,
      "logps/chosen": -482.31643021472394,
      "logps/rejected": -382.711236066879,
      "loss": 0.5202,
      "loss/base_kto": 3.112375259399414,
      "loss/total_with_kl": 4.161748886108398,
      "metrics/advantage_var": 630.25439453125,
      "regularization/advantage_var": 630.25439453125,
      "regularization/kl": 1.0493735074996948,
      "rewards/chosen": 0.6457256480960026,
      "rewards/margins": 0.9098842979050442,
      "rewards/rejected": -0.2641586498090416,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 14.14444351196289,
      "kl": 14.376886367797852,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -34573030.68131868,
      "logits/rejected": -35671637.20062208,
      "logps/chosen": -473.5124116954474,
      "logps/rejected": -392.93164366251943,
      "loss": 0.5181,
      "loss/base_kto": 3.2205846309661865,
      "loss/total_with_kl": 4.144931316375732,
      "metrics/advantage_var": 555.1632690429688,
      "regularization/advantage_var": 555.1632690429688,
      "regularization/kl": 0.9243467450141907,
      "rewards/chosen": 0.5032569969093407,
      "rewards/margins": 0.8434753584865422,
      "rewards/rejected": -0.3402183615772016,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 12.696953773498535,
      "kl": 16.796072006225586,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -34975784.15686274,
      "logits/rejected": -35659706.29497569,
      "logps/chosen": -472.6144419306184,
      "logps/rejected": -349.7544064019449,
      "loss": 0.5166,
      "loss/base_kto": 3.1975579261779785,
      "loss/total_with_kl": 4.13291072845459,
      "metrics/advantage_var": 561.7733764648438,
      "regularization/advantage_var": 561.7733764648438,
      "regularization/kl": 0.9353527426719666,
      "rewards/chosen": 0.5741287162401018,
      "rewards/margins": 0.8371576199972559,
      "rewards/rejected": -0.2630289037571541,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 7.896336555480957,
      "kl": 15.931853294372559,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -34911612.52950558,
      "logits/rejected": -35763725.32924962,
      "logps/chosen": -483.26226076555025,
      "logps/rejected": -358.62016653905056,
      "loss": 0.5232,
      "loss/base_kto": 3.2096874713897705,
      "loss/total_with_kl": 4.185441493988037,
      "metrics/advantage_var": 586.0383911132812,
      "regularization/advantage_var": 586.0383911132812,
      "regularization/kl": 0.9757537841796875,
      "rewards/chosen": 0.5678250496847588,
      "rewards/margins": 0.84200268273953,
      "rewards/rejected": -0.2741776330547713,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 16.918319702148438,
      "kl": 18.675029754638672,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -33134305.720430106,
      "logits/rejected": -34378620.13354531,
      "logps/chosen": -474.04877112135176,
      "logps/rejected": -351.6985790937997,
      "loss": 0.4963,
      "loss/base_kto": 3.1137306690216064,
      "loss/total_with_kl": 3.9700634479522705,
      "metrics/advantage_var": 514.3140258789062,
      "regularization/advantage_var": 514.3140258789062,
      "regularization/kl": 0.8563327789306641,
      "rewards/chosen": 0.7106950469830069,
      "rewards/margins": 0.9445379300703106,
      "rewards/rejected": -0.23384288308730375,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 19.034807205200195,
      "kl": 13.459898948669434,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -34716878.06811146,
      "logits/rejected": -35929924.643533126,
      "logps/chosen": -469.08126934984523,
      "logps/rejected": -394.77743000788644,
      "loss": 0.5015,
      "loss/base_kto": 3.2017509937286377,
      "loss/total_with_kl": 4.01181173324585,
      "metrics/advantage_var": 486.5226745605469,
      "regularization/advantage_var": 486.5226745605469,
      "regularization/kl": 0.8100603222846985,
      "rewards/chosen": 0.4974288231817192,
      "rewards/margins": 0.8555690646421885,
      "rewards/rejected": -0.35814024146046924,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 27.855440139770508,
      "kl": 18.5737247467041,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -34781177.69846154,
      "logits/rejected": -35781245.155555554,
      "logps/chosen": -493.2761538461538,
      "logps/rejected": -390.83214285714286,
      "loss": 0.5013,
      "loss/base_kto": 3.1283771991729736,
      "loss/total_with_kl": 4.010641574859619,
      "metrics/advantage_var": 529.8883666992188,
      "regularization/advantage_var": 529.8883666992188,
      "regularization/kl": 0.8822641372680664,
      "rewards/chosen": 0.6580948110727164,
      "rewards/margins": 0.9275835302257422,
      "rewards/rejected": -0.2694887191530258,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 23.49469757080078,
      "kl": 16.161033630371094,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -35162405.50318471,
      "logits/rejected": -37646178.944785275,
      "logps/chosen": -474.16142515923565,
      "logps/rejected": -371.1085601993865,
      "loss": 0.4972,
      "loss/base_kto": 3.156907558441162,
      "loss/total_with_kl": 3.977708578109741,
      "metrics/advantage_var": 492.9736022949219,
      "regularization/advantage_var": 492.9736022949219,
      "regularization/kl": 0.8208009600639343,
      "rewards/chosen": 0.6076517287333301,
      "rewards/margins": 0.8742902212473495,
      "rewards/rejected": -0.2666384925140194,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 22.5130558013916,
      "kl": 16.55893898010254,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -33921503.255813956,
      "logits/rejected": -35077348.324324325,
      "logps/chosen": -476.2427779796512,
      "logps/rejected": -374.5779930320946,
      "loss": 0.5097,
      "loss/base_kto": 3.1985280513763428,
      "loss/total_with_kl": 4.077399730682373,
      "metrics/advantage_var": 527.8506469726562,
      "regularization/advantage_var": 527.8506469726562,
      "regularization/kl": 0.8788713812828064,
      "rewards/chosen": 0.6884309192036473,
      "rewards/margins": 0.8503661326534863,
      "rewards/rejected": -0.161935213449839,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 21.97184944152832,
      "kl": 19.698619842529297,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -35926641.144674085,
      "logits/rejected": -36667335.37327189,
      "logps/chosen": -484.2610790937997,
      "logps/rejected": -388.83419738863284,
      "loss": 0.5118,
      "loss/base_kto": 3.1325860023498535,
      "loss/total_with_kl": 4.094533920288086,
      "metrics/advantage_var": 577.74658203125,
      "regularization/advantage_var": 577.74658203125,
      "regularization/kl": 0.961948037147522,
      "rewards/chosen": 0.7117422131172992,
      "rewards/margins": 0.9445239236467731,
      "rewards/rejected": -0.23278171052947388,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 12.882615089416504,
      "kl": 15.500436782836914,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -33707717.4379085,
      "logits/rejected": -34998179.74774775,
      "logps/chosen": -487.6294934640523,
      "logps/rejected": -376.18308933933935,
      "loss": 0.47,
      "loss/base_kto": 3.1957995891571045,
      "loss/total_with_kl": 3.759805679321289,
      "metrics/advantage_var": 338.74224853515625,
      "regularization/advantage_var": 338.74224853515625,
      "regularization/kl": 0.5640057921409607,
      "rewards/chosen": 0.512441248675577,
      "rewards/margins": 0.8036668080362073,
      "rewards/rejected": -0.29122555936063016,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 10.57919692993164,
      "kl": 13.648648262023926,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -34193498.75332349,
      "logits/rejected": -34592068.35157546,
      "logps/chosen": -457.49312223042836,
      "logps/rejected": -381.55161691542287,
      "loss": 0.4713,
      "loss/base_kto": 3.136463165283203,
      "loss/total_with_kl": 3.770439624786377,
      "metrics/advantage_var": 380.7665710449219,
      "regularization/advantage_var": 380.7665710449219,
      "regularization/kl": 0.6339762806892395,
      "rewards/chosen": 0.6271113025208872,
      "rewards/margins": 0.9073408615130358,
      "rewards/rejected": -0.28022955899214863,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 9.71323013305664,
      "kl": 19.46393394470215,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -33426026.264150944,
      "logits/rejected": -36114492.42236025,
      "logps/chosen": -497.7997248427673,
      "logps/rejected": -360.94589479813664,
      "loss": 0.4692,
      "loss/base_kto": 3.1342921257019043,
      "loss/total_with_kl": 3.753847122192383,
      "metrics/advantage_var": 372.105224609375,
      "regularization/advantage_var": 372.105224609375,
      "regularization/kl": 0.6195551753044128,
      "rewards/chosen": 0.6923610279395146,
      "rewards/margins": 0.8642818043067021,
      "rewards/rejected": -0.1719207763671875,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 13.907105445861816,
      "kl": 26.03472900390625,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -34063458.79326187,
      "logits/rejected": -36368384.0,
      "logps/chosen": -481.4810490045942,
      "logps/rejected": -384.0566437400319,
      "loss": 0.4609,
      "loss/base_kto": 3.116948127746582,
      "loss/total_with_kl": 3.6871414184570312,
      "metrics/advantage_var": 342.4584655761719,
      "regularization/advantage_var": 342.4584655761719,
      "regularization/kl": 0.5701932907104492,
      "rewards/chosen": 0.7419502760695827,
      "rewards/margins": 0.8515068726947572,
      "rewards/rejected": -0.10955659662517445,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 11.830923080444336,
      "kl": 15.881300926208496,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -34649600.80376766,
      "logits/rejected": -34635619.93157076,
      "logps/chosen": -481.37102629513345,
      "logps/rejected": -373.75758164852255,
      "loss": 0.4617,
      "loss/base_kto": 3.110450506210327,
      "loss/total_with_kl": 3.693946123123169,
      "metrics/advantage_var": 350.4477233886719,
      "regularization/advantage_var": 350.4477233886719,
      "regularization/kl": 0.5834954380989075,
      "rewards/chosen": 0.648567618904533,
      "rewards/margins": 0.891513929229669,
      "rewards/rejected": -0.24294631032513608,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 12.304580688476562,
      "kl": 13.850641250610352,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -34127951.79220779,
      "logits/rejected": -34919266.69879518,
      "logps/chosen": -471.86444805194805,
      "logps/rejected": -370.60791603915663,
      "loss": 0.4674,
      "loss/base_kto": 3.119288206100464,
      "loss/total_with_kl": 3.739403486251831,
      "metrics/advantage_var": 372.4416809082031,
      "regularization/advantage_var": 372.4416809082031,
      "regularization/kl": 0.6201154589653015,
      "rewards/chosen": 0.5691945459935572,
      "rewards/margins": 0.8991698350281134,
      "rewards/rejected": -0.3299752890345562,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 15.00333309173584,
      "kl": 10.780613899230957,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -34084100.770186335,
      "logits/rejected": -36909487.49685535,
      "logps/chosen": -489.0129076086956,
      "logps/rejected": -361.6150501179245,
      "loss": 0.4788,
      "loss/base_kto": 3.13610577583313,
      "loss/total_with_kl": 3.8302910327911377,
      "metrics/advantage_var": 416.9278869628906,
      "regularization/advantage_var": 416.9278869628906,
      "regularization/kl": 0.6941849589347839,
      "rewards/chosen": 0.5348799806203901,
      "rewards/margins": 0.894672246122552,
      "rewards/rejected": -0.3597922655021619,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 10.705175399780273,
      "kl": 12.834040641784668,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -34572940.65934066,
      "logits/rejected": -36874281.4059098,
      "logps/chosen": -524.3274627158555,
      "logps/rejected": -371.9184000777605,
      "loss": 0.4585,
      "loss/base_kto": 3.0802323818206787,
      "loss/total_with_kl": 3.6678178310394287,
      "metrics/advantage_var": 352.9039001464844,
      "regularization/advantage_var": 352.9039001464844,
      "regularization/kl": 0.5875850915908813,
      "rewards/chosen": 0.5883653961133732,
      "rewards/margins": 0.9297093722187266,
      "rewards/rejected": -0.34134397610535333,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.3886027336120605,
      "kl": 15.603533744812012,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -34152773.178625956,
      "logits/rejected": -36127411.4048,
      "logps/chosen": -473.0314885496183,
      "logps/rejected": -391.2823,
      "loss": 0.4593,
      "loss/base_kto": 3.057488203048706,
      "loss/total_with_kl": 3.67455792427063,
      "metrics/advantage_var": 370.61236572265625,
      "regularization/advantage_var": 370.61236572265625,
      "regularization/kl": 0.6170695424079895,
      "rewards/chosen": 0.6530079150017891,
      "rewards/margins": 0.9697260058221016,
      "rewards/rejected": -0.3167180908203125,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 18.078439712524414,
      "kl": 13.160568237304688,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -34044216.67175572,
      "logits/rejected": -35952535.1424,
      "logps/chosen": -492.331965648855,
      "logps/rejected": -364.4094,
      "loss": 0.4659,
      "loss/base_kto": 3.1652474403381348,
      "loss/total_with_kl": 3.7272117137908936,
      "metrics/advantage_var": 337.51605224609375,
      "regularization/advantage_var": 337.51605224609375,
      "regularization/kl": 0.561964213848114,
      "rewards/chosen": 0.5588921233897901,
      "rewards/margins": 0.84985140073354,
      "rewards/rejected": -0.29095927734375,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 7.16786527633667,
      "kl": 19.17708396911621,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -34342403.92036753,
      "logits/rejected": -34944413.19298246,
      "logps/chosen": -442.2739758805513,
      "logps/rejected": -371.99456738437004,
      "loss": 0.4596,
      "loss/base_kto": 3.1005120277404785,
      "loss/total_with_kl": 3.6771087646484375,
      "metrics/advantage_var": 346.304443359375,
      "regularization/advantage_var": 346.304443359375,
      "regularization/kl": 0.5765969157218933,
      "rewards/chosen": 0.6813067075486935,
      "rewards/margins": 0.9012701098176827,
      "rewards/rejected": -0.21996340226898922,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 11.32540512084961,
      "kl": 21.220792770385742,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -35406720.3987539,
      "logits/rejected": -36062256.15047022,
      "logps/chosen": -486.7122760903427,
      "logps/rejected": -391.71177018025077,
      "loss": 0.4586,
      "loss/base_kto": 3.080542802810669,
      "loss/total_with_kl": 3.6685128211975098,
      "metrics/advantage_var": 353.13525390625,
      "regularization/advantage_var": 353.13525390625,
      "regularization/kl": 0.5879701972007751,
      "rewards/chosen": 0.6702795296071846,
      "rewards/margins": 0.8975216914685924,
      "rewards/rejected": -0.22724216186140772,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 13.638971328735352,
      "kl": 17.907649993896484,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -36546572.740279935,
      "logits/rejected": -35178878.593406595,
      "logps/chosen": -487.7169031881804,
      "logps/rejected": -367.28115188383043,
      "loss": 0.4589,
      "loss/base_kto": 3.0799336433410645,
      "loss/total_with_kl": 3.671400785446167,
      "metrics/advantage_var": 355.23553466796875,
      "regularization/advantage_var": 355.23553466796875,
      "regularization/kl": 0.5914670825004578,
      "rewards/chosen": 0.6589480766426905,
      "rewards/margins": 0.9246811816898476,
      "rewards/rejected": -0.2657331050471571,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 14.756328582763672,
      "kl": 18.99579429626465,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -34947643.23296355,
      "logits/rejected": -36761795.64869029,
      "logps/chosen": -463.9481973058637,
      "logps/rejected": -354.99727946841296,
      "loss": 0.464,
      "loss/base_kto": 3.133209466934204,
      "loss/total_with_kl": 3.71171498298645,
      "metrics/advantage_var": 347.45062255859375,
      "regularization/advantage_var": 347.45062255859375,
      "regularization/kl": 0.5785053372383118,
      "rewards/chosen": 0.5697169477701565,
      "rewards/margins": 0.8377728209885694,
      "rewards/rejected": -0.26805587321841295,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 14.190794944763184,
      "kl": 17.259674072265625,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -35423563.627172194,
      "logits/rejected": -36394916.20401855,
      "logps/chosen": -454.6295912322275,
      "logps/rejected": -395.64364374034005,
      "loss": 0.4631,
      "loss/base_kto": 3.128011465072632,
      "loss/total_with_kl": 3.704951524734497,
      "metrics/advantage_var": 346.5107727050781,
      "regularization/advantage_var": 346.5107727050781,
      "regularization/kl": 0.5769404768943787,
      "rewards/chosen": 0.5691574602895438,
      "rewards/margins": 0.8692318062039807,
      "rewards/rejected": -0.3000743459144368,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 11.9459810256958,
      "kl": 12.720152854919434,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -35945476.777604975,
      "logits/rejected": -35812575.447409734,
      "logps/chosen": -470.9396870139969,
      "logps/rejected": -392.8932986656201,
      "loss": 0.4593,
      "loss/base_kto": 3.111800193786621,
      "loss/total_with_kl": 3.6745898723602295,
      "metrics/advantage_var": 338.0118103027344,
      "regularization/advantage_var": 338.0118103027344,
      "regularization/kl": 0.5627896785736084,
      "rewards/chosen": 0.5418180996670878,
      "rewards/margins": 0.8896113673647918,
      "rewards/rejected": -0.3477932676977041,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 9.321317672729492,
      "kl": 17.556133270263672,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -32271426.990654204,
      "logits/rejected": -35846343.02194358,
      "logps/chosen": -465.73481308411215,
      "logps/rejected": -385.6867163009404,
      "loss": 0.4578,
      "loss/base_kto": 3.1206257343292236,
      "loss/total_with_kl": 3.6624162197113037,
      "metrics/advantage_var": 325.39959716796875,
      "regularization/advantage_var": 325.39959716796875,
      "regularization/kl": 0.5417903065681458,
      "rewards/chosen": 0.6162986874209014,
      "rewards/margins": 0.874495168390178,
      "rewards/rejected": -0.25819648096927655,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 11.190893173217773,
      "kl": 17.787485122680664,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -34321935.61476725,
      "logits/rejected": -36091034.30136986,
      "logps/chosen": -473.31335272873196,
      "logps/rejected": -383.24210426179604,
      "loss": 0.4655,
      "loss/base_kto": 3.11572265625,
      "loss/total_with_kl": 3.7238259315490723,
      "metrics/advantage_var": 365.2272033691406,
      "regularization/advantage_var": 365.2272033691406,
      "regularization/kl": 0.6081032752990723,
      "rewards/chosen": 0.6468030216223164,
      "rewards/margins": 0.874947465546522,
      "rewards/rejected": -0.22814444392420566,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 11.218920707702637,
      "kl": 13.033807754516602,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -35571392.640915595,
      "logits/rejected": -36281053.19104991,
      "logps/chosen": -480.8800965665236,
      "logps/rejected": -388.6537489242685,
      "loss": 0.4603,
      "loss/base_kto": 3.087879180908203,
      "loss/total_with_kl": 3.6822800636291504,
      "metrics/advantage_var": 356.99755859375,
      "regularization/advantage_var": 356.99755859375,
      "regularization/kl": 0.5944010019302368,
      "rewards/chosen": 0.6077811005118919,
      "rewards/margins": 0.9137831149872039,
      "rewards/rejected": -0.30600201447531195,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 10.710734367370605,
      "kl": 19.303434371948242,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -35323335.48051948,
      "logits/rejected": -37117396.81927711,
      "logps/chosen": -529.1298701298701,
      "logps/rejected": -361.406438253012,
      "loss": 0.4617,
      "loss/base_kto": 3.061537027359009,
      "loss/total_with_kl": 3.693446397781372,
      "metrics/advantage_var": 379.52490234375,
      "regularization/advantage_var": 379.52490234375,
      "regularization/kl": 0.6319090127944946,
      "rewards/chosen": 0.6770719057553775,
      "rewards/margins": 0.9370477757269631,
      "rewards/rejected": -0.25997586997158556,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 9.4447603225708,
      "kl": 17.10456657409668,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -33333655.01577287,
      "logits/rejected": -35455067.9380805,
      "logps/chosen": -471.5881309148265,
      "logps/rejected": -356.6826141640867,
      "loss": 0.4577,
      "loss/base_kto": 3.0967190265655518,
      "loss/total_with_kl": 3.6612699031829834,
      "metrics/advantage_var": 339.0694885253906,
      "regularization/advantage_var": 339.0694885253906,
      "regularization/kl": 0.5645507574081421,
      "rewards/chosen": 0.659497185836455,
      "rewards/margins": 0.9093857447113824,
      "rewards/rejected": -0.24988855887492745,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 14.468193054199219,
      "kl": 17.700286865234375,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -35349790.848200314,
      "logits/rejected": -34952244.7176287,
      "logps/chosen": -459.523082942097,
      "logps/rejected": -382.5817570202808,
      "loss": 0.4593,
      "loss/base_kto": 3.1270058155059814,
      "loss/total_with_kl": 3.674060583114624,
      "metrics/advantage_var": 328.5613098144531,
      "regularization/advantage_var": 328.5613098144531,
      "regularization/kl": 0.5470545887947083,
      "rewards/chosen": 0.6088742538237236,
      "rewards/margins": 0.8522961208253568,
      "rewards/rejected": -0.24342186700163318,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 12.740854263305664,
      "kl": 16.302776336669922,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -35310003.54751131,
      "logits/rejected": -36473910.76823339,
      "logps/chosen": -472.34455128205127,
      "logps/rejected": -368.706619732577,
      "loss": 0.4617,
      "loss/base_kto": 3.1101086139678955,
      "loss/total_with_kl": 3.693207263946533,
      "metrics/advantage_var": 350.2093811035156,
      "regularization/advantage_var": 350.2093811035156,
      "regularization/kl": 0.5830985903739929,
      "rewards/chosen": 0.6291640613952912,
      "rewards/margins": 0.8746409804118279,
      "rewards/rejected": -0.24547691901653668,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 16.703655242919922,
      "kl": 18.12259864807129,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -34409088.396284826,
      "logits/rejected": -35824610.92744479,
      "logps/chosen": -490.77699303405575,
      "logps/rejected": -351.43954554416405,
      "loss": 0.4692,
      "loss/base_kto": 3.1362993717193604,
      "loss/total_with_kl": 3.7539360523223877,
      "metrics/advantage_var": 370.952880859375,
      "regularization/advantage_var": 370.952880859375,
      "regularization/kl": 0.6176366209983826,
      "rewards/chosen": 0.6702606862543538,
      "rewards/margins": 0.8669946122512484,
      "rewards/rejected": -0.1967339259968947,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 11.437090873718262,
      "kl": 17.222028732299805,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -35721556.32047478,
      "logits/rejected": -35439119.20792079,
      "logps/chosen": -467.7715133531157,
      "logps/rejected": -390.11785787953795,
      "loss": 0.4737,
      "loss/base_kto": 3.1542561054229736,
      "loss/total_with_kl": 3.7895684242248535,
      "metrics/advantage_var": 381.5690612792969,
      "regularization/advantage_var": 381.5690612792969,
      "regularization/kl": 0.6353124976158142,
      "rewards/chosen": 0.6299233818620178,
      "rewards/margins": 0.8236679909268061,
      "rewards/rejected": -0.19374460906478833,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 17.85501480102539,
      "kl": 23.396320343017578,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -35332604.69677419,
      "logits/rejected": -36375266.52121212,
      "logps/chosen": -479.29395161290324,
      "logps/rejected": -393.30485321969695,
      "loss": 0.4714,
      "loss/base_kto": 3.076240301132202,
      "loss/total_with_kl": 3.7709243297576904,
      "metrics/advantage_var": 417.2275390625,
      "regularization/advantage_var": 417.2275390625,
      "regularization/kl": 0.694683849811554,
      "rewards/chosen": 0.6907609508883569,
      "rewards/margins": 0.920711410686534,
      "rewards/rejected": -0.22995045979817708,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 11.629434585571289,
      "kl": 20.134891510009766,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -35256735.306501545,
      "logits/rejected": -36730201.64037855,
      "logps/chosen": -467.421536377709,
      "logps/rejected": -382.1418079652997,
      "loss": 0.4755,
      "loss/base_kto": 3.1732566356658936,
      "loss/total_with_kl": 3.8041954040527344,
      "metrics/advantage_var": 378.9418640136719,
      "regularization/advantage_var": 378.9418640136719,
      "regularization/kl": 0.6309382319450378,
      "rewards/chosen": 0.6376219002455011,
      "rewards/margins": 0.8101594301276975,
      "rewards/rejected": -0.17253752988219637,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 10.874473571777344,
      "kl": 19.402769088745117,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -33273703.354037266,
      "logits/rejected": -35857182.591194965,
      "logps/chosen": -501.71797360248445,
      "logps/rejected": -352.53387873427675,
      "loss": 0.4593,
      "loss/base_kto": 3.080603837966919,
      "loss/total_with_kl": 3.6747825145721436,
      "metrics/advantage_var": 356.86395263671875,
      "regularization/advantage_var": 356.86395263671875,
      "regularization/kl": 0.5941784381866455,
      "rewards/chosen": 0.723855249629998,
      "rewards/margins": 0.8998491630046049,
      "rewards/rejected": -0.17599391337460693,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 7.630674362182617,
      "kl": 21.488651275634766,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -34430249.71246006,
      "logits/rejected": -37031719.92660551,
      "logps/chosen": -489.7289337060703,
      "logps/rejected": -377.8819762996942,
      "loss": 0.4673,
      "loss/base_kto": 3.0745832920074463,
      "loss/total_with_kl": 3.7381234169006348,
      "metrics/advantage_var": 398.52264404296875,
      "regularization/advantage_var": 398.52264404296875,
      "regularization/kl": 0.6635401844978333,
      "rewards/chosen": 0.6944223227211461,
      "rewards/margins": 0.9343317835063383,
      "rewards/rejected": -0.2399094607851921,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 13.328434944152832,
      "kl": 20.705591201782227,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -34441051.25552051,
      "logits/rejected": -35800685.374613,
      "logps/chosen": -487.750542192429,
      "logps/rejected": -370.4547697368421,
      "loss": 0.4644,
      "loss/base_kto": 3.0393569469451904,
      "loss/total_with_kl": 3.7152962684631348,
      "metrics/advantage_var": 405.9695739746094,
      "regularization/advantage_var": 405.9695739746094,
      "regularization/kl": 0.6759393811225891,
      "rewards/chosen": 0.7298758233007197,
      "rewards/margins": 0.9655283099069987,
      "rewards/rejected": -0.23565248660627902,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 15.256182670593262,
      "kl": 21.343603134155273,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -34241160.47819063,
      "logits/rejected": -35653142.46293495,
      "logps/chosen": -496.66892164781905,
      "logps/rejected": -369.8054084720121,
      "loss": 0.4604,
      "loss/base_kto": 3.093393564224243,
      "loss/total_with_kl": 3.68314790725708,
      "metrics/advantage_var": 354.2069091796875,
      "regularization/advantage_var": 354.2069091796875,
      "regularization/kl": 0.5897545218467712,
      "rewards/chosen": 0.7125130747362177,
      "rewards/margins": 0.8750518195948599,
      "rewards/rejected": -0.1625387448586422,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 15.188468933105469,
      "kl": 19.63766860961914,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -35101333.403278686,
      "logits/rejected": -37411250.053731345,
      "logps/chosen": -490.96331967213115,
      "logps/rejected": -378.2164878731343,
      "loss": 0.4626,
      "loss/base_kto": 3.0533533096313477,
      "loss/total_with_kl": 3.7009949684143066,
      "metrics/advantage_var": 388.9740295410156,
      "regularization/advantage_var": 388.9740295410156,
      "regularization/kl": 0.6476417183876038,
      "rewards/chosen": 0.6866738241226946,
      "rewards/margins": 0.9182481303260528,
      "rewards/rejected": -0.2315743062033582,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 8.538738250732422,
      "kl": 16.847759246826172,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -34908842.666666664,
      "logits/rejected": -36901236.92867982,
      "logps/chosen": -468.32573470209337,
      "logps/rejected": -370.87239188163886,
      "loss": 0.4619,
      "loss/base_kto": 3.1275436878204346,
      "loss/total_with_kl": 3.695369005203247,
      "metrics/advantage_var": 341.0362854003906,
      "regularization/advantage_var": 341.0362854003906,
      "regularization/kl": 0.567825436592102,
      "rewards/chosen": 0.6539842295377919,
      "rewards/margins": 0.8895764594938096,
      "rewards/rejected": -0.23559222995601764,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 12.581542015075684,
      "kl": 18.04552459716797,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -35341651.16957211,
      "logits/rejected": -36038750.66872111,
      "logps/chosen": -466.6060320919176,
      "logps/rejected": -384.8725202234206,
      "loss": 0.474,
      "loss/base_kto": 3.106107473373413,
      "loss/total_with_kl": 3.7923743724823,
      "metrics/advantage_var": 412.1723327636719,
      "regularization/advantage_var": 412.1723327636719,
      "regularization/kl": 0.6862668991088867,
      "rewards/chosen": 0.6502018609628814,
      "rewards/margins": 0.870076614753268,
      "rewards/rejected": -0.21987475379038665,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 12.283934593200684,
      "kl": 18.97503089904785,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -35337595.01298701,
      "logits/rejected": -37089347.855421685,
      "logps/chosen": -492.56960227272725,
      "logps/rejected": -400.42469879518075,
      "loss": 0.4606,
      "loss/base_kto": 3.1100635528564453,
      "loss/total_with_kl": 3.685138702392578,
      "metrics/advantage_var": 345.39031982421875,
      "regularization/advantage_var": 345.39031982421875,
      "regularization/kl": 0.5750748515129089,
      "rewards/chosen": 0.641826580097149,
      "rewards/margins": 0.8806340744789731,
      "rewards/rejected": -0.23880749438182416,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 22.152240753173828,
      "kl": 18.58150291442871,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -34276528.551724136,
      "logits/rejected": -36221310.80373832,
      "logps/chosen": -462.7879604231975,
      "logps/rejected": -364.3088492990654,
      "loss": 0.4625,
      "loss/base_kto": 3.098907947540283,
      "loss/total_with_kl": 3.6998727321624756,
      "metrics/advantage_var": 360.9399719238281,
      "regularization/advantage_var": 360.9399719238281,
      "regularization/kl": 0.6009650826454163,
      "rewards/chosen": 0.641452083767021,
      "rewards/margins": 0.8731421548492104,
      "rewards/rejected": -0.23169007108218945,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 11.47033405303955,
      "kl": 18.98607635498047,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -35194479.925581396,
      "logits/rejected": -36467191.130708665,
      "logps/chosen": -492.0019864341085,
      "logps/rejected": -392.89675196850396,
      "loss": 0.464,
      "loss/base_kto": 3.0971181392669678,
      "loss/total_with_kl": 3.711954116821289,
      "metrics/advantage_var": 369.27099609375,
      "regularization/advantage_var": 369.27099609375,
      "regularization/kl": 0.6148362159729004,
      "rewards/chosen": 0.6864966104196948,
      "rewards/margins": 0.9122782929522981,
      "rewards/rejected": -0.22578168253260333,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 8.47675895690918,
      "kl": 15.725268363952637,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -34352238.32473445,
      "logits/rejected": -34257004.83091787,
      "logps/chosen": -468.64577010622156,
      "logps/rejected": -342.00367351046697,
      "loss": 0.452,
      "loss/base_kto": 3.081899404525757,
      "loss/total_with_kl": 3.6163601875305176,
      "metrics/advantage_var": 320.9973449707031,
      "regularization/advantage_var": 320.9973449707031,
      "regularization/kl": 0.5344605445861816,
      "rewards/chosen": 0.6775474721998292,
      "rewards/margins": 0.9099925425249098,
      "rewards/rejected": -0.23244507032508052,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.975174191323546e+17,
      "train_loss": 0.5005160395139652,
      "train_runtime": 11075.1717,
      "train_samples_per_second": 13.383,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.975174191323546e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}