{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.389772415161133,
      "kl": 47.63533401489258,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -38406302.7123696,
      "logits/rejected": -37807556.30870279,
      "logps/chosen": -457.81594634873323,
      "logps/rejected": -390.4787048440066,
      "loss": 0.4565,
      "loss/base_kto": 3.651801824569702,
      "metrics/advantage_var": 551.3017578125,
      "rewards/chosen": 0.8933472953148286,
      "rewards/margins": 0.2512501854747988,
      "rewards/rejected": 0.6420971098400298,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 13.195425033569336,
      "kl": 12.069128036499023,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37387905.259016395,
      "logits/rejected": -38080683.1761194,
      "logps/chosen": -481.9829918032787,
      "logps/rejected": -372.9957555970149,
      "loss": 0.4578,
      "loss/base_kto": 3.6623947620391846,
      "metrics/advantage_var": 629.2352905273438,
      "rewards/chosen": 0.25825915727459015,
      "rewards/margins": 0.3732206049191797,
      "rewards/rejected": -0.11496144764458956,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.603527069091797,
      "kl": 3.2372887134552,
      "learning_rate": 5.9e-07,
      "logits/chosen": -37479492.266666666,
      "logits/rejected": -38547525.316923074,
      "logps/chosen": -482.2787698412698,
      "logps/rejected": -351.85790865384615,
      "loss": 0.4407,
      "loss/base_kto": 3.5256569385528564,
      "metrics/advantage_var": 952.7435913085938,
      "rewards/chosen": -0.06462832859584264,
      "rewards/margins": 0.5826495034354073,
      "rewards/rejected": -0.64727783203125,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 10.463303565979004,
      "kl": 8.231558799743652,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37918667.98730159,
      "logits/rejected": -39891054.276923075,
      "logps/chosen": -489.1375992063492,
      "logps/rejected": -385.59139423076925,
      "loss": 0.4274,
      "loss/base_kto": 3.4195592403411865,
      "metrics/advantage_var": 1588.9461669921875,
      "rewards/chosen": 0.16997208368210565,
      "rewards/margins": 0.6811758847638365,
      "rewards/rejected": -0.5112038010817308,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 8.33227825164795,
      "kl": 12.142549514770508,
      "learning_rate": 9.9e-07,
      "logits/chosen": -38111445.53554502,
      "logits/rejected": -37718256.56877898,
      "logps/chosen": -492.62213665086887,
      "logps/rejected": -366.55549652241115,
      "loss": 0.4183,
      "loss/base_kto": 3.346160888671875,
      "metrics/advantage_var": 2415.298095703125,
      "rewards/chosen": 0.5311772013539199,
      "rewards/margins": 0.9226113652685016,
      "rewards/rejected": -0.39143416391458175,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 9.39903450012207,
      "kl": 24.029155731201172,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -39135715.85141159,
      "logits/rejected": -40351362.741350904,
      "logps/chosen": -472.12286404160477,
      "logps/rejected": -378.63529654036245,
      "loss": 0.3813,
      "loss/base_kto": 3.050102949142456,
      "metrics/advantage_var": 3768.57275390625,
      "rewards/chosen": 1.3469364341973904,
      "rewards/margins": 1.407133280871849,
      "rewards/rejected": -0.060196846674458786,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 9.532487869262695,
      "kl": 19.03123664855957,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -39336960.0,
      "logits/rejected": -40051772.42236025,
      "logps/chosen": -481.5810239779874,
      "logps/rejected": -411.50058229813664,
      "loss": 0.399,
      "loss/base_kto": 3.191612720489502,
      "metrics/advantage_var": 4326.13818359375,
      "rewards/chosen": 0.8986968034468357,
      "rewards/margins": 1.398232784619196,
      "rewards/rejected": -0.49953598117236025,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 8.447214126586914,
      "kl": 20.36687469482422,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -39655028.220858894,
      "logits/rejected": -40515189.401273884,
      "logps/chosen": -449.6458013803681,
      "logps/rejected": -388.88062300955414,
      "loss": 0.3818,
      "loss/base_kto": 3.05460262298584,
      "metrics/advantage_var": 4241.35400390625,
      "rewards/chosen": 1.2029486346098544,
      "rewards/margins": 1.5763467533275097,
      "rewards/rejected": -0.37339811871765527,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 7.352090358734131,
      "kl": 35.560909271240234,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -39940968.540192924,
      "logits/rejected": -40337252.3768997,
      "logps/chosen": -485.1162580385852,
      "logps/rejected": -371.94633358662617,
      "loss": 0.377,
      "loss/base_kto": 3.015608072280884,
      "metrics/advantage_var": 4760.81591796875,
      "rewards/chosen": 1.484773102297277,
      "rewards/margins": 1.578296006064315,
      "rewards/rejected": -0.0935229037670379,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 9.246931076049805,
      "kl": 13.78105354309082,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -42092757.59509202,
      "logits/rejected": -40996766.1656051,
      "logps/chosen": -470.5297162576687,
      "logps/rejected": -393.69737758757964,
      "loss": 0.3788,
      "loss/base_kto": 3.0301249027252197,
      "metrics/advantage_var": 4451.1181640625,
      "rewards/chosen": 0.716448543993242,
      "rewards/margins": 1.7276718207025374,
      "rewards/rejected": -1.0112232767092955,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 8.291376113891602,
      "kl": 29.377546310424805,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -43047583.744,
      "logits/rejected": -41190317.14198473,
      "logps/chosen": -476.7729,
      "logps/rejected": -359.94021946564885,
      "loss": 0.3642,
      "loss/base_kto": 2.9133713245391846,
      "metrics/advantage_var": 5095.07861328125,
      "rewards/chosen": 1.3700134765625,
      "rewards/margins": 1.8142961131321567,
      "rewards/rejected": -0.4442826365696565,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 6.037034511566162,
      "kl": 46.892433166503906,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -41527751.81452859,
      "logits/rejected": -42193924.85308057,
      "logps/chosen": -450.4008887171561,
      "logps/rejected": -365.4776856240126,
      "loss": 0.3526,
      "loss/base_kto": 2.8206701278686523,
      "metrics/advantage_var": 5228.74072265625,
      "rewards/chosen": 1.7855769869348919,
      "rewards/margins": 1.924404125537343,
      "rewards/rejected": -0.13882713860245113,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 8.512300491333008,
      "kl": 14.671371459960938,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -44227913.432646595,
      "logits/rejected": -45005872.91217257,
      "logps/chosen": -486.1079635499208,
      "logps/rejected": -397.2251540832049,
      "loss": 0.3561,
      "loss/base_kto": 2.8489062786102295,
      "metrics/advantage_var": 6176.60205078125,
      "rewards/chosen": 1.0215558567440075,
      "rewards/margins": 2.2437295622344253,
      "rewards/rejected": -1.222173705490418,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 9.194314002990723,
      "kl": 30.27271842956543,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -43314384.7133758,
      "logits/rejected": -43476948.02453988,
      "logps/chosen": -466.47462181528664,
      "logps/rejected": -374.32992235429447,
      "loss": 0.3479,
      "loss/base_kto": 2.782944440841675,
      "metrics/advantage_var": 6720.2744140625,
      "rewards/chosen": 1.835502089968153,
      "rewards/margins": 2.3519556515754667,
      "rewards/rejected": -0.516453561607314,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 8.246805191040039,
      "kl": 23.512361526489258,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -44900505.84037559,
      "logits/rejected": -44444689.5725429,
      "logps/chosen": -470.71112089201876,
      "logps/rejected": -381.8404592433697,
      "loss": 0.3549,
      "loss/base_kto": 2.8391671180725098,
      "metrics/advantage_var": 7269.17431640625,
      "rewards/chosen": 1.6048429247359155,
      "rewards/margins": 2.248600380356406,
      "rewards/rejected": -0.6437574556204905,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 7.926791191101074,
      "kl": 28.59177017211914,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -42740652.972972974,
      "logits/rejected": -43164114.97068404,
      "logps/chosen": -468.4436936936937,
      "logps/rejected": -354.4130191368078,
      "loss": 0.3396,
      "loss/base_kto": 2.716761350631714,
      "metrics/advantage_var": 6583.8837890625,
      "rewards/chosen": 1.8008454010651276,
      "rewards/margins": 2.4791936068303455,
      "rewards/rejected": -0.6783482057652178,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 8.163418769836426,
      "kl": 27.2202091217041,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -43318928.96385542,
      "logits/rejected": -43874051.32467532,
      "logps/chosen": -462.8449736445783,
      "logps/rejected": -392.1993709415584,
      "loss": 0.3595,
      "loss/base_kto": 2.8757095336914062,
      "metrics/advantage_var": 6862.53662109375,
      "rewards/chosen": 1.682490337325866,
      "rewards/margins": 2.2010155476197224,
      "rewards/rejected": -0.5185252102938566,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 8.326271057128906,
      "kl": 32.66901779174805,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -42703674.385964915,
      "logits/rejected": -42824407.620214395,
      "logps/chosen": -483.09006180223287,
      "logps/rejected": -362.03189605666154,
      "loss": 0.3395,
      "loss/base_kto": 2.716346263885498,
      "metrics/advantage_var": 7354.09375,
      "rewards/chosen": 1.8718207205691786,
      "rewards/margins": 2.6472364886886752,
      "rewards/rejected": -0.7754157681194965,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 9.613969802856445,
      "kl": 9.25661849975586,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -44944765.7472,
      "logits/rejected": -45180043.1389313,
      "logps/chosen": -487.5415,
      "logps/rejected": -393.5746183206107,
      "loss": 0.3328,
      "loss/base_kto": 2.6622776985168457,
      "metrics/advantage_var": 8602.853515625,
      "rewards/chosen": 0.8969716796875,
      "rewards/margins": 2.875173835579676,
      "rewards/rejected": -1.9782021558921756,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 8.464766502380371,
      "kl": 28.85038185119629,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -44117981.34153846,
      "logits/rejected": -45089457.168253966,
      "logps/chosen": -472.04365384615386,
      "logps/rejected": -391.177058531746,
      "loss": 0.3282,
      "loss/base_kto": 2.6255855560302734,
      "metrics/advantage_var": 7806.515625,
      "rewards/chosen": 1.9826173753004808,
      "rewards/margins": 2.7731585730621378,
      "rewards/rejected": -0.7905411977616568,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 8.245142936706543,
      "kl": 14.65202522277832,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44670906.469135806,
      "logits/rejected": -46875116.55696203,
      "logps/chosen": -480.6135223765432,
      "logps/rejected": -370.848397943038,
      "loss": 0.3341,
      "loss/base_kto": 2.672945499420166,
      "metrics/advantage_var": 8595.6455078125,
      "rewards/chosen": 1.4219384840977045,
      "rewards/margins": 2.9975295593671865,
      "rewards/rejected": -1.5755910752694817,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 8.22985553741455,
      "kl": 30.664514541625977,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -43924364.758842446,
      "logits/rejected": -44938856.26747721,
      "logps/chosen": -461.2988344051447,
      "logps/rejected": -384.89855623100306,
      "loss": 0.3369,
      "loss/base_kto": 2.6951730251312256,
      "metrics/advantage_var": 9284.2763671875,
      "rewards/chosen": 1.7856076353999195,
      "rewards/margins": 2.958200957968309,
      "rewards/rejected": -1.172593322568389,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 5.799438953399658,
      "kl": 50.48659896850586,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -44397791.41818182,
      "logits/rejected": -44613483.35483871,
      "logps/chosen": -436.6492424242424,
      "logps/rejected": -382.8592741935484,
      "loss": 0.329,
      "loss/base_kto": 2.632227659225464,
      "metrics/advantage_var": 8154.85498046875,
      "rewards/chosen": 2.547796445904356,
      "rewards/margins": 2.721452651112427,
      "rewards/rejected": -0.17365620520807082,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 8.989084243774414,
      "kl": 29.10886573791504,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -48569896.66126418,
      "logits/rejected": -47298353.03770739,
      "logps/chosen": -482.85595623987035,
      "logps/rejected": -374.7420814479638,
      "loss": 0.3312,
      "loss/base_kto": 2.6498098373413086,
      "metrics/advantage_var": 9052.373046875,
      "rewards/chosen": 1.925774127583063,
      "rewards/margins": 2.9796671579021616,
      "rewards/rejected": -1.0538930303190988,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 7.970582485198975,
      "kl": 34.053043365478516,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -46713117.402756505,
      "logits/rejected": -46109504.91866029,
      "logps/chosen": -475.3674866003063,
      "logps/rejected": -390.2767145135566,
      "loss": 0.3211,
      "loss/base_kto": 2.569037437438965,
      "metrics/advantage_var": 9849.2734375,
      "rewards/chosen": 2.1553962555632658,
      "rewards/margins": 3.101857520920373,
      "rewards/rejected": -0.9464612653571073,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.372785568237305,
      "kl": 26.26869773864746,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -48282539.70731708,
      "logits/rejected": -47864142.76923077,
      "logps/chosen": -460.3074504573171,
      "logps/rejected": -377.1211939102564,
      "loss": 0.3256,
      "loss/base_kto": 2.6048405170440674,
      "metrics/advantage_var": 9842.2724609375,
      "rewards/chosen": 1.9590448519078696,
      "rewards/margins": 3.0523682905033724,
      "rewards/rejected": -1.0933234385955029,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 8.239922523498535,
      "kl": 52.59576416015625,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -48873639.44654088,
      "logits/rejected": -49180035.97515528,
      "logps/chosen": -458.720322327044,
      "logps/rejected": -381.3061432453416,
      "loss": 0.3288,
      "loss/base_kto": 2.630648374557495,
      "metrics/advantage_var": 9683.5693359375,
      "rewards/chosen": 2.6747438442782037,
      "rewards/margins": 2.926806126485053,
      "rewards/rejected": -0.2520622822068493,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 9.227850914001465,
      "kl": 21.218448638916016,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -46331279.295597486,
      "logits/rejected": -47721618.28571428,
      "logps/chosen": -454.13384433962267,
      "logps/rejected": -371.9747185559006,
      "loss": 0.3051,
      "loss/base_kto": 2.4406096935272217,
      "metrics/advantage_var": 10768.3876953125,
      "rewards/chosen": 2.211166285868711,
      "rewards/margins": 3.502381696752834,
      "rewards/rejected": -1.2912154108841227,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 7.125072479248047,
      "kl": 34.98481750488281,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -46227072.38670695,
      "logits/rejected": -46066644.919093855,
      "logps/chosen": -459.86508685800607,
      "logps/rejected": -372.3851132686084,
      "loss": 0.3282,
      "loss/base_kto": 2.6253113746643066,
      "metrics/advantage_var": 11001.2177734375,
      "rewards/chosen": 2.5339547240842144,
      "rewards/margins": 3.2950123092727006,
      "rewards/rejected": -0.7610575851884861,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.275957107543945,
      "kl": 33.37077331542969,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -47322094.03508772,
      "logits/rejected": -47008286.57886677,
      "logps/chosen": -459.7950558213716,
      "logps/rejected": -382.4612366003063,
      "loss": 0.3233,
      "loss/base_kto": 2.5865111351013184,
      "metrics/advantage_var": 11704.7568359375,
      "rewards/chosen": 2.4234196664423844,
      "rewards/margins": 3.510266992186546,
      "rewards/rejected": -1.0868473257441615,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 7.228305816650391,
      "kl": 12.117892265319824,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -49392219.51690821,
      "logits/rejected": -47948938.29438543,
      "logps/chosen": -495.15841384863126,
      "logps/rejected": -411.4466995447648,
      "loss": 0.3299,
      "loss/base_kto": 2.6388490200042725,
      "metrics/advantage_var": 12980.6669921875,
      "rewards/chosen": 1.1754036379705113,
      "rewards/margins": 3.4501155594424384,
      "rewards/rejected": -2.274711921471927,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 9.235511779785156,
      "kl": 18.60725212097168,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -47427814.482315116,
      "logits/rejected": -47068841.62917933,
      "logps/chosen": -464.88102893890675,
      "logps/rejected": -397.7166365881459,
      "loss": 0.3285,
      "loss/base_kto": 2.628096342086792,
      "metrics/advantage_var": 11702.447265625,
      "rewards/chosen": 1.6093233850607918,
      "rewards/margins": 3.174098382389344,
      "rewards/rejected": -1.5647749973285525,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 7.23322057723999,
      "kl": 22.347009658813477,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -48391976.585365854,
      "logits/rejected": -47666609.23076923,
      "logps/chosen": -483.9475038109756,
      "logps/rejected": -394.9490184294872,
      "loss": 0.3257,
      "loss/base_kto": 2.6056020259857178,
      "metrics/advantage_var": 13653.556640625,
      "rewards/chosen": 2.2879113918397485,
      "rewards/margins": 3.574918845953831,
      "rewards/rejected": -1.2870074541140826,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 7.010107040405273,
      "kl": 42.226776123046875,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -46929463.439490445,
      "logits/rejected": -45645820.858895704,
      "logps/chosen": -449.6611265923567,
      "logps/rejected": -379.30272239263803,
      "loss": 0.3121,
      "loss/base_kto": 2.4970703125,
      "metrics/advantage_var": 13127.6640625,
      "rewards/chosen": 2.2514016704194866,
      "rewards/margins": 3.6759871181404415,
      "rewards/rejected": -1.4245854477209547,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 6.621373176574707,
      "kl": 38.65568542480469,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -48262849.35347432,
      "logits/rejected": -47633981.307443365,
      "logps/chosen": -451.99679003021146,
      "logps/rejected": -375.6196652508091,
      "loss": 0.325,
      "loss/base_kto": 2.600301742553711,
      "metrics/advantage_var": 11315.232421875,
      "rewards/chosen": 2.659457572637368,
      "rewards/margins": 3.1546796951754192,
      "rewards/rejected": -0.49522212253805115,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 7.346182823181152,
      "kl": 17.64263916015625,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -47635133.896713614,
      "logits/rejected": -46112748.8299532,
      "logps/chosen": -480.8606220657277,
      "logps/rejected": -394.9130752730109,
      "loss": 0.3243,
      "loss/base_kto": 2.5942459106445312,
      "metrics/advantage_var": 12418.57421875,
      "rewards/chosen": 1.482877489546655,
      "rewards/margins": 3.6101688312003213,
      "rewards/rejected": -2.1272913416536663,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 8.256329536437988,
      "kl": 40.221649169921875,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -48079565.4522293,
      "logits/rejected": -48238667.38650307,
      "logps/chosen": -452.71616242038215,
      "logps/rejected": -368.75603911042947,
      "loss": 0.3152,
      "loss/base_kto": 2.521874189376831,
      "metrics/advantage_var": 11227.6708984375,
      "rewards/chosen": 2.4987431422919983,
      "rewards/margins": 3.437864692869548,
      "rewards/rejected": -0.9391215505775499,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 6.824674129486084,
      "kl": 28.154876708984375,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -47101648.47425897,
      "logits/rejected": -47173353.16431925,
      "logps/chosen": -485.76028666146647,
      "logps/rejected": -384.87404636150234,
      "loss": 0.3173,
      "loss/base_kto": 2.5386312007904053,
      "metrics/advantage_var": 13187.3505859375,
      "rewards/chosen": 2.403803832817375,
      "rewards/margins": 3.792621275232281,
      "rewards/rejected": -1.388817442414906,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 6.028614044189453,
      "kl": 24.844501495361328,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -48460178.570982836,
      "logits/rejected": -48257503.0455259,
      "logps/chosen": -469.759360374415,
      "logps/rejected": -379.34561420722133,
      "loss": 0.2968,
      "loss/base_kto": 2.374429702758789,
      "metrics/advantage_var": 13482.1533203125,
      "rewards/chosen": 2.166554943447738,
      "rewards/margins": 4.012237946967655,
      "rewards/rejected": -1.8456830035199177,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 9.08834457397461,
      "kl": 20.602874755859375,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -47292737.37846154,
      "logits/rejected": -46673464.88888889,
      "logps/chosen": -463.24221153846156,
      "logps/rejected": -403.7625248015873,
      "loss": 0.2446,
      "loss/base_kto": 1.9567406177520752,
      "metrics/advantage_var": 14699.2001953125,
      "rewards/chosen": 2.7593828876201925,
      "rewards/margins": 5.1381240520547165,
      "rewards/rejected": -2.378741164434524,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 8.160238265991211,
      "kl": 19.674734115600586,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -48202742.518518515,
      "logits/rejected": -48369903.79746836,
      "logps/chosen": -475.44140625,
      "logps/rejected": -422.9123318829114,
      "loss": 0.2557,
      "loss/base_kto": 2.045527696609497,
      "metrics/advantage_var": 16601.380859375,
      "rewards/chosen": 2.578687126253858,
      "rewards/margins": 5.213243570639241,
      "rewards/rejected": -2.6345564443853835,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 6.527251243591309,
      "kl": 32.78919219970703,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -48127182.11916264,
      "logits/rejected": -48193337.1047041,
      "logps/chosen": -468.6815619967794,
      "logps/rejected": -413.415686646434,
      "loss": 0.2525,
      "loss/base_kto": 2.020085573196411,
      "metrics/advantage_var": 14671.9345703125,
      "rewards/chosen": 2.7362296431536834,
      "rewards/margins": 4.915611133811877,
      "rewards/rejected": -2.1793814906581943,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 7.973196029663086,
      "kl": 8.747920036315918,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -48637005.0048,
      "logits/rejected": -48764981.154198475,
      "logps/chosen": -471.8269,
      "logps/rejected": -418.7538167938931,
      "loss": 0.2555,
      "loss/base_kto": 2.0437705516815186,
      "metrics/advantage_var": 17556.083984375,
      "rewards/chosen": 1.8711177734375,
      "rewards/margins": 5.295105428494752,
      "rewards/rejected": -3.423987655057252,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.407348155975342,
      "kl": 25.11842918395996,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -49298159.147335425,
      "logits/rejected": -48831242.367601246,
      "logps/chosen": -449.38700039184954,
      "logps/rejected": -381.8530957943925,
      "loss": 0.2617,
      "loss/base_kto": 2.0939595699310303,
      "metrics/advantage_var": 15131.697265625,
      "rewards/chosen": 2.6031605113636362,
      "rewards/margins": 4.881092176326643,
      "rewards/rejected": -2.277931664963006,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 6.892596244812012,
      "kl": 11.93945598602295,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -49092113.32923077,
      "logits/rejected": -49441834.26031746,
      "logps/chosen": -456.97432692307694,
      "logps/rejected": -407.94742063492066,
      "loss": 0.2669,
      "loss/base_kto": 2.135260820388794,
      "metrics/advantage_var": 16834.97265625,
      "rewards/chosen": 2.1193592247596156,
      "rewards/margins": 4.897466011093941,
      "rewards/rejected": -2.7781067863343254,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 8.031292915344238,
      "kl": 31.970178604125977,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -49624292.079877116,
      "logits/rejected": -49447781.3418124,
      "logps/chosen": -460.8429339477727,
      "logps/rejected": -372.7867647058824,
      "loss": 0.2352,
      "loss/base_kto": 1.8812607526779175,
      "metrics/advantage_var": 14783.8779296875,
      "rewards/chosen": 3.347041960685484,
      "rewards/margins": 5.363860882088504,
      "rewards/rejected": -2.0168189214030208,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 7.753678321838379,
      "kl": 25.85993766784668,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -47928455.12241653,
      "logits/rejected": -47511127.29953917,
      "logps/chosen": -452.45901232114466,
      "logps/rejected": -363.9561491935484,
      "loss": 0.2634,
      "loss/base_kto": 2.107530355453491,
      "metrics/advantage_var": 13522.9609375,
      "rewards/chosen": 2.5884316934121623,
      "rewards/margins": 4.535507556267385,
      "rewards/rejected": -1.9470758628552227,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 8.16309928894043,
      "kl": 20.483793258666992,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -48593606.4,
      "logits/rejected": -49353206.4,
      "logps/chosen": -417.97294921875,
      "logps/rejected": -397.7453857421875,
      "loss": 0.268,
      "loss/base_kto": 2.1443779468536377,
      "metrics/advantage_var": 14067.8466796875,
      "rewards/chosen": 2.1633033752441406,
      "rewards/margins": 4.678990745544434,
      "rewards/rejected": -2.515687370300293,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 6.558415412902832,
      "kl": 28.014001846313477,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -48475016.01284109,
      "logits/rejected": -50391460.82191781,
      "logps/chosen": -488.72893258426967,
      "logps/rejected": -408.69244672754945,
      "loss": 0.2429,
      "loss/base_kto": 1.943566918373108,
      "metrics/advantage_var": 15186.3173828125,
      "rewards/chosen": 2.8327777795194624,
      "rewards/margins": 5.087047267663964,
      "rewards/rejected": -2.2542694881445016,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 7.960604667663574,
      "kl": 13.513842582702637,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -51427026.63291139,
      "logits/rejected": -48594767.01234568,
      "logps/chosen": -449.3992286392405,
      "logps/rejected": -396.8494888117284,
      "loss": 0.2591,
      "loss/base_kto": 2.0725789070129395,
      "metrics/advantage_var": 14900.515625,
      "rewards/chosen": 2.4082622286639634,
      "rewards/margins": 4.899628579756267,
      "rewards/rejected": -2.4913663510923034,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 8.487668991088867,
      "kl": 21.949365615844727,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -49826210.27534039,
      "logits/rejected": -50941670.77221325,
      "logps/chosen": -448.42256051437215,
      "logps/rejected": -397.01726575121165,
      "loss": 0.244,
      "loss/base_kto": 1.9521198272705078,
      "metrics/advantage_var": 14752.8896484375,
      "rewards/chosen": 2.896393884148071,
      "rewards/margins": 5.071983450232482,
      "rewards/rejected": -2.1755895660844105,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.264450073242188,
      "kl": 27.853168487548828,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -49997271.798518516,
      "logits/rejected": -50615778.38016529,
      "logps/chosen": -460.35203703703706,
      "logps/rejected": -378.03768078512394,
      "loss": 0.2493,
      "loss/base_kto": 1.994248628616333,
      "metrics/advantage_var": 12929.7939453125,
      "rewards/chosen": 2.9150424985532406,
      "rewards/margins": 4.735268237835266,
      "rewards/rejected": -1.8202257392820247,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 9.001532554626465,
      "kl": 15.930495262145996,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -51689064.8861912,
      "logits/rejected": -51946744.99194847,
      "logps/chosen": -469.39344650986345,
      "logps/rejected": -409.47632347020937,
      "loss": 0.2511,
      "loss/base_kto": 2.008457660675049,
      "metrics/advantage_var": 15899.4736328125,
      "rewards/chosen": 2.12914446308327,
      "rewards/margins": 5.047912272375239,
      "rewards/rejected": -2.9187678092919684,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 8.886826515197754,
      "kl": 18.990217208862305,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -52011945.48694316,
      "logits/rejected": -52379332.17170111,
      "logps/chosen": -479.4702860983103,
      "logps/rejected": -386.3915192766296,
      "loss": 0.2724,
      "loss/base_kto": 2.179483652114868,
      "metrics/advantage_var": 13855.7021484375,
      "rewards/chosen": 2.3804954142065093,
      "rewards/margins": 4.486999530052595,
      "rewards/rejected": -2.106504115846085,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 6.141109943389893,
      "kl": 30.30926513671875,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -50506341.13580247,
      "logits/rejected": -50303163.94936709,
      "logps/chosen": -452.74508101851853,
      "logps/rejected": -372.39700356012656,
      "loss": 0.2546,
      "loss/base_kto": 2.036782741546631,
      "metrics/advantage_var": 14251.7470703125,
      "rewards/chosen": 3.120146009657118,
      "rewards/margins": 4.849291687440939,
      "rewards/rejected": -1.7291456777838212,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.584972381591797,
      "kl": 22.3951416015625,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -51018727.88697017,
      "logits/rejected": -50664933.72317263,
      "logps/chosen": -453.2104101255887,
      "logps/rejected": -396.54315707620526,
      "loss": 0.2477,
      "loss/base_kto": 1.9816032648086548,
      "metrics/advantage_var": 16047.587890625,
      "rewards/chosen": 2.28303947029533,
      "rewards/margins": 5.198438858538914,
      "rewards/rejected": -2.9153993882435847,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 7.433993339538574,
      "kl": 17.95124626159668,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -49613046.7856025,
      "logits/rejected": -50742749.653666146,
      "logps/chosen": -476.1836854460094,
      "logps/rejected": -415.5039976599064,
      "loss": 0.2548,
      "loss/base_kto": 2.0387260913848877,
      "metrics/advantage_var": 15328.6748046875,
      "rewards/chosen": 2.675205284441021,
      "rewards/margins": 5.106290594259663,
      "rewards/rejected": -2.4310853098186427,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 6.105855464935303,
      "kl": 17.84329605102539,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -47158237.65853658,
      "logits/rejected": -50338474.666666664,
      "logps/chosen": -455.8099275914634,
      "logps/rejected": -390.8770783253205,
      "loss": 0.2544,
      "loss/base_kto": 2.0349042415618896,
      "metrics/advantage_var": 13977.494140625,
      "rewards/chosen": 2.4411368021150914,
      "rewards/margins": 4.890623363425688,
      "rewards/rejected": -2.449486561310597,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 6.1178879737854,
      "kl": 20.046993255615234,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50292838.080996886,
      "logits/rejected": -49619740.08777429,
      "logps/chosen": -441.6804906542056,
      "logps/rejected": -386.70307601880876,
      "loss": 0.2609,
      "loss/base_kto": 2.087083101272583,
      "metrics/advantage_var": 13117.759765625,
      "rewards/chosen": 2.7567787051572235,
      "rewards/margins": 4.597649529510671,
      "rewards/rejected": -1.8408708243534482,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 6.218991756439209,
      "kl": 13.484720230102539,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -51252059.69529984,
      "logits/rejected": -50508739.76470588,
      "logps/chosen": -458.5921799027553,
      "logps/rejected": -377.68382352941177,
      "loss": 0.275,
      "loss/base_kto": 2.199876546859741,
      "metrics/advantage_var": 13742.1064453125,
      "rewards/chosen": 1.808982317412885,
      "rewards/margins": 4.709338512324744,
      "rewards/rejected": -2.900356194911859,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 6.822948455810547,
      "kl": 24.661109924316406,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -51494668.190476194,
      "logits/rejected": -51958806.05538461,
      "logps/chosen": -475.4358630952381,
      "logps/rejected": -402.63795673076925,
      "loss": 0.2566,
      "loss/base_kto": 2.0528342723846436,
      "metrics/advantage_var": 14411.15625,
      "rewards/chosen": 2.8241772848462303,
      "rewards/margins": 4.8616267665169035,
      "rewards/rejected": -2.037449481670673,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 5.703562259674072,
      "kl": 35.027462005615234,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -52037708.96732026,
      "logits/rejected": -52339742.65868264,
      "logps/chosen": -462.3002450980392,
      "logps/rejected": -412.0844872754491,
      "loss": 0.2549,
      "loss/base_kto": 2.03886342048645,
      "metrics/advantage_var": 14913.2177734375,
      "rewards/chosen": 2.8195912479575163,
      "rewards/margins": 4.789924872700219,
      "rewards/rejected": -1.9703336247427021,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 9.470081329345703,
      "kl": 24.76580238342285,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -50662217.028213166,
      "logits/rejected": -50396405.632398754,
      "logps/chosen": -473.09370101880876,
      "logps/rejected": -396.8532904984424,
      "loss": 0.232,
      "loss/base_kto": 1.8556301593780518,
      "metrics/advantage_var": 14833.1708984375,
      "rewards/chosen": 2.840141081137343,
      "rewards/margins": 5.319337409749103,
      "rewards/rejected": -2.47919632861176,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 7.3936872482299805,
      "kl": 16.468170166015625,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -51449756.332810044,
      "logits/rejected": -51321140.95178849,
      "logps/chosen": -477.38844191522765,
      "logps/rejected": -398.4009282659409,
      "loss": 0.2407,
      "loss/base_kto": 1.9254833459854126,
      "metrics/advantage_var": 16836.380859375,
      "rewards/chosen": 2.589753299156201,
      "rewards/margins": 5.541966473789269,
      "rewards/rejected": -2.9522131746330675,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 10.445863723754883,
      "kl": 30.038633346557617,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -53743021.82716049,
      "logits/rejected": -51698046.37974683,
      "logps/chosen": -458.8755787037037,
      "logps/rejected": -397.19852650316454,
      "loss": 0.26,
      "loss/base_kto": 2.080324411392212,
      "metrics/advantage_var": 15504.6064453125,
      "rewards/chosen": 2.703242925950039,
      "rewards/margins": 4.670523446618551,
      "rewards/rejected": -1.9672805206685127,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 8.893047332763672,
      "kl": 21.676542282104492,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -51821296.0,
      "logits/rejected": -50792192.0,
      "logps/chosen": -489.333984375,
      "logps/rejected": -384.91728515625,
      "loss": 0.2509,
      "loss/base_kto": 2.0074877738952637,
      "metrics/advantage_var": 14328.572265625,
      "rewards/chosen": 2.416120719909668,
      "rewards/margins": 4.925666618347169,
      "rewards/rejected": -2.5095458984375,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 6.958967208862305,
      "kl": 17.569808959960938,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -49479295.79119086,
      "logits/rejected": -49944173.76911544,
      "logps/chosen": -443.45417006525287,
      "logps/rejected": -394.2795164917541,
      "loss": 0.2545,
      "loss/base_kto": 2.036057472229004,
      "metrics/advantage_var": 15028.1669921875,
      "rewards/chosen": 2.4243399042873164,
      "rewards/margins": 4.9938617839724735,
      "rewards/rejected": -2.5695218796851576,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 7.773034572601318,
      "kl": 23.792673110961914,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -50371183.30434783,
      "logits/rejected": -51787280.10062893,
      "logps/chosen": -442.39780667701865,
      "logps/rejected": -429.15954205974845,
      "loss": 0.2575,
      "loss/base_kto": 2.0603244304656982,
      "metrics/advantage_var": 14647.125,
      "rewards/chosen": 2.392054966517857,
      "rewards/margins": 4.850691364353451,
      "rewards/rejected": -2.4586363978355936,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 6.8994832038879395,
      "kl": 26.021778106689453,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -51890331.00161031,
      "logits/rejected": -52612660.05462822,
      "logps/chosen": -461.7973027375201,
      "logps/rejected": -412.12395675265554,
      "loss": 0.2551,
      "loss/base_kto": 2.040802478790283,
      "metrics/advantage_var": 14155.7734375,
      "rewards/chosen": 2.7738247439865136,
      "rewards/margins": 4.97032312280385,
      "rewards/rejected": -2.1964983788173367,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.21739673614502,
      "kl": 22.458288192749023,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -50118678.43505477,
      "logits/rejected": -49249246.4524181,
      "logps/chosen": -443.7821302816901,
      "logps/rejected": -380.8603256630265,
      "loss": 0.2577,
      "loss/base_kto": 2.0616891384124756,
      "metrics/advantage_var": 13250.5341796875,
      "rewards/chosen": 2.6506359118251175,
      "rewards/margins": 4.795738854614704,
      "rewards/rejected": -2.1451029427895865,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 8.37536334991455,
      "kl": 30.660608291625977,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -50617106.16774193,
      "logits/rejected": -50973444.654545456,
      "logps/chosen": -463.07631048387094,
      "logps/rejected": -389.0999053030303,
      "loss": 0.2537,
      "loss/base_kto": 2.029655933380127,
      "metrics/advantage_var": 12852.322265625,
      "rewards/chosen": 2.814442886844758,
      "rewards/margins": 4.745141720888319,
      "rewards/rejected": -1.9306988340435607,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 8.114055633544922,
      "kl": 19.990629196166992,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -52263602.89156626,
      "logits/rejected": -50285930.38961039,
      "logps/chosen": -465.64984939759034,
      "logps/rejected": -403.7388392857143,
      "loss": 0.2475,
      "loss/base_kto": 1.9802414178848267,
      "metrics/advantage_var": 13638.9423828125,
      "rewards/chosen": 2.736863101821348,
      "rewards/margins": 4.915339014336567,
      "rewards/rejected": -2.178475912515219,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 7.683564186096191,
      "kl": 21.631948471069336,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -50590506.86697966,
      "logits/rejected": -50809387.93135726,
      "logps/chosen": -460.38810641627543,
      "logps/rejected": -386.3815815132605,
      "loss": 0.2445,
      "loss/base_kto": 1.9557570219039917,
      "metrics/advantage_var": 13694.1279296875,
      "rewards/chosen": 2.935483260893486,
      "rewards/margins": 5.136769588713786,
      "rewards/rejected": -2.2012863278203003,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 8.713532447814941,
      "kl": 14.259411811828613,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -52655586.43161094,
      "logits/rejected": -51266102.327974275,
      "logps/chosen": -456.429141337386,
      "logps/rejected": -396.64253416398714,
      "loss": 0.2434,
      "loss/base_kto": 1.9473152160644531,
      "metrics/advantage_var": 13935.6845703125,
      "rewards/chosen": 2.8355577463074657,
      "rewards/margins": 5.060517631883229,
      "rewards/rejected": -2.2249598855757635,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 7.959493637084961,
      "kl": 25.214923858642578,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -52473988.3537415,
      "logits/rejected": -52252316.85549133,
      "logps/chosen": -443.0916772959184,
      "logps/rejected": -395.4490606936416,
      "loss": 0.2622,
      "loss/base_kto": 2.0979087352752686,
      "metrics/advantage_var": 14081.1298828125,
      "rewards/chosen": 2.602545705782313,
      "rewards/margins": 4.721474471757295,
      "rewards/rejected": -2.118928765974982,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 6.154829025268555,
      "kl": 13.369463920593262,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -52219029.853658535,
      "logits/rejected": -52134872.615384616,
      "logps/chosen": -477.7408060213415,
      "logps/rejected": -382.8152043269231,
      "loss": 0.2455,
      "loss/base_kto": 1.9636309146881104,
      "metrics/advantage_var": 14574.7529296875,
      "rewards/chosen": 2.274860661204268,
      "rewards/margins": 5.1025322012934105,
      "rewards/rejected": -2.827671540089143,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 7.846475124359131,
      "kl": 16.547452926635742,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -51591907.21893491,
      "logits/rejected": -52209226.59602649,
      "logps/chosen": -461.81850961538464,
      "logps/rejected": -389.6706332781457,
      "loss": 0.2457,
      "loss/base_kto": 1.9656704664230347,
      "metrics/advantage_var": 14242.876953125,
      "rewards/chosen": 2.8158327362241122,
      "rewards/margins": 4.997085727350974,
      "rewards/rejected": -2.1812529911268626,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 5.958191394805908,
      "kl": 29.157068252563477,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -51625569.64510779,
      "logits/rejected": -52310252.65777778,
      "logps/chosen": -456.7199937810945,
      "logps/rejected": -382.9580324074074,
      "loss": 0.2103,
      "loss/base_kto": 1.6824930906295776,
      "metrics/advantage_var": 15329.853515625,
      "rewards/chosen": 3.0148241539697347,
      "rewards/margins": 5.539358659178069,
      "rewards/rejected": -2.5245345052083334,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 6.810640811920166,
      "kl": 18.33027458190918,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -51165085.062801935,
      "logits/rejected": -51187286.23975721,
      "logps/chosen": -443.048309178744,
      "logps/rejected": -402.1879742033384,
      "loss": 0.2023,
      "loss/base_kto": 1.6187645196914673,
      "metrics/advantage_var": 15290.6318359375,
      "rewards/chosen": 2.663107560260668,
      "rewards/margins": 5.793123001506874,
      "rewards/rejected": -3.1300154412462065,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 4.337776184082031,
      "kl": 17.0222225189209,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -49851196.33121019,
      "logits/rejected": -51849448.44171779,
      "logps/chosen": -432.84146098726114,
      "logps/rejected": -406.1946894171779,
      "loss": 0.1933,
      "loss/base_kto": 1.5461317300796509,
      "metrics/advantage_var": 15725.984375,
      "rewards/chosen": 3.019858585041799,
      "rewards/margins": 6.097644708867527,
      "rewards/rejected": -3.0777861238257285,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 5.722031593322754,
      "kl": 18.55806541442871,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -51315540.80495356,
      "logits/rejected": -52570457.64037855,
      "logps/chosen": -461.937886996904,
      "logps/rejected": -408.2663643533123,
      "loss": 0.2,
      "loss/base_kto": 1.600163459777832,
      "metrics/advantage_var": 16523.453125,
      "rewards/chosen": 2.6366288034539473,
      "rewards/margins": 6.083913682446456,
      "rewards/rejected": -3.447284878992508,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 13.217945098876953,
      "kl": 30.77508544921875,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -49535898.27814569,
      "logits/rejected": -50719916.686390534,
      "logps/chosen": -429.2412044701987,
      "logps/rejected": -397.5885724852071,
      "loss": 0.2172,
      "loss/base_kto": 1.7376642227172852,
      "metrics/advantage_var": 14375.2919921875,
      "rewards/chosen": 2.7198288267021935,
      "rewards/margins": 5.497414550398569,
      "rewards/rejected": -2.777585723696376,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 6.928693771362305,
      "kl": 15.719565391540527,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -51029232.047477745,
      "logits/rejected": -51983566.15181518,
      "logps/chosen": -446.40995919881306,
      "logps/rejected": -396.5913778877888,
      "loss": 0.2143,
      "loss/base_kto": 1.7145565748214722,
      "metrics/advantage_var": 15240.5029296875,
      "rewards/chosen": 2.615926227512982,
      "rewards/margins": 5.480235204797429,
      "rewards/rejected": -2.864308977284447,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 7.271140098571777,
      "kl": 19.24259376525879,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -50473962.53892215,
      "logits/rejected": -50950324.705882356,
      "logps/chosen": -447.1455838323353,
      "logps/rejected": -382.8148999183006,
      "loss": 0.1973,
      "loss/base_kto": 1.5785809755325317,
      "metrics/advantage_var": 14588.8984375,
      "rewards/chosen": 2.841123112661396,
      "rewards/margins": 5.781759043140358,
      "rewards/rejected": -2.940635930478962,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 8.26968765258789,
      "kl": 23.165260314941406,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51096412.98299845,
      "logits/rejected": -49997028.09478673,
      "logps/chosen": -449.54791344667694,
      "logps/rejected": -388.7860880726698,
      "loss": 0.2058,
      "loss/base_kto": 1.6466184854507446,
      "metrics/advantage_var": 14070.4189453125,
      "rewards/chosen": 3.1233227123985703,
      "rewards/margins": 5.540160238933918,
      "rewards/rejected": -2.4168375265353474,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.453213691711426,
      "kl": 24.0378475189209,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -51170121.95555556,
      "logits/rejected": -52540230.10461538,
      "logps/chosen": -449.71170634920634,
      "logps/rejected": -378.6451923076923,
      "loss": 0.2143,
      "loss/base_kto": 1.7145299911499023,
      "metrics/advantage_var": 15124.421875,
      "rewards/chosen": 2.6893165225074407,
      "rewards/margins": 5.637288652916094,
      "rewards/rejected": -2.9479721304086537,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 7.117490291595459,
      "kl": 13.310893058776855,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -53527604.9118541,
      "logits/rejected": -52066139.36977492,
      "logps/chosen": -487.0769376899696,
      "logps/rejected": -395.3713072749196,
      "loss": 0.1942,
      "loss/base_kto": 1.5535629987716675,
      "metrics/advantage_var": 15743.8505859375,
      "rewards/chosen": 2.905693448423252,
      "rewards/margins": 6.0703902430374,
      "rewards/rejected": -3.164696794614148,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 8.523292541503906,
      "kl": 9.435015678405762,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -52503106.983458646,
      "logits/rejected": -52026812.565853655,
      "logps/chosen": -458.8041353383459,
      "logps/rejected": -397.1086636178862,
      "loss": 0.2147,
      "loss/base_kto": 1.7179268598556519,
      "metrics/advantage_var": 15920.5966796875,
      "rewards/chosen": 2.546815158012218,
      "rewards/margins": 5.578676105039657,
      "rewards/rejected": -3.031860947027439,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.27126407623291,
      "kl": 18.656579971313477,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -51284638.84094488,
      "logits/rejected": -52479618.97674418,
      "logps/chosen": -462.7442421259843,
      "logps/rejected": -396.7521560077519,
      "loss": 0.2098,
      "loss/base_kto": 1.6782121658325195,
      "metrics/advantage_var": 16229.3251953125,
      "rewards/chosen": 2.876315860297736,
      "rewards/margins": 5.866616475002194,
      "rewards/rejected": -2.9903006147044575,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 9.956695556640625,
      "kl": 19.218399047851562,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -53085059.28205128,
      "logits/rejected": -53581461.853658535,
      "logps/chosen": -448.7760416666667,
      "logps/rejected": -414.6380049542683,
      "loss": 0.2131,
      "loss/base_kto": 1.7047923803329468,
      "metrics/advantage_var": 15295.837890625,
      "rewards/chosen": 2.752549782777444,
      "rewards/margins": 5.645609303964086,
      "rewards/rejected": -2.8930595211866423,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 6.9843621253967285,
      "kl": 18.419477462768555,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -50652528.64,
      "logits/rejected": -52052174.36335878,
      "logps/chosen": -470.1084,
      "logps/rejected": -411.04742366412216,
      "loss": 0.2113,
      "loss/base_kto": 1.6904957294464111,
      "metrics/advantage_var": 16657.041015625,
      "rewards/chosen": 2.6572134765625,
      "rewards/margins": 5.88140240786021,
      "rewards/rejected": -3.22418893129771,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 7.0096259117126465,
      "kl": 12.720903396606445,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -52172171.779141106,
      "logits/rejected": -53297830.318471335,
      "logps/chosen": -458.0583780674847,
      "logps/rejected": -419.7462679140127,
      "loss": 0.2031,
      "loss/base_kto": 1.6250914335250854,
      "metrics/advantage_var": 16653.916015625,
      "rewards/chosen": 2.894802350939417,
      "rewards/margins": 6.069655438929564,
      "rewards/rejected": -3.1748530879901473,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 8.27895736694336,
      "kl": 17.4371395111084,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -53293252.21556886,
      "logits/rejected": -52639489.67320261,
      "logps/chosen": -440.76974176646706,
      "logps/rejected": -424.7828839869281,
      "loss": 0.1977,
      "loss/base_kto": 1.5816789865493774,
      "metrics/advantage_var": 14772.7255859375,
      "rewards/chosen": 2.7534731562266095,
      "rewards/margins": 5.898639889912271,
      "rewards/rejected": -3.1451667336856617,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 5.753119468688965,
      "kl": 21.079870223999023,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -53754379.906976745,
      "logits/rejected": -52337610.784251966,
      "logps/chosen": -469.2296511627907,
      "logps/rejected": -387.3084153543307,
      "loss": 0.1987,
      "loss/base_kto": 1.589290738105774,
      "metrics/advantage_var": 15189.6533203125,
      "rewards/chosen": 3.0228050054505813,
      "rewards/margins": 5.7660919072099315,
      "rewards/rejected": -2.74328690175935,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 8.482635498046875,
      "kl": 16.362096786499023,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -52656223.47067669,
      "logits/rejected": -51659437.99674797,
      "logps/chosen": -463.7524906015038,
      "logps/rejected": -418.1763211382114,
      "loss": 0.22,
      "loss/base_kto": 1.7602322101593018,
      "metrics/advantage_var": 15808.220703125,
      "rewards/chosen": 3.0392983802279137,
      "rewards/margins": 5.632142082590718,
      "rewards/rejected": -2.592843702362805,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 8.359546661376953,
      "kl": 20.033288955688477,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -50505083.61370716,
      "logits/rejected": -53133639.42319749,
      "logps/chosen": -479.8252531152648,
      "logps/rejected": -386.9917221786834,
      "loss": 0.2022,
      "loss/base_kto": 1.6174923181533813,
      "metrics/advantage_var": 15608.984375,
      "rewards/chosen": 2.9505763544100465,
      "rewards/margins": 5.919393129549643,
      "rewards/rejected": -2.9688167751395964,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 6.454035758972168,
      "kl": 18.603702545166016,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -51773555.86544342,
      "logits/rejected": -52918828.166134186,
      "logps/chosen": -440.5784594801223,
      "logps/rejected": -401.768794928115,
      "loss": 0.2248,
      "loss/base_kto": 1.7981531620025635,
      "metrics/advantage_var": 14751.2919921875,
      "rewards/chosen": 2.5384420692373855,
      "rewards/margins": 5.27170751639693,
      "rewards/rejected": -2.733265447159545,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 7.05014181137085,
      "kl": 20.547834396362305,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -52540588.00627943,
      "logits/rejected": -52747510.8429238,
      "logps/chosen": -462.1351059654631,
      "logps/rejected": -421.39011469673403,
      "loss": 0.2165,
      "loss/base_kto": 1.732211709022522,
      "metrics/advantage_var": 17090.8125,
      "rewards/chosen": 2.817033273645997,
      "rewards/margins": 5.648834431548408,
      "rewards/rejected": -2.8318011579024107,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 9.174979209899902,
      "kl": 26.36089515686035,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -52650930.716981135,
      "logits/rejected": -53045127.1552795,
      "logps/chosen": -451.7421875,
      "logps/rejected": -429.9204192546584,
      "loss": 0.2084,
      "loss/base_kto": 1.6668694019317627,
      "metrics/advantage_var": 15893.34375,
      "rewards/chosen": 2.9159428818420796,
      "rewards/margins": 5.797485016571311,
      "rewards/rejected": -2.8815421347292314,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.659214973449707,
      "kl": 20.512054443359375,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -50370115.347068146,
      "logits/rejected": -52382443.882896766,
      "logps/chosen": -486.13431061806654,
      "logps/rejected": -401.66896186440675,
      "loss": 0.2005,
      "loss/base_kto": 1.6037886142730713,
      "metrics/advantage_var": 16707.20703125,
      "rewards/chosen": 2.9694340580427894,
      "rewards/margins": 6.0927611869189455,
      "rewards/rejected": -3.1233271288761557,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 10.550207138061523,
      "kl": 12.67325210571289,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52556268.48093842,
      "logits/rejected": -52346379.98662207,
      "logps/chosen": -459.1465359237537,
      "logps/rejected": -410.18653323578593,
      "loss": 0.2122,
      "loss/base_kto": 1.6974483728408813,
      "metrics/advantage_var": 15797.478515625,
      "rewards/chosen": 2.6588947374450145,
      "rewards/margins": 5.798360273607222,
      "rewards/rejected": -3.1394655361622075,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 6.611340045928955,
      "kl": 19.12717628479004,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -53003676.17610063,
      "logits/rejected": -53865535.60248447,
      "logps/chosen": -473.6758549528302,
      "logps/rejected": -414.92342779503105,
      "loss": 0.2046,
      "loss/base_kto": 1.6370079517364502,
      "metrics/advantage_var": 16002.5283203125,
      "rewards/chosen": 2.987971083922956,
      "rewards/margins": 5.849641536538446,
      "rewards/rejected": -2.8616704526154892,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 8.27307415008545,
      "kl": 17.498138427734375,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -52917409.33762058,
      "logits/rejected": -54464972.6443769,
      "logps/chosen": -489.0168810289389,
      "logps/rejected": -361.63445098784194,
      "loss": 0.2184,
      "loss/base_kto": 1.747402548789978,
      "metrics/advantage_var": 15314.0439453125,
      "rewards/chosen": 2.673038593059184,
      "rewards/margins": 5.558903221906069,
      "rewards/rejected": -2.8858646288468845,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 7.73489236831665,
      "kl": 20.97342300415039,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -51807078.4,
      "logits/rejected": -52076121.6,
      "logps/chosen": -439.680224609375,
      "logps/rejected": -410.398681640625,
      "loss": 0.2077,
      "loss/base_kto": 1.6616114377975464,
      "metrics/advantage_var": 16345.5283203125,
      "rewards/chosen": 2.898772430419922,
      "rewards/margins": 5.813119125366211,
      "rewards/rejected": -2.914346694946289,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.893743515014648,
      "kl": 18.39280128479004,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -51742174.08201893,
      "logits/rejected": -53058696.32198142,
      "logps/chosen": -466.2358537066246,
      "logps/rejected": -402.3814096362229,
      "loss": 0.2021,
      "loss/base_kto": 1.6166282892227173,
      "metrics/advantage_var": 16531.447265625,
      "rewards/chosen": 2.782011110323837,
      "rewards/margins": 6.094929475140981,
      "rewards/rejected": -3.312918364817144,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 68.18772888183594,
      "kl": 16.023277282714844,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -51808847.79220779,
      "logits/rejected": -52280751.807228915,
      "logps/chosen": -449.70824878246754,
      "logps/rejected": -390.80294615963857,
      "loss": 0.2201,
      "loss/base_kto": 1.7605113983154297,
      "metrics/advantage_var": 16087.9638671875,
      "rewards/chosen": 2.6439783666040992,
      "rewards/margins": 5.52344900795858,
      "rewards/rejected": -2.8794706413544806,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 6.0411295890808105,
      "kl": 16.658052444458008,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -52695826.73170732,
      "logits/rejected": -53972630.974358976,
      "logps/chosen": -455.8776676829268,
      "logps/rejected": -410.6433293269231,
      "loss": 0.2212,
      "loss/base_kto": 1.7692569494247437,
      "metrics/advantage_var": 14827.8408203125,
      "rewards/chosen": 2.6817597179878048,
      "rewards/margins": 5.596522166029672,
      "rewards/rejected": -2.914762448041867,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 6.565061092376709,
      "kl": 17.83756446838379,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -52485077.19614148,
      "logits/rejected": -51947989.981762916,
      "logps/chosen": -476.3875100482315,
      "logps/rejected": -414.3374810030395,
      "loss": 0.2101,
      "loss/base_kto": 1.6810640096664429,
      "metrics/advantage_var": 19462.033203125,
      "rewards/chosen": 3.0427821119498595,
      "rewards/margins": 6.14089621838033,
      "rewards/rejected": -3.098114106430471,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 6.715926647186279,
      "kl": 19.532516479492188,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -51633677.41140216,
      "logits/rejected": -52861577.12836767,
      "logps/chosen": -422.4470338983051,
      "logps/rejected": -390.1222266244057,
      "loss": 0.2153,
      "loss/base_kto": 1.7227131128311157,
      "metrics/advantage_var": 14244.3701171875,
      "rewards/chosen": 2.758965866958783,
      "rewards/margins": 5.391663067582792,
      "rewards/rejected": -2.6326972006240097,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 7.742791652679443,
      "kl": 20.519500732421875,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -53814366.78603946,
      "logits/rejected": -53390799.355877616,
      "logps/chosen": -480.35399279210924,
      "logps/rejected": -401.73090277777777,
      "loss": 0.1977,
      "loss/base_kto": 1.581398367881775,
      "metrics/advantage_var": 16275.658203125,
      "rewards/chosen": 3.0262747548961495,
      "rewards/margins": 6.12475463550706,
      "rewards/rejected": -3.0984798806109097,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 8.513627052307129,
      "kl": 19.383682250976562,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -51164498.066985644,
      "logits/rejected": -51537223.74272588,
      "logps/chosen": -452.1843102073365,
      "logps/rejected": -391.86485451761104,
      "loss": 0.213,
      "loss/base_kto": 1.7036470174789429,
      "metrics/advantage_var": 14536.775390625,
      "rewards/chosen": 2.9792410380532295,
      "rewards/margins": 5.64744869442287,
      "rewards/rejected": -2.66820765636964,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 7.5082902908325195,
      "kl": 20.06099510192871,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -54347665.12101911,
      "logits/rejected": -53955678.23312883,
      "logps/chosen": -460.7467655254777,
      "logps/rejected": -422.7647622699387,
      "loss": 0.2077,
      "loss/base_kto": 1.6615208387374878,
      "metrics/advantage_var": 17052.80078125,
      "rewards/chosen": 2.729910880896696,
      "rewards/margins": 5.756208795366597,
      "rewards/rejected": -3.0262979144699003,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 6.819154262542725,
      "kl": 19.58155632019043,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -51551179.77125193,
      "logits/rejected": -53412732.96682464,
      "logps/chosen": -456.2817329984544,
      "logps/rejected": -414.9955568720379,
      "loss": 0.2005,
      "loss/base_kto": 1.6039291620254517,
      "metrics/advantage_var": 15772.376953125,
      "rewards/chosen": 2.918591365195131,
      "rewards/margins": 5.92761415473897,
      "rewards/rejected": -3.009022789543839,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.863317489624023,
      "kl": 18.824533462524414,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -51992081.436532505,
      "logits/rejected": -52817858.62460568,
      "logps/chosen": -457.50541795665634,
      "logps/rejected": -387.4557866719243,
      "loss": 0.2272,
      "loss/base_kto": 1.8178986310958862,
      "metrics/advantage_var": 14906.2841796875,
      "rewards/chosen": 2.748334855118034,
      "rewards/margins": 5.385879986476473,
      "rewards/rejected": -2.6375451313584386,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 7.7461771965026855,
      "kl": 20.338605880737305,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -51067443.36569579,
      "logits/rejected": -51839430.7673716,
      "logps/chosen": -480.4406351132686,
      "logps/rejected": -402.48682968277944,
      "loss": 0.2209,
      "loss/base_kto": 1.7669308185577393,
      "metrics/advantage_var": 16044.369140625,
      "rewards/chosen": 2.5220058404126213,
      "rewards/margins": 5.390470837580295,
      "rewards/rejected": -2.8684649971676737,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.960196480656998e+17,
      "train_loss": 0.2718521337023259,
      "train_runtime": 11043.5973,
      "train_samples_per_second": 13.421,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.960196480656998e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}