{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.78728199005127,
      "kl": 6.045640468597412,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37436245.333333336,
      "logits/rejected": -39276200.65882353,
      "logps/chosen": -485.2478125,
      "logps/rejected": -358.6040670955882,
      "loss": 0.4642,
      "loss/base_kto": 3.7135796546936035,
      "metrics/advantage_var": 399.55609130859375,
      "rewards/chosen": -0.16165313720703126,
      "rewards/margins": 0.2856965906479779,
      "rewards/rejected": -0.4473497278550092,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.290058135986328,
      "kl": 7.082744121551514,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -36603852.558869705,
      "logits/rejected": -37564747.247278385,
      "logps/chosen": -464.4825843799058,
      "logps/rejected": -382.692675933126,
      "loss": 0.4521,
      "loss/base_kto": 3.61698842048645,
      "metrics/advantage_var": 725.0475463867188,
      "rewards/chosen": 0.0748193522263172,
      "rewards/margins": 0.4525030006516671,
      "rewards/rejected": -0.3776836484253499,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 11.527665138244629,
      "kl": 5.735081672668457,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36186875.10828026,
      "logits/rejected": -38534866.45398773,
      "logps/chosen": -479.4423765923567,
      "logps/rejected": -381.6414877300613,
      "loss": 0.4266,
      "loss/base_kto": 3.412820816040039,
      "metrics/advantage_var": 935.1497192382812,
      "rewards/chosen": 0.272069019876468,
      "rewards/margins": 0.7197896469883593,
      "rewards/rejected": -0.4477206271118913,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 8.851114273071289,
      "kl": 5.344668865203857,
      "learning_rate": 7.9e-07,
      "logits/chosen": -38228702.0032,
      "logits/rejected": -39082499.908396944,
      "logps/chosen": -478.27695,
      "logps/rejected": -376.62304389312976,
      "loss": 0.4305,
      "loss/base_kto": 3.4443156719207764,
      "metrics/advantage_var": 1630.0986328125,
      "rewards/chosen": 0.14659896240234374,
      "rewards/margins": 0.7242343765841185,
      "rewards/rejected": -0.5776354141817748,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 10.306970596313477,
      "kl": 16.163732528686523,
      "learning_rate": 9.9e-07,
      "logits/chosen": -37611377.270664506,
      "logits/rejected": -38828652.88687783,
      "logps/chosen": -441.5427471636953,
      "logps/rejected": -363.75973322021116,
      "loss": 0.4155,
      "loss/base_kto": 3.3236663341522217,
      "metrics/advantage_var": 2278.3662109375,
      "rewards/chosen": 0.639388667126722,
      "rewards/margins": 0.949373107304536,
      "rewards/rejected": -0.3099844401778139,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 8.703822135925293,
      "kl": 19.105472564697266,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -38187107.38823529,
      "logits/rejected": -38690228.906666666,
      "logps/chosen": -477.3513786764706,
      "logps/rejected": -371.125703125,
      "loss": 0.4164,
      "loss/base_kto": 3.331439256668091,
      "metrics/advantage_var": 3020.926513671875,
      "rewards/chosen": 0.6670391307157628,
      "rewards/margins": 0.9879135101916743,
      "rewards/rejected": -0.32087437947591146,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 7.378037452697754,
      "kl": 47.66223907470703,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -38584833.528358206,
      "logits/rejected": -39298165.50819672,
      "logps/chosen": -460.7433768656716,
      "logps/rejected": -353.40066598360653,
      "loss": 0.3796,
      "loss/base_kto": 3.0371625423431396,
      "metrics/advantage_var": 2964.495361328125,
      "rewards/chosen": 1.6767709305037313,
      "rewards/margins": 1.3247292044695356,
      "rewards/rejected": 0.3520417260341957,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 7.053978443145752,
      "kl": 10.050368309020996,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -41215980.679245286,
      "logits/rejected": -40242430.40993789,
      "logps/chosen": -470.062106918239,
      "logps/rejected": -374.5430415372671,
      "loss": 0.3795,
      "loss/base_kto": 3.0357117652893066,
      "metrics/advantage_var": 4414.76513671875,
      "rewards/chosen": 0.6166294025924971,
      "rewards/margins": 1.6501214020587236,
      "rewards/rejected": -1.0334919994662266,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 10.467313766479492,
      "kl": 13.880614280700684,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -40612920.26373626,
      "logits/rejected": -40778782.25816485,
      "logps/chosen": -464.33070054945057,
      "logps/rejected": -358.9967680793157,
      "loss": 0.3659,
      "loss/base_kto": 2.9274158477783203,
      "metrics/advantage_var": 5033.04248046875,
      "rewards/chosen": 0.90423583984375,
      "rewards/margins": 1.9941266713950718,
      "rewards/rejected": -1.0898908315513218,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 8.401222229003906,
      "kl": 34.443275451660156,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -42970434.54434251,
      "logits/rejected": -41785458.50479233,
      "logps/chosen": -495.66800458715596,
      "logps/rejected": -364.3836861022364,
      "loss": 0.3681,
      "loss/base_kto": 2.944542407989502,
      "metrics/advantage_var": 4823.06640625,
      "rewards/chosen": 1.6523476696889334,
      "rewards/margins": 1.8160656281819072,
      "rewards/rejected": -0.16371795849297374,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 7.976482391357422,
      "kl": 16.6918888092041,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -42382349.98482549,
      "logits/rejected": -43137208.68276972,
      "logps/chosen": -486.696936646434,
      "logps/rejected": -382.6753723832528,
      "loss": 0.371,
      "loss/base_kto": 2.96830153465271,
      "metrics/advantage_var": 5592.57373046875,
      "rewards/chosen": 1.4093304693427542,
      "rewards/margins": 1.9811324826969712,
      "rewards/rejected": -0.571802013354217,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 7.520793914794922,
      "kl": 30.6082820892334,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -41710281.45567652,
      "logits/rejected": -41567834.82574568,
      "logps/chosen": -460.29806570762054,
      "logps/rejected": -371.8941326530612,
      "loss": 0.3473,
      "loss/base_kto": 2.7783279418945312,
      "metrics/advantage_var": 6056.45361328125,
      "rewards/chosen": 1.6413401459831842,
      "rewards/margins": 2.216954176902042,
      "rewards/rejected": -0.5756140309188579,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 8.243804931640625,
      "kl": 14.915915489196777,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -42622353.537007876,
      "logits/rejected": -42973828.56434108,
      "logps/chosen": -479.83922244094487,
      "logps/rejected": -391.5109738372093,
      "loss": 0.3694,
      "loss/base_kto": 2.9554238319396973,
      "metrics/advantage_var": 6400.8681640625,
      "rewards/chosen": 0.871806275375246,
      "rewards/margins": 2.077575848261632,
      "rewards/rejected": -1.2057695728863858,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.776176929473877,
      "kl": 19.597034454345703,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -40272461.824,
      "logits/rejected": -41789380.59236641,
      "logps/chosen": -469.8292,
      "logps/rejected": -394.7070610687023,
      "loss": 0.3643,
      "loss/base_kto": 2.9144322872161865,
      "metrics/advantage_var": 6827.59619140625,
      "rewards/chosen": 1.0211619140625,
      "rewards/margins": 2.1597158188215646,
      "rewards/rejected": -1.138553904759065,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 7.238597393035889,
      "kl": 14.930597305297852,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -41993617.38271605,
      "logits/rejected": -42283986.63291139,
      "logps/chosen": -468.4034047067901,
      "logps/rejected": -399.5572339794304,
      "loss": 0.3445,
      "loss/base_kto": 2.7562012672424316,
      "metrics/advantage_var": 6334.48095703125,
      "rewards/chosen": 1.5196476689091436,
      "rewards/margins": 2.4504255928794265,
      "rewards/rejected": -0.9307779239702828,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 7.258980751037598,
      "kl": 36.62384033203125,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -44909393.996992484,
      "logits/rejected": -43970276.94308943,
      "logps/chosen": -451.784492481203,
      "logps/rejected": -391.7923526422764,
      "loss": 0.34,
      "loss/base_kto": 2.720338821411133,
      "metrics/advantage_var": 6735.33935546875,
      "rewards/chosen": 2.063958235432331,
      "rewards/margins": 2.3941122822120566,
      "rewards/rejected": -0.3301540467797256,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 8.35556411743164,
      "kl": 19.140226364135742,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -46471030.614664584,
      "logits/rejected": -44903640.33802817,
      "logps/chosen": -494.1961290951638,
      "logps/rejected": -376.35766334115806,
      "loss": 0.3404,
      "loss/base_kto": 2.722824811935425,
      "metrics/advantage_var": 8471.8046875,
      "rewards/chosen": 1.7114452058551093,
      "rewards/margins": 2.770817989868216,
      "rewards/rejected": -1.0593727840131064,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 9.343971252441406,
      "kl": 13.9060697555542,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -43613165.54954955,
      "logits/rejected": -43185035.25732899,
      "logps/chosen": -478.3201951951952,
      "logps/rejected": -399.224730252443,
      "loss": 0.3565,
      "loss/base_kto": 2.8517332077026367,
      "metrics/advantage_var": 8707.2890625,
      "rewards/chosen": 1.4123950305285755,
      "rewards/margins": 2.5237740512940476,
      "rewards/rejected": -1.1113790207654723,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 8.0005521774292,
      "kl": 31.26420021057129,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -42496073.14285714,
      "logits/rejected": -44363827.52201258,
      "logps/chosen": -478.9863159937888,
      "logps/rejected": -366.2966047562893,
      "loss": 0.3337,
      "loss/base_kto": 2.6695454120635986,
      "metrics/advantage_var": 8384.955078125,
      "rewards/chosen": 1.8745265036636258,
      "rewards/margins": 2.831779246023099,
      "rewards/rejected": -0.9572527423594732,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 9.551956176757812,
      "kl": 30.651647567749023,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -44210673.64485981,
      "logits/rejected": -45153241.479623824,
      "logps/chosen": -462.75515965732086,
      "logps/rejected": -386.5633571708464,
      "loss": 0.3509,
      "loss/base_kto": 2.8070526123046875,
      "metrics/advantage_var": 7753.9462890625,
      "rewards/chosen": 1.7811591127579829,
      "rewards/margins": 2.4744187555473394,
      "rewards/rejected": -0.6932596427893564,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 8.593188285827637,
      "kl": 33.22036361694336,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44223064.493827164,
      "logits/rejected": -43488385.62025317,
      "logps/chosen": -467.7560763888889,
      "logps/rejected": -389.8130933544304,
      "loss": 0.3453,
      "loss/base_kto": 2.7623486518859863,
      "metrics/advantage_var": 9086.478515625,
      "rewards/chosen": 1.87820585274402,
      "rewards/margins": 2.616768424595533,
      "rewards/rejected": -0.7385625718515131,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 13.203508377075195,
      "kl": 18.998483657836914,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -45640985.03470032,
      "logits/rejected": -46091413.00309598,
      "logps/chosen": -476.49191640378547,
      "logps/rejected": -430.8054373065016,
      "loss": 0.3401,
      "loss/base_kto": 2.721027374267578,
      "metrics/advantage_var": 9048.0234375,
      "rewards/chosen": 1.0728805012507394,
      "rewards/margins": 2.8573349414355302,
      "rewards/rejected": -1.784454440184791,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 8.664673805236816,
      "kl": 28.86256217956543,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -42183146.09191759,
      "logits/rejected": -43393292.22804315,
      "logps/chosen": -485.44958399366084,
      "logps/rejected": -378.23045069337445,
      "loss": 0.3378,
      "loss/base_kto": 2.702357053756714,
      "metrics/advantage_var": 8466.8408203125,
      "rewards/chosen": 1.8133991803684628,
      "rewards/margins": 2.888673302515516,
      "rewards/rejected": -1.0752741221470532,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 7.829504489898682,
      "kl": 37.354576110839844,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -45578671.807228915,
      "logits/rejected": -45010784.415584415,
      "logps/chosen": -464.59892695783134,
      "logps/rejected": -383.26407771915586,
      "loss": 0.3198,
      "loss/base_kto": 2.55859375,
      "metrics/advantage_var": 8599.970703125,
      "rewards/chosen": 2.297871968832361,
      "rewards/margins": 2.9343021890028154,
      "rewards/rejected": -0.6364302201704546,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 9.73172664642334,
      "kl": 29.568395614624023,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -45679112.6779661,
      "logits/rejected": -46415651.296354994,
      "logps/chosen": -490.24913328197226,
      "logps/rejected": -382.99640946909665,
      "loss": 0.3263,
      "loss/base_kto": 2.610485792160034,
      "metrics/advantage_var": 9661.892578125,
      "rewards/chosen": 2.01228038629141,
      "rewards/margins": 3.09106845443102,
      "rewards/rejected": -1.0787880681396098,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.299348831176758,
      "kl": 25.486047744750977,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -45759279.8699187,
      "logits/rejected": -47092001.49172933,
      "logps/chosen": -488.6234756097561,
      "logps/rejected": -358.422227443609,
      "loss": 0.3261,
      "loss/base_kto": 2.608835458755493,
      "metrics/advantage_var": 9507.947265625,
      "rewards/chosen": 1.7613670287093497,
      "rewards/margins": 3.0828396555890487,
      "rewards/rejected": -1.3214726268796992,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 7.881642818450928,
      "kl": 47.24177932739258,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -45693681.538931295,
      "logits/rejected": -45166647.7056,
      "logps/chosen": -446.0384541984733,
      "logps/rejected": -400.538975,
      "loss": 0.3247,
      "loss/base_kto": 2.5978293418884277,
      "metrics/advantage_var": 10352.3291015625,
      "rewards/chosen": 2.509184346672233,
      "rewards/margins": 3.2096743857347327,
      "rewards/rejected": -0.7004900390625,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 7.793478488922119,
      "kl": 35.15746307373047,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -44939849.615508884,
      "logits/rejected": -46229974.94704992,
      "logps/chosen": -464.09995961227787,
      "logps/rejected": -372.51092095310133,
      "loss": 0.3203,
      "loss/base_kto": 2.562568426132202,
      "metrics/advantage_var": 9777.9736328125,
      "rewards/chosen": 2.1015311442977587,
      "rewards/margins": 3.1540492646291693,
      "rewards/rejected": -1.0525181203314107,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 8.537753105163574,
      "kl": 21.723495483398438,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -45649936.56957929,
      "logits/rejected": -45545744.24169184,
      "logps/chosen": -466.2949534789644,
      "logps/rejected": -369.1769967900302,
      "loss": 0.3378,
      "loss/base_kto": 2.702486753463745,
      "metrics/advantage_var": 10140.341796875,
      "rewards/chosen": 1.6531990322866101,
      "rewards/margins": 2.910111309830983,
      "rewards/rejected": -1.256912277544373,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.486123561859131,
      "kl": 29.603656768798828,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -46145442.05504587,
      "logits/rejected": -46587112.28115016,
      "logps/chosen": -468.3471903669725,
      "logps/rejected": -391.3940195686901,
      "loss": 0.3374,
      "loss/base_kto": 2.699201822280884,
      "metrics/advantage_var": 10859.9677734375,
      "rewards/chosen": 2.073822674765864,
      "rewards/margins": 3.0097128481447077,
      "rewards/rejected": -0.9358901733788438,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 9.535039901733398,
      "kl": 24.991872787475586,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -46978959.326860845,
      "logits/rejected": -47023280.33836858,
      "logps/chosen": -478.23391990291265,
      "logps/rejected": -400.0674093655589,
      "loss": 0.3257,
      "loss/base_kto": 2.605404853820801,
      "metrics/advantage_var": 12756.2958984375,
      "rewards/chosen": 1.9604095162697208,
      "rewards/margins": 3.527833264735544,
      "rewards/rejected": -1.5674237484658233,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 7.184971809387207,
      "kl": 24.45808219909668,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -46707373.783866055,
      "logits/rejected": -47414063.255216695,
      "logps/chosen": -445.0344368340944,
      "logps/rejected": -383.34402588282506,
      "loss": 0.332,
      "loss/base_kto": 2.656392812728882,
      "metrics/advantage_var": 10849.8701171875,
      "rewards/chosen": 2.03060100212852,
      "rewards/margins": 3.1241544748035803,
      "rewards/rejected": -1.0935534726750602,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 7.227497100830078,
      "kl": 37.22367477416992,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -47398545.05359878,
      "logits/rejected": -46995420.07017544,
      "logps/chosen": -472.5529287901991,
      "logps/rejected": -374.84190590111643,
      "loss": 0.3065,
      "loss/base_kto": 2.451979160308838,
      "metrics/advantage_var": 11067.1982421875,
      "rewards/chosen": 2.5833558904814318,
      "rewards/margins": 3.576648642736715,
      "rewards/rejected": -0.9932927522552831,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 7.81859016418457,
      "kl": 22.77915382385254,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -47092352.80503145,
      "logits/rejected": -46101885.61490683,
      "logps/chosen": -465.61576257861634,
      "logps/rejected": -376.53144409937886,
      "loss": 0.3287,
      "loss/base_kto": 2.629446506500244,
      "metrics/advantage_var": 11029.83203125,
      "rewards/chosen": 2.13449231033805,
      "rewards/margins": 3.29012447154365,
      "rewards/rejected": -1.1556321612055998,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 9.141914367675781,
      "kl": 27.3914852142334,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -48338286.78119935,
      "logits/rejected": -48161626.73906486,
      "logps/chosen": -471.70006077795784,
      "logps/rejected": -404.33300339366514,
      "loss": 0.3152,
      "loss/base_kto": 2.5217628479003906,
      "metrics/advantage_var": 13034.7158203125,
      "rewards/chosen": 1.950958796026641,
      "rewards/margins": 3.6061690810120295,
      "rewards/rejected": -1.6552102849853885,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 7.617023468017578,
      "kl": 18.97572898864746,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -45358883.44615385,
      "logits/rejected": -47993508.16507936,
      "logps/chosen": -444.6782692307692,
      "logps/rejected": -399.07599206349204,
      "loss": 0.3121,
      "loss/base_kto": 2.4965858459472656,
      "metrics/advantage_var": 12820.9453125,
      "rewards/chosen": 1.9479655573918269,
      "rewards/margins": 3.761633557143811,
      "rewards/rejected": -1.8136679997519842,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 8.201563835144043,
      "kl": 22.254520416259766,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -47446659.926380366,
      "logits/rejected": -47940621.04458599,
      "logps/chosen": -477.0963861196319,
      "logps/rejected": -373.2238505175159,
      "loss": 0.3392,
      "loss/base_kto": 2.7132928371429443,
      "metrics/advantage_var": 12771.9248046875,
      "rewards/chosen": 2.1605134741660277,
      "rewards/margins": 3.263844633040928,
      "rewards/rejected": -1.1033311588749004,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 9.163492202758789,
      "kl": 28.68179702758789,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -46050212.71961102,
      "logits/rejected": -46882056.108597286,
      "logps/chosen": -482.05085089141005,
      "logps/rejected": -386.1900452488688,
      "loss": 0.3257,
      "loss/base_kto": 2.6057794094085693,
      "metrics/advantage_var": 12102.6533203125,
      "rewards/chosen": 1.9439724963913088,
      "rewards/margins": 3.1843624512013577,
      "rewards/rejected": -1.240389954810049,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 6.806163787841797,
      "kl": 25.385343551635742,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47408088.36129032,
      "logits/rejected": -47495398.32218845,
      "logps/chosen": -497.61275201612904,
      "logps/rejected": -389.4937072568389,
      "loss": 0.3046,
      "loss/base_kto": 2.4370665550231934,
      "metrics/advantage_var": 12872.2626953125,
      "rewards/chosen": 2.1698866321194554,
      "rewards/margins": 3.769127569975648,
      "rewards/rejected": -1.599240937856193,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 6.389303684234619,
      "kl": 17.691709518432617,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -48771504.17637795,
      "logits/rejected": -48449129.5751938,
      "logps/chosen": -465.32273622047245,
      "logps/rejected": -400.1376937984496,
      "loss": 0.2682,
      "loss/base_kto": 2.1456105709075928,
      "metrics/advantage_var": 13492.072265625,
      "rewards/chosen": 1.9728844349778543,
      "rewards/margins": 4.646865100554405,
      "rewards/rejected": -2.6739806655765506,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 6.119379043579102,
      "kl": 19.591279983520508,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -48208287.806060605,
      "logits/rejected": -48525847.12258065,
      "logps/chosen": -458.76458333333335,
      "logps/rejected": -387.18455141129033,
      "loss": 0.2534,
      "loss/base_kto": 2.027486562728882,
      "metrics/advantage_var": 13143.611328125,
      "rewards/chosen": 2.6688169537168562,
      "rewards/margins": 4.806769007648308,
      "rewards/rejected": -2.1379520539314516,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 7.692666053771973,
      "kl": 34.7623405456543,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -49127913.95215311,
      "logits/rejected": -49812977.10260337,
      "logps/chosen": -456.0070773524721,
      "logps/rejected": -436.2947454058193,
      "loss": 0.26,
      "loss/base_kto": 2.0802910327911377,
      "metrics/advantage_var": 14263.2255859375,
      "rewards/chosen": 2.7569722201455344,
      "rewards/margins": 4.696360290828919,
      "rewards/rejected": -1.9393880706833844,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 7.434744834899902,
      "kl": 12.84818172454834,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -50147663.30658106,
      "logits/rejected": -50447433.25418569,
      "logps/chosen": -444.97396669341896,
      "logps/rejected": -407.2007229832572,
      "loss": 0.2614,
      "loss/base_kto": 2.0915536880493164,
      "metrics/advantage_var": 15056.923828125,
      "rewards/chosen": 1.9565668733697834,
      "rewards/margins": 4.866010210448836,
      "rewards/rejected": -2.9094433370790527,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 6.86375093460083,
      "kl": 20.340803146362305,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -50494052.54380665,
      "logits/rejected": -50472234.25242718,
      "logps/chosen": -453.93249622356495,
      "logps/rejected": -396.7928802588997,
      "loss": 0.2556,
      "loss/base_kto": 2.045093536376953,
      "metrics/advantage_var": 13487.6455078125,
      "rewards/chosen": 2.620534844989143,
      "rewards/margins": 4.822372576014123,
      "rewards/rejected": -2.20183773102498,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 7.61278772354126,
      "kl": 11.203639030456543,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -49845101.26348228,
      "logits/rejected": -50176350.52931854,
      "logps/chosen": -462.35275423728814,
      "logps/rejected": -387.9735538827258,
      "loss": 0.2587,
      "loss/base_kto": 2.069570541381836,
      "metrics/advantage_var": 14044.2470703125,
      "rewards/chosen": 2.3151443551979005,
      "rewards/margins": 4.664339735794176,
      "rewards/rejected": -2.3491953805962758,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 6.555257797241211,
      "kl": 21.843231201171875,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -51787978.54945055,
      "logits/rejected": -52390773.44945568,
      "logps/chosen": -467.1393249607535,
      "logps/rejected": -391.87551030326597,
      "loss": 0.2603,
      "loss/base_kto": 2.082411289215088,
      "metrics/advantage_var": 14247.1123046875,
      "rewards/chosen": 2.302504223594486,
      "rewards/margins": 4.600613975246897,
      "rewards/rejected": -2.2981097516524107,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 9.335948944091797,
      "kl": 23.824705123901367,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -50488478.283911675,
      "logits/rejected": -49709978.551083595,
      "logps/chosen": -470.3002760252366,
      "logps/rejected": -389.8466282894737,
      "loss": 0.2699,
      "loss/base_kto": 2.1592204570770264,
      "metrics/advantage_var": 13056.5576171875,
      "rewards/chosen": 2.5590252319721016,
      "rewards/margins": 4.345054681529474,
      "rewards/rejected": -1.7860294495573723,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 5.899699687957764,
      "kl": 18.416074752807617,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -51300281.48982786,
      "logits/rejected": -51180459.332293294,
      "logps/chosen": -427.04631259780905,
      "logps/rejected": -393.7709877145086,
      "loss": 0.2491,
      "loss/base_kto": 1.9927533864974976,
      "metrics/advantage_var": 15910.4501953125,
      "rewards/chosen": 2.523304349753032,
      "rewards/margins": 5.119925382563094,
      "rewards/rejected": -2.5966210328100625,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 6.821751594543457,
      "kl": 28.01690673828125,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -50323961.43589743,
      "logits/rejected": -50340445.65853658,
      "logps/chosen": -475.4713541666667,
      "logps/rejected": -385.641982660061,
      "loss": 0.2514,
      "loss/base_kto": 2.011336326599121,
      "metrics/advantage_var": 13328.1220703125,
      "rewards/chosen": 2.7215857872596154,
      "rewards/margins": 4.759149948606796,
      "rewards/rejected": -2.0375641613471798,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 7.234650135040283,
      "kl": 25.949859619140625,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -51581178.65074627,
      "logits/rejected": -51804243.93442623,
      "logps/chosen": -473.0375,
      "logps/rejected": -406.4485911885246,
      "loss": 0.2553,
      "loss/base_kto": 2.0423760414123535,
      "metrics/advantage_var": 13629.1279296875,
      "rewards/chosen": 2.8382909063083024,
      "rewards/margins": 4.770606720217113,
      "rewards/rejected": -1.9323158139088115,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 5.109222888946533,
      "kl": 19.274444580078125,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -50138758.7368421,
      "logits/rejected": -50881342.1829653,
      "logps/chosen": -448.0441660216718,
      "logps/rejected": -384.5434739747634,
      "loss": 0.2523,
      "loss/base_kto": 2.018216371536255,
      "metrics/advantage_var": 12616.2705078125,
      "rewards/chosen": 2.5717410627902475,
      "rewards/margins": 4.748237983691766,
      "rewards/rejected": -2.176496920901518,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.433963775634766,
      "kl": 19.19364356994629,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -51527698.61818182,
      "logits/rejected": -49797925.98709677,
      "logps/chosen": -481.74583333333334,
      "logps/rejected": -386.2210181451613,
      "loss": 0.2463,
      "loss/base_kto": 1.970375418663025,
      "metrics/advantage_var": 13977.0859375,
      "rewards/chosen": 2.6085739598129734,
      "rewards/margins": 4.939082481108336,
      "rewards/rejected": -2.330508521295363,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 7.160104274749756,
      "kl": 19.392568588256836,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -51063385.21821036,
      "logits/rejected": -52179596.93934681,
      "logps/chosen": -487.41012558869704,
      "logps/rejected": -395.7419809486781,
      "loss": 0.2474,
      "loss/base_kto": 1.9794098138809204,
      "metrics/advantage_var": 13968.1201171875,
      "rewards/chosen": 2.3944735684237637,
      "rewards/margins": 4.971222572117387,
      "rewards/rejected": -2.5767490036936236,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 6.981439113616943,
      "kl": 20.005552291870117,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -50218137.52086553,
      "logits/rejected": -50817425.18799368,
      "logps/chosen": -451.67610123647603,
      "logps/rejected": -392.22013230647707,
      "loss": 0.2563,
      "loss/base_kto": 2.0505902767181396,
      "metrics/advantage_var": 12431.8095703125,
      "rewards/chosen": 2.4141296669725656,
      "rewards/margins": 4.498154539232637,
      "rewards/rejected": -2.084024872260071,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 6.792326927185059,
      "kl": 11.254304885864258,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -50549148.17610063,
      "logits/rejected": -50661592.2484472,
      "logps/chosen": -448.44752358490564,
      "logps/rejected": -397.70450795807454,
      "loss": 0.2511,
      "loss/base_kto": 2.0086920261383057,
      "metrics/advantage_var": 14288.7939453125,
      "rewards/chosen": 2.2998998869889937,
      "rewards/margins": 4.963305216533346,
      "rewards/rejected": -2.663405329544352,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 9.0466890335083,
      "kl": 16.8918514251709,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -49829091.376377955,
      "logits/rejected": -51076523.063565895,
      "logps/chosen": -457.84104330708664,
      "logps/rejected": -399.90581395348835,
      "loss": 0.2621,
      "loss/base_kto": 2.0966598987579346,
      "metrics/advantage_var": 13364.3486328125,
      "rewards/chosen": 2.2398460568405514,
      "rewards/margins": 4.591905165367683,
      "rewards/rejected": -2.352059108527132,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 8.03991413116455,
      "kl": 15.425745010375977,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -50888763.7471637,
      "logits/rejected": -53589345.689291105,
      "logps/chosen": -461.6916531604538,
      "logps/rejected": -386.0625471342383,
      "loss": 0.2486,
      "loss/base_kto": 1.9890469312667847,
      "metrics/advantage_var": 13639.9716796875,
      "rewards/chosen": 2.305420515409745,
      "rewards/margins": 4.8323844297314835,
      "rewards/rejected": -2.526963914321738,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 6.370032787322998,
      "kl": 26.204471588134766,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -51099644.680713125,
      "logits/rejected": -50851358.11764706,
      "logps/chosen": -466.8704923014587,
      "logps/rejected": -391.82850207390646,
      "loss": 0.2509,
      "loss/base_kto": 2.0070436000823975,
      "metrics/advantage_var": 14002.6279296875,
      "rewards/chosen": 2.7109810258812805,
      "rewards/margins": 4.972625760397683,
      "rewards/rejected": -2.261644734516403,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 8.21532917022705,
      "kl": 25.85410499572754,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50496609.37423313,
      "logits/rejected": -52054051.87261146,
      "logps/chosen": -466.07419478527606,
      "logps/rejected": -400.6569715366242,
      "loss": 0.2562,
      "loss/base_kto": 2.049304723739624,
      "metrics/advantage_var": 14128.8779296875,
      "rewards/chosen": 2.778415796946894,
      "rewards/margins": 4.772426187696297,
      "rewards/rejected": -1.994010390749403,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 8.312483787536621,
      "kl": 14.62979793548584,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -50865794.60406886,
      "logits/rejected": -50939977.48517941,
      "logps/chosen": -452.6448552425665,
      "logps/rejected": -407.5865347113885,
      "loss": 0.2436,
      "loss/base_kto": 1.9488365650177002,
      "metrics/advantage_var": 13803.8154296875,
      "rewards/chosen": 2.395782900528169,
      "rewards/margins": 5.012799006953676,
      "rewards/rejected": -2.617016106425507,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 9.717442512512207,
      "kl": 26.102514266967773,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -52090053.047318615,
      "logits/rejected": -51147164.136222914,
      "logps/chosen": -447.686415615142,
      "logps/rejected": -426.5359907120743,
      "loss": 0.2489,
      "loss/base_kto": 1.991180419921875,
      "metrics/advantage_var": 13899.3818359375,
      "rewards/chosen": 2.9700094036499407,
      "rewards/margins": 4.8545486505781525,
      "rewards/rejected": -1.884539246928212,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 7.653444766998291,
      "kl": 24.483278274536133,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -52653258.08291874,
      "logits/rejected": -52675346.52880354,
      "logps/chosen": -477.04477611940297,
      "logps/rejected": -373.59608105613,
      "loss": 0.2556,
      "loss/base_kto": 2.044577121734619,
      "metrics/advantage_var": 14217.7216796875,
      "rewards/chosen": 2.2566705453850537,
      "rewards/margins": 4.839719108092772,
      "rewards/rejected": -2.583048562707718,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 11.300180435180664,
      "kl": 18.176010131835938,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -51462939.58118361,
      "logits/rejected": -49455661.346215785,
      "logps/chosen": -484.98852427921094,
      "logps/rejected": -430.2469806763285,
      "loss": 0.2575,
      "loss/base_kto": 2.0600175857543945,
      "metrics/advantage_var": 13463.3359375,
      "rewards/chosen": 2.6043593735181148,
      "rewards/margins": 4.869201039870269,
      "rewards/rejected": -2.264841666352154,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 7.970808029174805,
      "kl": 24.66413688659668,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -52837982.63507109,
      "logits/rejected": -52683818.73261206,
      "logps/chosen": -474.78041074249603,
      "logps/rejected": -406.2067233384853,
      "loss": 0.2559,
      "loss/base_kto": 2.0468387603759766,
      "metrics/advantage_var": 14646.4560546875,
      "rewards/chosen": 2.4331729641834516,
      "rewards/margins": 4.927368869865928,
      "rewards/rejected": -2.494195905682477,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 7.020127296447754,
      "kl": 19.669519424438477,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -51798627.2969697,
      "logits/rejected": -53283086.86451613,
      "logps/chosen": -476.07651515151514,
      "logps/rejected": -379.72452116935483,
      "loss": 0.2366,
      "loss/base_kto": 1.8930256366729736,
      "metrics/advantage_var": 13605.1845703125,
      "rewards/chosen": 2.772806433475379,
      "rewards/margins": 5.087551739674975,
      "rewards/rejected": -2.3147453061995966,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 11.152055740356445,
      "kl": 17.177064895629883,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -48855737.461300306,
      "logits/rejected": -51137997.93059937,
      "logps/chosen": -483.4895510835913,
      "logps/rejected": -382.77481762618294,
      "loss": 0.2547,
      "loss/base_kto": 2.037907361984253,
      "metrics/advantage_var": 13901.1376953125,
      "rewards/chosen": 2.4697624655330883,
      "rewards/margins": 5.0859972779837985,
      "rewards/rejected": -2.6162348124507098,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 7.5428948402404785,
      "kl": 18.95989227294922,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -50488190.81846154,
      "logits/rejected": -52109048.68571428,
      "logps/chosen": -467.8317307692308,
      "logps/rejected": -389.0001488095238,
      "loss": 0.2513,
      "loss/base_kto": 2.010078191757202,
      "metrics/advantage_var": 14102.5361328125,
      "rewards/chosen": 2.512865459735577,
      "rewards/margins": 5.080259510454823,
      "rewards/rejected": -2.567394050719246,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 8.855107307434082,
      "kl": 21.101959228515625,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -52115810.77165354,
      "logits/rejected": -52631317.03565892,
      "logps/chosen": -454.15187007874016,
      "logps/rejected": -393.9096414728682,
      "loss": 0.2497,
      "loss/base_kto": 1.9974640607833862,
      "metrics/advantage_var": 13606.6240234375,
      "rewards/chosen": 2.5867137518454726,
      "rewards/margins": 4.852088555018922,
      "rewards/rejected": -2.2653748031734495,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 8.883349418640137,
      "kl": 16.477609634399414,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -52024444.411214955,
      "logits/rejected": -51389911.87460815,
      "logps/chosen": -467.2625097352025,
      "logps/rejected": -381.31132445141066,
      "loss": 0.2475,
      "loss/base_kto": 1.9803965091705322,
      "metrics/advantage_var": 12575.1845703125,
      "rewards/chosen": 2.3062188929857865,
      "rewards/margins": 4.80757085039664,
      "rewards/rejected": -2.5013519574108543,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.243531227111816,
      "kl": 17.660175323486328,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -52642564.14239482,
      "logits/rejected": -52224761.03927492,
      "logps/chosen": -465.48553802589,
      "logps/rejected": -408.9011518126888,
      "loss": 0.2633,
      "loss/base_kto": 2.1064612865448,
      "metrics/advantage_var": 14548.8095703125,
      "rewards/chosen": 2.3217147283955297,
      "rewards/margins": 4.738218819041772,
      "rewards/rejected": -2.4165040906462423,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 9.128072738647461,
      "kl": 29.375097274780273,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -50154355.14064915,
      "logits/rejected": -49508727.304897316,
      "logps/chosen": -436.9555158423493,
      "logps/rejected": -403.6753060821485,
      "loss": 0.2548,
      "loss/base_kto": 2.0382015705108643,
      "metrics/advantage_var": 13969.5693359375,
      "rewards/chosen": 2.3645096886471215,
      "rewards/margins": 4.911086165987169,
      "rewards/rejected": -2.5465764773400474,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 9.225976943969727,
      "kl": 22.192514419555664,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -53056889.77080063,
      "logits/rejected": -53704936.510108866,
      "logps/chosen": -470.7689364207221,
      "logps/rejected": -397.0832280326594,
      "loss": 0.2515,
      "loss/base_kto": 2.0117104053497314,
      "metrics/advantage_var": 13975.228515625,
      "rewards/chosen": 2.679721227433281,
      "rewards/margins": 4.832321647825971,
      "rewards/rejected": -2.1526004203926905,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 6.5635223388671875,
      "kl": 17.10091781616211,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -52599308.8,
      "logits/rejected": -52682214.4,
      "logps/chosen": -462.09482421875,
      "logps/rejected": -398.41201171875,
      "loss": 0.2551,
      "loss/base_kto": 2.040544033050537,
      "metrics/advantage_var": 15413.7578125,
      "rewards/chosen": 2.1388565063476563,
      "rewards/margins": 4.8688764572143555,
      "rewards/rejected": -2.730019950866699,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 8.384824752807617,
      "kl": 21.283100128173828,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -53873727.209876545,
      "logits/rejected": -52510023.29113924,
      "logps/chosen": -441.6039737654321,
      "logps/rejected": -390.2602600870253,
      "loss": 0.2497,
      "loss/base_kto": 1.9978981018066406,
      "metrics/advantage_var": 14391.3466796875,
      "rewards/chosen": 2.6387082323615934,
      "rewards/margins": 4.906072541165489,
      "rewards/rejected": -2.2673643088038964,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 9.146220207214355,
      "kl": 20.053346633911133,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -50999080.585365854,
      "logits/rejected": -51287384.615384616,
      "logps/chosen": -451.2948742378049,
      "logps/rejected": -393.21314102564105,
      "loss": 0.2599,
      "loss/base_kto": 2.078908681869507,
      "metrics/advantage_var": 14017.6591796875,
      "rewards/chosen": 2.7764029153963414,
      "rewards/margins": 4.802241914044178,
      "rewards/rejected": -2.0258389986478367,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 8.22525691986084,
      "kl": 20.598541259765625,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -52111584.25764895,
      "logits/rejected": -53519566.6646434,
      "logps/chosen": -478.40841384863126,
      "logps/rejected": -376.4315724582701,
      "loss": 0.2603,
      "loss/base_kto": 2.082355260848999,
      "metrics/advantage_var": 13044.7890625,
      "rewards/chosen": 2.4278499584842996,
      "rewards/margins": 4.723116927554386,
      "rewards/rejected": -2.295266969070087,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 8.048925399780273,
      "kl": 16.850187301635742,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -50448719.01234568,
      "logits/rejected": -51392998.07594936,
      "logps/chosen": -421.76957947530866,
      "logps/rejected": -394.25674940664555,
      "loss": 0.2535,
      "loss/base_kto": 2.0280842781066895,
      "metrics/advantage_var": 13453.8984375,
      "rewards/chosen": 2.5178375244140625,
      "rewards/margins": 4.781443921825554,
      "rewards/rejected": -2.2636063974114915,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 8.18259048461914,
      "kl": 22.198036193847656,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -52280503.45276873,
      "logits/rejected": -51407273.638554215,
      "logps/chosen": -429.43297027687294,
      "logps/rejected": -378.1626270707831,
      "loss": 0.2277,
      "loss/base_kto": 1.821925163269043,
      "metrics/advantage_var": 14607.0263671875,
      "rewards/chosen": 2.6215007173121947,
      "rewards/margins": 5.2629680760047775,
      "rewards/rejected": -2.6414673586925828,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 7.484233379364014,
      "kl": 12.324950218200684,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -51822439.590697676,
      "logits/rejected": -51960907.792125985,
      "logps/chosen": -461.7236918604651,
      "logps/rejected": -412.37598425196853,
      "loss": 0.2154,
      "loss/base_kto": 1.722840666770935,
      "metrics/advantage_var": 15416.4501953125,
      "rewards/chosen": 2.524923351199128,
      "rewards/margins": 5.516593734442238,
      "rewards/rejected": -2.99167038324311,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 7.056107521057129,
      "kl": 13.95053768157959,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -52132353.625396825,
      "logits/rejected": -53285191.68,
      "logps/chosen": -464.37132936507936,
      "logps/rejected": -383.63980769230767,
      "loss": 0.1986,
      "loss/base_kto": 1.5886918306350708,
      "metrics/advantage_var": 15551.6064453125,
      "rewards/chosen": 2.8239622085813494,
      "rewards/margins": 5.818112223605388,
      "rewards/rejected": -2.9941500150240383,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 7.444598197937012,
      "kl": 12.505131721496582,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -52138753.187017,
      "logits/rejected": -52800827.45023697,
      "logps/chosen": -445.2485510046368,
      "logps/rejected": -409.02399289099526,
      "loss": 0.2243,
      "loss/base_kto": 1.794498085975647,
      "metrics/advantage_var": 14556.9033203125,
      "rewards/chosen": 2.469342427791731,
      "rewards/margins": 5.232089615353441,
      "rewards/rejected": -2.7627471875617102,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 5.655614376068115,
      "kl": 13.41953182220459,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -52270431.52238806,
      "logits/rejected": -52245782.66229508,
      "logps/chosen": -466.24118470149256,
      "logps/rejected": -414.2294057377049,
      "loss": 0.2034,
      "loss/base_kto": 1.627501368522644,
      "metrics/advantage_var": 14324.3251953125,
      "rewards/chosen": 2.5005901279734144,
      "rewards/margins": 5.578346115422185,
      "rewards/rejected": -3.0777559874487705,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 7.273128509521484,
      "kl": 26.179365158081055,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -50385255.872204475,
      "logits/rejected": -52798060.036697246,
      "logps/chosen": -443.0647963258786,
      "logps/rejected": -388.5483323776758,
      "loss": 0.2091,
      "loss/base_kto": 1.6730972528457642,
      "metrics/advantage_var": 14618.921875,
      "rewards/chosen": 2.985624758199381,
      "rewards/margins": 5.507712951946992,
      "rewards/rejected": -2.522088193747611,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 6.6122283935546875,
      "kl": 22.36004066467285,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -52280876.75259259,
      "logits/rejected": -51246538.68429752,
      "logps/chosen": -463.63194444444446,
      "logps/rejected": -403.38119834710744,
      "loss": 0.1987,
      "loss/base_kto": 1.5892845392227173,
      "metrics/advantage_var": 14510.6015625,
      "rewards/chosen": 2.9543858506944445,
      "rewards/margins": 5.739317733442379,
      "rewards/rejected": -2.7849318827479337,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 7.424135684967041,
      "kl": 21.711545944213867,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -52776490.12658228,
      "logits/rejected": -52171839.209876545,
      "logps/chosen": -462.67948971518985,
      "logps/rejected": -395.7844569830247,
      "loss": 0.2083,
      "loss/base_kto": 1.666551947593689,
      "metrics/advantage_var": 15177.5224609375,
      "rewards/chosen": 2.850885125655162,
      "rewards/margins": 5.720151776950011,
      "rewards/rejected": -2.8692666512948497,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 9.241097450256348,
      "kl": 17.936187744140625,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -52626990.39745628,
      "logits/rejected": -54468249.3640553,
      "logps/chosen": -476.0680147058824,
      "logps/rejected": -378.2085733486943,
      "loss": 0.2002,
      "loss/base_kto": 1.6016310453414917,
      "metrics/advantage_var": 13864.609375,
      "rewards/chosen": 2.5726293130216615,
      "rewards/margins": 5.62121029720465,
      "rewards/rejected": -3.048580984182988,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 8.677379608154297,
      "kl": 17.456422805786133,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -52103591.39969372,
      "logits/rejected": -51538292.36363637,
      "logps/chosen": -459.0336906584992,
      "logps/rejected": -414.3886313795853,
      "loss": 0.2033,
      "loss/base_kto": 1.6267732381820679,
      "metrics/advantage_var": 14518.798828125,
      "rewards/chosen": 2.7230377898281968,
      "rewards/margins": 5.754052215540916,
      "rewards/rejected": -3.031014425712719,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 6.065794467926025,
      "kl": 23.13743782043457,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -51528764.8,
      "logits/rejected": -52346649.6,
      "logps/chosen": -478.83916015625,
      "logps/rejected": -425.9013671875,
      "loss": 0.1956,
      "loss/base_kto": 1.5644174814224243,
      "metrics/advantage_var": 15502.2568359375,
      "rewards/chosen": 3.003139877319336,
      "rewards/margins": 5.866967582702637,
      "rewards/rejected": -2.863827705383301,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 8.646085739135742,
      "kl": 19.484638214111328,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -53728719.08280255,
      "logits/rejected": -53097283.53374233,
      "logps/chosen": -474.18222531847135,
      "logps/rejected": -397.84077837423314,
      "loss": 0.2045,
      "loss/base_kto": 1.6361578702926636,
      "metrics/advantage_var": 16025.0693359375,
      "rewards/chosen": 2.7687875541152467,
      "rewards/margins": 5.923817599887869,
      "rewards/rejected": -3.1550300457726226,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 7.306393146514893,
      "kl": 18.7791690826416,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -52419255.54716981,
      "logits/rejected": -53539369.34161491,
      "logps/chosen": -458.7684257075472,
      "logps/rejected": -397.62533967391306,
      "loss": 0.2064,
      "loss/base_kto": 1.6515058279037476,
      "metrics/advantage_var": 14051.9248046875,
      "rewards/chosen": 2.7090047200520835,
      "rewards/margins": 5.527774281630112,
      "rewards/rejected": -2.818769561578028,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 7.299766540527344,
      "kl": 19.348020553588867,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -51917969.331158236,
      "logits/rejected": -53623284.48575712,
      "logps/chosen": -441.7658034257749,
      "logps/rejected": -408.68988943028484,
      "loss": 0.1881,
      "loss/base_kto": 1.5046846866607666,
      "metrics/advantage_var": 15916.6005859375,
      "rewards/chosen": 2.6475585140956364,
      "rewards/margins": 5.903848703961641,
      "rewards/rejected": -3.2562901898660046,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 9.079314231872559,
      "kl": 17.321130752563477,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -52036113.32923077,
      "logits/rejected": -52050241.82857143,
      "logps/chosen": -455.6421634615385,
      "logps/rejected": -395.63385416666665,
      "loss": 0.2054,
      "loss/base_kto": 1.6432907581329346,
      "metrics/advantage_var": 14563.306640625,
      "rewards/chosen": 2.6505673452524037,
      "rewards/margins": 5.668501411782662,
      "rewards/rejected": -3.017934066530258,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 7.00515079498291,
      "kl": 17.82425308227539,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -51968006.11343284,
      "logits/rejected": -54259913.44262295,
      "logps/chosen": -458.44272388059704,
      "logps/rejected": -416.29928278688527,
      "loss": 0.2076,
      "loss/base_kto": 1.6604769229888916,
      "metrics/advantage_var": 14892.939453125,
      "rewards/chosen": 2.832628119169776,
      "rewards/margins": 5.514739215161066,
      "rewards/rejected": -2.682111095991291,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.637384414672852,
      "kl": 26.615041732788086,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -51986283.17219589,
      "logits/rejected": -51022384.27202473,
      "logps/chosen": -472.0436413902054,
      "logps/rejected": -394.59010336166926,
      "loss": 0.2016,
      "loss/base_kto": 1.6125751733779907,
      "metrics/advantage_var": 14819.5888671875,
      "rewards/chosen": 3.209165648449842,
      "rewards/margins": 5.748421347831604,
      "rewards/rejected": -2.539255699381762,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 6.848017692565918,
      "kl": 9.814329147338867,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -53291416.371814094,
      "logits/rejected": -52309788.815660685,
      "logps/chosen": -464.25159295352324,
      "logps/rejected": -418.0154465742251,
      "loss": 0.2048,
      "loss/base_kto": 1.6382410526275635,
      "metrics/advantage_var": 16801.287109375,
      "rewards/chosen": 2.736009680706522,
      "rewards/margins": 5.989718387884336,
      "rewards/rejected": -3.253708707177814,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.6930036544799805,
      "kl": 16.38731575012207,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -51957545.636070855,
      "logits/rejected": -52754142.98027314,
      "logps/chosen": -444.3054549114332,
      "logps/rejected": -400.0576631259484,
      "loss": 0.21,
      "loss/base_kto": 1.680148959159851,
      "metrics/advantage_var": 15314.6279296875,
      "rewards/chosen": 2.6964651897141705,
      "rewards/margins": 5.63595980750628,
      "rewards/rejected": -2.9394946177921093,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 6.98551082611084,
      "kl": 24.062509536743164,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -53703912.44171779,
      "logits/rejected": -53829821.14649682,
      "logps/chosen": -446.0058953220859,
      "logps/rejected": -421.1812798566879,
      "loss": 0.2096,
      "loss/base_kto": 1.6764150857925415,
      "metrics/advantage_var": 16047.712890625,
      "rewards/chosen": 2.5807792803992524,
      "rewards/margins": 5.571978166371585,
      "rewards/rejected": -2.9911988859723326,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 7.3181562423706055,
      "kl": 15.073596000671387,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -54002159.69426752,
      "logits/rejected": -53693295.50920245,
      "logps/chosen": -464.53712181528664,
      "logps/rejected": -402.6618577453988,
      "loss": 0.2111,
      "loss/base_kto": 1.6886459589004517,
      "metrics/advantage_var": 15776.330078125,
      "rewards/chosen": 2.7690118680334397,
      "rewards/margins": 5.763737756525579,
      "rewards/rejected": -2.9947258884921397,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 6.888214588165283,
      "kl": 15.283075332641602,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -53212281.38601824,
      "logits/rejected": -53768587.11254019,
      "logps/chosen": -465.1954787234043,
      "logps/rejected": -408.79119774919616,
      "loss": 0.1973,
      "loss/base_kto": 1.5781694650650024,
      "metrics/advantage_var": 14977.0615234375,
      "rewards/chosen": 2.866462313295498,
      "rewards/margins": 5.860386273303536,
      "rewards/rejected": -2.9939239600080385,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 7.8260297775268555,
      "kl": 9.290362358093262,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -52013418.85885886,
      "logits/rejected": -52967787.570032574,
      "logps/chosen": -463.9862049549549,
      "logps/rejected": -396.20549165309444,
      "loss": 0.2102,
      "loss/base_kto": 1.6817753314971924,
      "metrics/advantage_var": 15169.3505859375,
      "rewards/chosen": 2.7301703559027777,
      "rewards/margins": 5.604926016466194,
      "rewards/rejected": -2.874755660563416,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 6.119743824005127,
      "kl": 13.453554153442383,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -53113685.0719755,
      "logits/rejected": -52189785.00797448,
      "logps/chosen": -470.26804173047475,
      "logps/rejected": -397.8858153907496,
      "loss": 0.2208,
      "loss/base_kto": 1.7666877508163452,
      "metrics/advantage_var": 16165.1123046875,
      "rewards/chosen": 2.440948252656011,
      "rewards/margins": 5.580639521844607,
      "rewards/rejected": -3.1396912691885963,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 7.360220432281494,
      "kl": 23.2429256439209,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -52083296.25276461,
      "logits/rejected": -53096606.26893354,
      "logps/chosen": -433.90496642969987,
      "logps/rejected": -402.75613408037094,
      "loss": 0.215,
      "loss/base_kto": 1.7202949523925781,
      "metrics/advantage_var": 15276.4013671875,
      "rewards/chosen": 2.571273249284163,
      "rewards/margins": 5.496104916775952,
      "rewards/rejected": -2.924831667491789,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 6.64047384262085,
      "kl": 15.690417289733887,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -53221443.516483516,
      "logits/rejected": -54514887.06687403,
      "logps/chosen": -462.11803375196234,
      "logps/rejected": -408.3264968895801,
      "loss": 0.2134,
      "loss/base_kto": 1.706957221031189,
      "metrics/advantage_var": 14106.6533203125,
      "rewards/chosen": 2.615590812647174,
      "rewards/margins": 5.379577830289087,
      "rewards/rejected": -2.763987017641913,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 6.458062171936035,
      "kl": 12.019271850585938,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -51360204.881141044,
      "logits/rejected": -53352842.45300462,
      "logps/chosen": -472.2121632329636,
      "logps/rejected": -391.82140793528504,
      "loss": 0.212,
      "loss/base_kto": 1.695715308189392,
      "metrics/advantage_var": 14661.03125,
      "rewards/chosen": 2.5609312707384113,
      "rewards/margins": 5.544705887937949,
      "rewards/rejected": -2.9837746171995376,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 8.161032676696777,
      "kl": 15.232950210571289,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -53328244.07453416,
      "logits/rejected": -53104240.704402514,
      "logps/chosen": -461.04396350931677,
      "logps/rejected": -410.7378390330189,
      "loss": 0.2054,
      "loss/base_kto": 1.6435362100601196,
      "metrics/advantage_var": 14669.265625,
      "rewards/chosen": 2.6601926436335406,
      "rewards/margins": 5.704665775880986,
      "rewards/rejected": -3.044473132247445,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 9.841913223266602,
      "kl": 13.96495246887207,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -52528856.79279279,
      "logits/rejected": -53861933.02931596,
      "logps/chosen": -466.80259947447445,
      "logps/rejected": -394.4695897801303,
      "loss": 0.1968,
      "loss/base_kto": 1.5746790170669556,
      "metrics/advantage_var": 15459.4501953125,
      "rewards/chosen": 2.742815080705706,
      "rewards/margins": 5.83864663798889,
      "rewards/rejected": -3.095831557283184,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 7.300918102264404,
      "kl": 20.875574111938477,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -51876761.6,
      "logits/rejected": -53363110.012121215,
      "logps/chosen": -465.5453629032258,
      "logps/rejected": -411.0222537878788,
      "loss": 0.207,
      "loss/base_kto": 1.655659556388855,
      "metrics/advantage_var": 15438.736328125,
      "rewards/chosen": 2.7058333858366934,
      "rewards/margins": 5.918691088843323,
      "rewards/rejected": -3.212857703006629,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 9.059916496276855,
      "kl": 17.868297576904297,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -52797909.52639517,
      "logits/rejected": -53588980.38249595,
      "logps/chosen": -470.601338612368,
      "logps/rejected": -413.7048723662885,
      "loss": 0.2025,
      "loss/base_kto": 1.619645118713379,
      "metrics/advantage_var": 13850.1376953125,
      "rewards/chosen": 2.6566224709535255,
      "rewards/margins": 5.526776061917869,
      "rewards/rejected": -2.8701535909643434,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 8.5919189453125,
      "kl": 16.793075561523438,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -54307853.04458599,
      "logits/rejected": -54554624.0,
      "logps/chosen": -469.29140127388536,
      "logps/rejected": -393.99012653374234,
      "loss": 0.2167,
      "loss/base_kto": 1.7337658405303955,
      "metrics/advantage_var": 14650.9267578125,
      "rewards/chosen": 2.7456175202776674,
      "rewards/margins": 5.435390050113749,
      "rewards/rejected": -2.6897725298360813,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 6.7300214767456055,
      "kl": 17.726110458374023,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -54416618.3322884,
      "logits/rejected": -53772032.797507785,
      "logps/chosen": -463.94734521943576,
      "logps/rejected": -392.9740800233645,
      "loss": 0.2173,
      "loss/base_kto": 1.7380943298339844,
      "metrics/advantage_var": 15046.2314453125,
      "rewards/chosen": 2.5639112705721003,
      "rewards/margins": 5.547979573665462,
      "rewards/rejected": -2.984068303093361,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 7.305049896240234,
      "kl": 21.30866813659668,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -52208295.368760064,
      "logits/rejected": -52240744.497723825,
      "logps/chosen": -469.00946054750403,
      "logps/rejected": -408.43996585735965,
      "loss": 0.2164,
      "loss/base_kto": 1.731019377708435,
      "metrics/advantage_var": 15013.5537109375,
      "rewards/chosen": 2.6099717881944446,
      "rewards/margins": 5.480226235332628,
      "rewards/rejected": -2.870254447138183,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 8.381829261779785,
      "kl": 14.172942161560059,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -52908790.88673139,
      "logits/rejected": -52410337.06344411,
      "logps/chosen": -431.7904530744337,
      "logps/rejected": -416.3897753021148,
      "loss": 0.2108,
      "loss/base_kto": 1.6866282224655151,
      "metrics/advantage_var": 16544.642578125,
      "rewards/chosen": 2.800618094533778,
      "rewards/margins": 5.837725625571354,
      "rewards/rejected": -3.0371075310375755,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 6.838298797607422,
      "kl": 19.05963897705078,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -52345202.84502447,
      "logits/rejected": -53091056.26386806,
      "logps/chosen": -448.6211256117455,
      "logps/rejected": -375.99433095952025,
      "loss": 0.212,
      "loss/base_kto": 1.695879340171814,
      "metrics/advantage_var": 14115.3251953125,
      "rewards/chosen": 2.662252951990722,
      "rewards/margins": 5.491003493712048,
      "rewards/rejected": -2.828750541721327,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.845784664154053,
      "kl": 15.954147338867188,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -53338313.030674845,
      "logits/rejected": -50993624.866242036,
      "logps/chosen": -424.1664110429448,
      "logps/rejected": -384.4034136146497,
      "loss": 0.2162,
      "loss/base_kto": 1.729690432548523,
      "metrics/advantage_var": 14194.8515625,
      "rewards/chosen": 2.4134079634777605,
      "rewards/margins": 5.440207450937951,
      "rewards/rejected": -3.026799487460191,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 5.568167686462402,
      "kl": 14.366287231445312,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -53023664.98267716,
      "logits/rejected": -53818925.24651163,
      "logps/chosen": -446.8387303149606,
      "logps/rejected": -400.86308139534884,
      "loss": 0.2198,
      "loss/base_kto": 1.7587683200836182,
      "metrics/advantage_var": 14852.1162109375,
      "rewards/chosen": 2.6796111820250985,
      "rewards/margins": 5.5208675410367265,
      "rewards/rejected": -2.841256359011628,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.97585917447635e+17,
      "train_loss": 0.27257865101150486,
      "train_runtime": 11039.5355,
      "train_samples_per_second": 13.426,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.97585917447635e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}