{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.319388389587402,
      "kl": 16.727964401245117,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37122425.3480315,
      "logits/rejected": -37737895.88837209,
      "logps/chosen": -463.6321358267717,
      "logps/rejected": -374.01157945736435,
      "loss": 0.4688,
      "loss/base_kto": 3.7502057552337646,
      "metrics/advantage_var": 476.1357421875,
      "rewards/chosen": 0.39666541392408955,
      "rewards/margins": 0.25662812570188753,
      "rewards/rejected": 0.14003728822220204,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.064275741577148,
      "kl": 13.24753475189209,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37318842.18181818,
      "logits/rejected": -38839044.46275753,
      "logps/chosen": -480.88530431432974,
      "logps/rejected": -376.75906299524564,
      "loss": 0.4448,
      "loss/base_kto": 3.558764696121216,
      "metrics/advantage_var": 669.9764404296875,
      "rewards/chosen": 0.49594534710853233,
      "rewards/margins": 0.5053853006420133,
      "rewards/rejected": -0.009439953533480927,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.0316801071167,
      "kl": 12.896925926208496,
      "learning_rate": 5.9e-07,
      "logits/chosen": -37219113.674418606,
      "logits/rejected": -37509648.93228346,
      "logps/chosen": -470.2107073643411,
      "logps/rejected": -360.15996555118113,
      "loss": 0.426,
      "loss/base_kto": 3.4081313610076904,
      "metrics/advantage_var": 1473.3863525390625,
      "rewards/chosen": 0.5174255844234497,
      "rewards/margins": 0.6775934647445618,
      "rewards/rejected": -0.1601678803211122,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 9.75800609588623,
      "kl": 22.625085830688477,
      "learning_rate": 7.9e-07,
      "logits/chosen": -40073294.40735069,
      "logits/rejected": -40570089.543859646,
      "logps/chosen": -483.6102603369066,
      "logps/rejected": -376.1366128389155,
      "loss": 0.42,
      "loss/base_kto": 3.360121965408325,
      "metrics/advantage_var": 2046.6541748046875,
      "rewards/chosen": 0.9029619529455398,
      "rewards/margins": 0.7994733654489663,
      "rewards/rejected": 0.10348858749657347,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 10.479662895202637,
      "kl": 7.0752153396606445,
      "learning_rate": 9.9e-07,
      "logits/chosen": -39373104.80267558,
      "logits/rejected": -39785216.75073314,
      "logps/chosen": -505.0452550167224,
      "logps/rejected": -389.1406708211144,
      "loss": 0.4144,
      "loss/base_kto": 3.315232038497925,
      "metrics/advantage_var": 2967.476806640625,
      "rewards/chosen": 0.3449013496322377,
      "rewards/margins": 1.078456133786151,
      "rewards/rejected": -0.7335547841539132,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 7.468899726867676,
      "kl": 29.222803115844727,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -38817840.07511737,
      "logits/rejected": -39936057.510140404,
      "logps/chosen": -447.14568661971833,
      "logps/rejected": -359.3209584633385,
      "loss": 0.4112,
      "loss/base_kto": 3.2892253398895264,
      "metrics/advantage_var": 3382.028076171875,
      "rewards/chosen": 0.9050804938135759,
      "rewards/margins": 0.9898408294174298,
      "rewards/rejected": -0.08476033560385385,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 8.750174522399902,
      "kl": 12.275041580200195,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -42091332.37267081,
      "logits/rejected": -41663439.69811321,
      "logps/chosen": -491.394264363354,
      "logps/rejected": -385.76407724056605,
      "loss": 0.3905,
      "loss/base_kto": 3.1238603591918945,
      "metrics/advantage_var": 4545.78857421875,
      "rewards/chosen": 0.9180897777865392,
      "rewards/margins": 1.4688283223706096,
      "rewards/rejected": -0.5507385445840703,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 8.236615180969238,
      "kl": 16.803390502929688,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -40913541.08868501,
      "logits/rejected": -40773331.01597444,
      "logps/chosen": -465.6788035168196,
      "logps/rejected": -370.9947084664537,
      "loss": 0.3854,
      "loss/base_kto": 3.083263635635376,
      "metrics/advantage_var": 4547.63671875,
      "rewards/chosen": 0.8942556585376051,
      "rewards/margins": 1.597708844377761,
      "rewards/rejected": -0.7034531858401557,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 8.821699142456055,
      "kl": 21.234865188598633,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -41670907.288343556,
      "logits/rejected": -41453613.65605096,
      "logps/chosen": -453.00896280674846,
      "logps/rejected": -369.6968053343949,
      "loss": 0.3838,
      "loss/base_kto": 3.070784091949463,
      "metrics/advantage_var": 4485.45849609375,
      "rewards/chosen": 1.1577457357769363,
      "rewards/margins": 1.5722619660746324,
      "rewards/rejected": -0.4145162302976961,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 9.399519920349121,
      "kl": 16.941009521484375,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -43490414.276923075,
      "logits/rejected": -42209328.76190476,
      "logps/chosen": -481.90163461538464,
      "logps/rejected": -407.65835813492066,
      "loss": 0.3737,
      "loss/base_kto": 2.9893739223480225,
      "metrics/advantage_var": 5094.6171875,
      "rewards/chosen": 1.177627704326923,
      "rewards/margins": 1.7559307719732522,
      "rewards/rejected": -0.5783030676463293,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 8.513607025146484,
      "kl": 19.285432815551758,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -42722276.67189953,
      "logits/rejected": -41917699.58320373,
      "logps/chosen": -482.56323587127156,
      "logps/rejected": -386.4064930015552,
      "loss": 0.3646,
      "loss/base_kto": 2.9167091846466064,
      "metrics/advantage_var": 6109.7998046875,
      "rewards/chosen": 1.1926577102445546,
      "rewards/margins": 1.9864442174313255,
      "rewards/rejected": -0.793786507186771,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 11.65726375579834,
      "kl": 14.581194877624512,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -44301136.36245955,
      "logits/rejected": -43484528.145015106,
      "logps/chosen": -471.87591019417476,
      "logps/rejected": -390.4674046450151,
      "loss": 0.3696,
      "loss/base_kto": 2.956815719604492,
      "metrics/advantage_var": 6047.51123046875,
      "rewards/chosen": 0.8881642363218042,
      "rewards/margins": 1.9577712362982016,
      "rewards/rejected": -1.0696069999763973,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 9.075246810913086,
      "kl": 13.598471641540527,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -43582794.21498372,
      "logits/rejected": -43640010.95495495,
      "logps/chosen": -507.21686685667754,
      "logps/rejected": -383.68947072072075,
      "loss": 0.3573,
      "loss/base_kto": 2.8587589263916016,
      "metrics/advantage_var": 6647.6220703125,
      "rewards/chosen": 1.0324669257049064,
      "rewards/margins": 2.329373634659955,
      "rewards/rejected": -1.2969067089550488,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.176731586456299,
      "kl": 17.565298080444336,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -44210205.4912,
      "logits/rejected": -44493703.62137405,
      "logps/chosen": -507.2881,
      "logps/rejected": -404.58382633587786,
      "loss": 0.3346,
      "loss/base_kto": 2.6767992973327637,
      "metrics/advantage_var": 6408.30615234375,
      "rewards/chosen": 0.931298828125,
      "rewards/margins": 2.5220157070014313,
      "rewards/rejected": -1.5907168788764312,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 8.242661476135254,
      "kl": 9.632750511169434,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -43166918.4,
      "logits/rejected": -43010860.8,
      "logps/chosen": -483.6794921875,
      "logps/rejected": -361.163037109375,
      "loss": 0.3398,
      "loss/base_kto": 2.7182796001434326,
      "metrics/advantage_var": 7442.8330078125,
      "rewards/chosen": 1.415898895263672,
      "rewards/margins": 2.555503082275391,
      "rewards/rejected": -1.1396041870117188,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 6.161278247833252,
      "kl": 11.036481857299805,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -43577174.89908257,
      "logits/rejected": -45461203.01597444,
      "logps/chosen": -480.4206804281346,
      "logps/rejected": -401.44393969648564,
      "loss": 0.3462,
      "loss/base_kto": 2.7698185443878174,
      "metrics/advantage_var": 8291.142578125,
      "rewards/chosen": 0.9927509086212252,
      "rewards/margins": 2.7852506824206458,
      "rewards/rejected": -1.7924997737994208,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 9.249044418334961,
      "kl": 28.86688804626465,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -46465939.79785605,
      "logits/rejected": -45670164.82296651,
      "logps/chosen": -476.4023258039816,
      "logps/rejected": -384.04410885167465,
      "loss": 0.3626,
      "loss/base_kto": 2.9010062217712402,
      "metrics/advantage_var": 7303.671875,
      "rewards/chosen": 1.7468029916012635,
      "rewards/margins": 2.238367095842691,
      "rewards/rejected": -0.49156410424142744,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 9.388398170471191,
      "kl": 25.781204223632812,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -45289124.102564104,
      "logits/rejected": -45208925.65853658,
      "logps/chosen": -481.48011818910254,
      "logps/rejected": -395.6542492378049,
      "loss": 0.3361,
      "loss/base_kto": 2.6887481212615967,
      "metrics/advantage_var": 8222.0947265625,
      "rewards/chosen": 1.5168069692758412,
      "rewards/margins": 2.7474430077071483,
      "rewards/rejected": -1.230636038431307,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 9.163446426391602,
      "kl": 19.06619644165039,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -44553825.940902025,
      "logits/rejected": -44534547.692307696,
      "logps/chosen": -471.5203635303266,
      "logps/rejected": -364.72792386185245,
      "loss": 0.3452,
      "loss/base_kto": 2.7618725299835205,
      "metrics/advantage_var": 7323.47998046875,
      "rewards/chosen": 1.4123068137636081,
      "rewards/margins": 2.6482218735224876,
      "rewards/rejected": -1.2359150597588795,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 8.153302192687988,
      "kl": 22.222400665283203,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -44122445.98153846,
      "logits/rejected": -45661642.36190476,
      "logps/chosen": -492.3990865384615,
      "logps/rejected": -374.6800843253968,
      "loss": 0.3357,
      "loss/base_kto": 2.6852593421936035,
      "metrics/advantage_var": 8430.4921875,
      "rewards/chosen": 1.713446326622596,
      "rewards/margins": 2.7226771673964056,
      "rewards/rejected": -1.0092308407738095,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 7.898141860961914,
      "kl": 20.997982025146484,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -46517715.75308642,
      "logits/rejected": -45683141.67088608,
      "logps/chosen": -496.25810185185185,
      "logps/rejected": -403.4624456091772,
      "loss": 0.3356,
      "loss/base_kto": 2.6848278045654297,
      "metrics/advantage_var": 9729.1015625,
      "rewards/chosen": 1.6485524118682484,
      "rewards/margins": 3.0480907658968777,
      "rewards/rejected": -1.3995383540286293,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 8.437301635742188,
      "kl": 27.33742332458496,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -47016659.97452229,
      "logits/rejected": -46722393.521472394,
      "logps/chosen": -472.54428742038215,
      "logps/rejected": -388.16971817484665,
      "loss": 0.3348,
      "loss/base_kto": 2.678339719772339,
      "metrics/advantage_var": 10187.8037109375,
      "rewards/chosen": 1.7116751701209196,
      "rewards/margins": 2.885346120232356,
      "rewards/rejected": -1.173670950111436,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 10.420513153076172,
      "kl": 29.8809814453125,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -45925395.14018691,
      "logits/rejected": -45740982.169278994,
      "logps/chosen": -458.6536214953271,
      "logps/rejected": -377.3076998432602,
      "loss": 0.3142,
      "loss/base_kto": 2.513406276702881,
      "metrics/advantage_var": 9882.564453125,
      "rewards/chosen": 1.8956435729410046,
      "rewards/margins": 3.2017841497251442,
      "rewards/rejected": -1.3061405767841399,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 8.041691780090332,
      "kl": 29.984716415405273,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -46290135.748427674,
      "logits/rejected": -46272499.27950311,
      "logps/chosen": -453.2558962264151,
      "logps/rejected": -389.4093070652174,
      "loss": 0.3321,
      "loss/base_kto": 2.656989336013794,
      "metrics/advantage_var": 9887.2763671875,
      "rewards/chosen": 2.058879348466981,
      "rewards/margins": 2.9575133172230466,
      "rewards/rejected": -0.8986339687560656,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 8.568934440612793,
      "kl": 14.765207290649414,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -46030885.57798165,
      "logits/rejected": -44988641.73801917,
      "logps/chosen": -455.4039564220183,
      "logps/rejected": -384.76402755591056,
      "loss": 0.3225,
      "loss/base_kto": 2.579798936843872,
      "metrics/advantage_var": 9816.142578125,
      "rewards/chosen": 1.8881363708309442,
      "rewards/margins": 3.233141589044303,
      "rewards/rejected": -1.3450052182133587,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 6.7368597984313965,
      "kl": 30.476104736328125,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -45823653.71980676,
      "logits/rejected": -45456227.05918058,
      "logps/chosen": -443.19444444444446,
      "logps/rejected": -379.547467754173,
      "loss": 0.3302,
      "loss/base_kto": 2.6419026851654053,
      "metrics/advantage_var": 9941.6689453125,
      "rewards/chosen": 1.8815548416113124,
      "rewards/margins": 3.1469455184773025,
      "rewards/rejected": -1.2653906768659902,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 9.077353477478027,
      "kl": 32.544036865234375,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -46674559.04903418,
      "logits/rejected": -47708240.9752883,
      "logps/chosen": -480.0802841753343,
      "logps/rejected": -386.7947899505766,
      "loss": 0.316,
      "loss/base_kto": 2.5276782512664795,
      "metrics/advantage_var": 10337.0224609375,
      "rewards/chosen": 2.4605105259797546,
      "rewards/margins": 3.265589274403097,
      "rewards/rejected": -0.8050787484233423,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 6.8249897956848145,
      "kl": 51.18595504760742,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -46264067.2,
      "logits/rejected": -47504531.2,
      "logps/chosen": -462.7865234375,
      "logps/rejected": -388.075048828125,
      "loss": 0.3242,
      "loss/base_kto": 2.593670606613159,
      "metrics/advantage_var": 9831.8037109375,
      "rewards/chosen": 2.6040317535400392,
      "rewards/margins": 3.0919196128845217,
      "rewards/rejected": -0.4878878593444824,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 5.395700931549072,
      "kl": 16.848234176635742,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -46107721.37071651,
      "logits/rejected": -46889512.12539185,
      "logps/chosen": -458.63721767912773,
      "logps/rejected": -370.9887343260188,
      "loss": 0.3291,
      "loss/base_kto": 2.632402181625366,
      "metrics/advantage_var": 11003.7841796875,
      "rewards/chosen": 1.5970843068535825,
      "rewards/margins": 3.241402412138653,
      "rewards/rejected": -1.6443181052850706,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.256164073944092,
      "kl": 40.421199798583984,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -47410145.6474259,
      "logits/rejected": -48573313.40219092,
      "logps/chosen": -472.3001170046802,
      "logps/rejected": -365.27787558685446,
      "loss": 0.3276,
      "loss/base_kto": 2.620436668395996,
      "metrics/advantage_var": 12412.2998046875,
      "rewards/chosen": 2.108003089654836,
      "rewards/margins": 2.9913788514958473,
      "rewards/rejected": -0.8833757618410113,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 9.761085510253906,
      "kl": 29.126300811767578,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -47805701.14374034,
      "logits/rejected": -47122011.39968404,
      "logps/chosen": -470.47498068006183,
      "logps/rejected": -376.11349723538706,
      "loss": 0.3132,
      "loss/base_kto": 2.5052177906036377,
      "metrics/advantage_var": 11753.0498046875,
      "rewards/chosen": 2.456938235063273,
      "rewards/margins": 3.7659804440075364,
      "rewards/rejected": -1.3090422089442635,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 7.527853965759277,
      "kl": 18.0603084564209,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -46224885.10638298,
      "logits/rejected": -47408901.76205788,
      "logps/chosen": -451.393047112462,
      "logps/rejected": -392.36600683279744,
      "loss": 0.3303,
      "loss/base_kto": 2.6424295902252197,
      "metrics/advantage_var": 12895.171875,
      "rewards/chosen": 1.7432306631720176,
      "rewards/margins": 3.2953440888653454,
      "rewards/rejected": -1.552113425693328,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 7.182866096496582,
      "kl": 20.800764083862305,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -47058805.152542375,
      "logits/rejected": -47342600.1141046,
      "logps/chosen": -462.9138097072419,
      "logps/rejected": -400.24985142630743,
      "loss": 0.3148,
      "loss/base_kto": 2.5186355113983154,
      "metrics/advantage_var": 13971.6015625,
      "rewards/chosen": 2.1760500303953196,
      "rewards/margins": 3.83824800793593,
      "rewards/rejected": -1.6621979775406102,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 8.891425132751465,
      "kl": 17.636144638061523,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -44455313.82278481,
      "logits/rejected": -44785935.802469134,
      "logps/chosen": -461.2084157436709,
      "logps/rejected": -389.9142554012346,
      "loss": 0.3401,
      "loss/base_kto": 2.7205283641815186,
      "metrics/advantage_var": 13725.75,
      "rewards/chosen": 1.3341555052165743,
      "rewards/margins": 3.3661077862736732,
      "rewards/rejected": -2.031952281057099,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 7.126125812530518,
      "kl": 20.660459518432617,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -45936999.52351097,
      "logits/rejected": -46289495.7258567,
      "logps/chosen": -450.60624020376173,
      "logps/rejected": -373.2173870716511,
      "loss": 0.3209,
      "loss/base_kto": 2.567134141921997,
      "metrics/advantage_var": 12918.2548828125,
      "rewards/chosen": 1.6922259196218652,
      "rewards/margins": 3.6139123000735784,
      "rewards/rejected": -1.9216863804517135,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 6.310122013092041,
      "kl": 27.32685661315918,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -47984717.40472441,
      "logits/rejected": -47533724.37829457,
      "logps/chosen": -470.71077755905515,
      "logps/rejected": -380.2222625968992,
      "loss": 0.3195,
      "loss/base_kto": 2.5560810565948486,
      "metrics/advantage_var": 14027.7880859375,
      "rewards/chosen": 2.4276184562623033,
      "rewards/margins": 3.788381461955133,
      "rewards/rejected": -1.3607630056928295,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 10.711471557617188,
      "kl": 18.596837997436523,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -47200074.45101088,
      "logits/rejected": -48807614.492935635,
      "logps/chosen": -459.070567651633,
      "logps/rejected": -409.03264815541604,
      "loss": 0.3205,
      "loss/base_kto": 2.564175844192505,
      "metrics/advantage_var": 17230.35546875,
      "rewards/chosen": 1.8940837854174766,
      "rewards/margins": 4.052348233023443,
      "rewards/rejected": -2.1582644476059656,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 5.282236576080322,
      "kl": 16.107288360595703,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -45939061.867528275,
      "logits/rejected": -45671859.3161876,
      "logps/chosen": -464.7586833602585,
      "logps/rejected": -385.3533472012103,
      "loss": 0.324,
      "loss/base_kto": 2.5922391414642334,
      "metrics/advantage_var": 14907.953125,
      "rewards/chosen": 1.3830086210432653,
      "rewards/margins": 3.8709718603350582,
      "rewards/rejected": -2.4879632392917927,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 6.966801643371582,
      "kl": 33.476539611816406,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47285591.44358578,
      "logits/rejected": -47422079.39144216,
      "logps/chosen": -481.49927550231837,
      "logps/rejected": -387.48085875594296,
      "loss": 0.2973,
      "loss/base_kto": 2.3787217140197754,
      "metrics/advantage_var": 14550.7412109375,
      "rewards/chosen": 2.6301737436002703,
      "rewards/margins": 4.041040555023111,
      "rewards/rejected": -1.4108668114228406,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 7.4706525802612305,
      "kl": 26.45621681213379,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -46360979.2,
      "logits/rejected": -46993430.4,
      "logps/chosen": -458.044189453125,
      "logps/rejected": -392.3923828125,
      "loss": 0.27,
      "loss/base_kto": 2.159776210784912,
      "metrics/advantage_var": 14934.84375,
      "rewards/chosen": 2.360272026062012,
      "rewards/margins": 4.595606422424316,
      "rewards/rejected": -2.2353343963623047,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 7.161087512969971,
      "kl": 20.279638290405273,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -46109183.23922734,
      "logits/rejected": -47211197.78583196,
      "logps/chosen": -464.3273588410104,
      "logps/rejected": -389.81806013179573,
      "loss": 0.2719,
      "loss/base_kto": 2.175161123275757,
      "metrics/advantage_var": 14320.359375,
      "rewards/chosen": 2.531824256477526,
      "rewards/margins": 4.456279826905862,
      "rewards/rejected": -1.9244555704283361,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 4.648787021636963,
      "kl": 22.47624397277832,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -48343649.215189874,
      "logits/rejected": -47700252.44444445,
      "logps/chosen": -455.1236155063291,
      "logps/rejected": -385.0729166666667,
      "loss": 0.255,
      "loss/base_kto": 2.0397450923919678,
      "metrics/advantage_var": 13408.5517578125,
      "rewards/chosen": 2.3995195219788372,
      "rewards/margins": 4.7239884407674175,
      "rewards/rejected": -2.3244689187885803,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 8.42148494720459,
      "kl": 18.592853546142578,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -47214087.78115501,
      "logits/rejected": -49079750.37942122,
      "logps/chosen": -466.7443009118541,
      "logps/rejected": -395.5042453778135,
      "loss": 0.2471,
      "loss/base_kto": 1.9770435094833374,
      "metrics/advantage_var": 15593.9033203125,
      "rewards/chosen": 2.7582436419547873,
      "rewards/margins": 5.262467431770904,
      "rewards/rejected": -2.5042237898161175,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.439265727996826,
      "kl": 11.96108341217041,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -46719675.470769234,
      "logits/rejected": -48405676.29206349,
      "logps/chosen": -473.23932692307693,
      "logps/rejected": -387.12862103174604,
      "loss": 0.2452,
      "loss/base_kto": 1.9619816541671753,
      "metrics/advantage_var": 16483.15625,
      "rewards/chosen": 2.296476487379808,
      "rewards/margins": 5.290778710222069,
      "rewards/rejected": -2.9943022228422618,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 7.7969584465026855,
      "kl": 32.29633331298828,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -47421925.218461536,
      "logits/rejected": -48460608.203174606,
      "logps/chosen": -468.878125,
      "logps/rejected": -376.4671626984127,
      "loss": 0.2681,
      "loss/base_kto": 2.1451966762542725,
      "metrics/advantage_var": 14988.861328125,
      "rewards/chosen": 2.5041038161057694,
      "rewards/margins": 4.649634298806662,
      "rewards/rejected": -2.145530482700893,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 7.675873279571533,
      "kl": 29.6217098236084,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -47351358.43902439,
      "logits/rejected": -49409986.40601504,
      "logps/chosen": -454.938262195122,
      "logps/rejected": -417.4201127819549,
      "loss": 0.2552,
      "loss/base_kto": 2.0416419506073,
      "metrics/advantage_var": 15909.7724609375,
      "rewards/chosen": 2.3638814786585365,
      "rewards/margins": 5.011706607887859,
      "rewards/rejected": -2.6478251292293233,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 7.191428184509277,
      "kl": 25.549884796142578,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -48202389.08485857,
      "logits/rejected": -47982344.671575844,
      "logps/chosen": -471.9688019966722,
      "logps/rejected": -407.2490795287187,
      "loss": 0.2533,
      "loss/base_kto": 2.026344060897827,
      "metrics/advantage_var": 15896.3876953125,
      "rewards/chosen": 2.4040238924708817,
      "rewards/margins": 4.864776579096434,
      "rewards/rejected": -2.4607526866255522,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 8.419614791870117,
      "kl": 16.007797241210938,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -49458072.95597484,
      "logits/rejected": -50207880.745341614,
      "logps/chosen": -478.63674331761007,
      "logps/rejected": -393.54697204968943,
      "loss": 0.2544,
      "loss/base_kto": 2.0351531505584717,
      "metrics/advantage_var": 15565.5380859375,
      "rewards/chosen": 2.083188230886399,
      "rewards/margins": 4.963903168653285,
      "rewards/rejected": -2.8807149377668866,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 11.954970359802246,
      "kl": 22.1839542388916,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -48730670.54545455,
      "logits/rejected": -50354456.67469879,
      "logps/chosen": -483.3146814123377,
      "logps/rejected": -423.3035579819277,
      "loss": 0.2647,
      "loss/base_kto": 2.1177659034729004,
      "metrics/advantage_var": 13992.3408203125,
      "rewards/chosen": 2.2816132384461243,
      "rewards/margins": 4.731054068171275,
      "rewards/rejected": -2.4494408297251504,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 8.50296401977539,
      "kl": 22.303434371948242,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -48743870.03680982,
      "logits/rejected": -48878109.35031847,
      "logps/chosen": -458.819018404908,
      "logps/rejected": -404.78393710191085,
      "loss": 0.2466,
      "loss/base_kto": 1.9726238250732422,
      "metrics/advantage_var": 14543.2841796875,
      "rewards/chosen": 2.665093357577646,
      "rewards/margins": 5.108054060267236,
      "rewards/rejected": -2.44296070268959,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 6.078616619110107,
      "kl": 23.710634231567383,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -48693167.32121212,
      "logits/rejected": -50649269.67741936,
      "logps/chosen": -471.01226325757574,
      "logps/rejected": -380.1893397177419,
      "loss": 0.2634,
      "loss/base_kto": 2.107532501220703,
      "metrics/advantage_var": 13206.234375,
      "rewards/chosen": 2.6341057054924244,
      "rewards/margins": 4.685351807117929,
      "rewards/rejected": -2.051246101625504,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.119199752807617,
      "kl": 23.26380729675293,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -49470722.77829457,
      "logits/rejected": -50942855.055118114,
      "logps/chosen": -468.51017441860466,
      "logps/rejected": -386.84995078740155,
      "loss": 0.2521,
      "loss/base_kto": 2.016773223876953,
      "metrics/advantage_var": 13627.8154296875,
      "rewards/chosen": 2.7830914637839146,
      "rewards/margins": 4.81463481700734,
      "rewards/rejected": -2.0315433532234253,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 8.171038627624512,
      "kl": 23.872556686401367,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -50876945.766561516,
      "logits/rejected": -51860790.6873065,
      "logps/chosen": -471.33147673501577,
      "logps/rejected": -376.60531153250776,
      "loss": 0.2431,
      "loss/base_kto": 1.9447319507598877,
      "metrics/advantage_var": 13762.4755859375,
      "rewards/chosen": 2.4297029031940065,
      "rewards/margins": 5.0148431668961155,
      "rewards/rejected": -2.585140263702109,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 8.106106758117676,
      "kl": 16.061141967773438,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -49467057.63265306,
      "logits/rejected": -49775573.00155521,
      "logps/chosen": -453.4469191522763,
      "logps/rejected": -412.4076594090202,
      "loss": 0.262,
      "loss/base_kto": 2.0961430072784424,
      "metrics/advantage_var": 13220.462890625,
      "rewards/chosen": 2.22434262166405,
      "rewards/margins": 4.581668814801686,
      "rewards/rejected": -2.357326193137636,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 8.971443176269531,
      "kl": 20.7944393157959,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -51214772.86535552,
      "logits/rejected": -50312550.978998385,
      "logps/chosen": -445.6907148260212,
      "logps/rejected": -378.6051342891761,
      "loss": 0.2542,
      "loss/base_kto": 2.0339062213897705,
      "metrics/advantage_var": 12345.4638671875,
      "rewards/chosen": 2.8841517646085477,
      "rewards/margins": 4.582793096393685,
      "rewards/rejected": -1.6986413317851372,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.93839168548584,
      "kl": 27.64991569519043,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -50742585.2044374,
      "logits/rejected": -51969685.103235744,
      "logps/chosen": -455.51550118858955,
      "logps/rejected": -418.8906009244992,
      "loss": 0.2555,
      "loss/base_kto": 2.043959379196167,
      "metrics/advantage_var": 14285.34375,
      "rewards/chosen": 2.7165968421899764,
      "rewards/margins": 4.896977265196525,
      "rewards/rejected": -2.1803804230065484,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 7.332820892333984,
      "kl": 27.189178466796875,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -51423125.40063091,
      "logits/rejected": -50545923.9628483,
      "logps/chosen": -452.23718454258676,
      "logps/rejected": -400.2590944272446,
      "loss": 0.2566,
      "loss/base_kto": 2.0524163246154785,
      "metrics/advantage_var": 14592.6845703125,
      "rewards/chosen": 2.5982067216458007,
      "rewards/margins": 4.938431799649864,
      "rewards/rejected": -2.3402250780040634,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 7.5371832847595215,
      "kl": 19.167118072509766,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -49825617.63467492,
      "logits/rejected": -51343172.643533126,
      "logps/chosen": -486.61992066563465,
      "logps/rejected": -386.4467665615142,
      "loss": 0.2356,
      "loss/base_kto": 1.8851795196533203,
      "metrics/advantage_var": 14707.625,
      "rewards/chosen": 2.4065221071981426,
      "rewards/margins": 5.221589434559144,
      "rewards/rejected": -2.8150673273610014,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 9.608060836791992,
      "kl": 22.010770797729492,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -51650882.49350649,
      "logits/rejected": -51695082.409638554,
      "logps/chosen": -469.02404626623377,
      "logps/rejected": -414.7276920180723,
      "loss": 0.2506,
      "loss/base_kto": 2.00508451461792,
      "metrics/advantage_var": 16096.7578125,
      "rewards/chosen": 2.6388319932021105,
      "rewards/margins": 5.131839974835676,
      "rewards/rejected": -2.4930079816335655,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 7.547207355499268,
      "kl": 19.215185165405273,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -48986785.77198697,
      "logits/rejected": -50203435.81981982,
      "logps/chosen": -476.5796009771987,
      "logps/rejected": -382.5185341591592,
      "loss": 0.2648,
      "loss/base_kto": 2.118427276611328,
      "metrics/advantage_var": 15124.556640625,
      "rewards/chosen": 2.5517281895740025,
      "rewards/margins": 4.824689871959513,
      "rewards/rejected": -2.2729616823855103,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 8.951262474060059,
      "kl": 19.080490112304688,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -52171221.40031397,
      "logits/rejected": -52181638.56920684,
      "logps/chosen": -454.0738324175824,
      "logps/rejected": -409.19012441679627,
      "loss": 0.2503,
      "loss/base_kto": 2.002168655395508,
      "metrics/advantage_var": 16247.0625,
      "rewards/chosen": 2.1811902871124413,
      "rewards/margins": 5.068549574024377,
      "rewards/rejected": -2.8873592869119364,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 7.5546159744262695,
      "kl": 16.825531005859375,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -51035750.4,
      "logits/rejected": -50864886.065671645,
      "logps/chosen": -432.0246413934426,
      "logps/rejected": -381.96408582089555,
      "loss": 0.249,
      "loss/base_kto": 1.9919849634170532,
      "metrics/advantage_var": 15307.09375,
      "rewards/chosen": 2.4189148949795083,
      "rewards/margins": 5.1869454089137434,
      "rewards/rejected": -2.768030513934235,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 9.785651206970215,
      "kl": 14.184760093688965,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -50667513.88656716,
      "logits/rejected": -51588018.78032787,
      "logps/chosen": -462.3922574626866,
      "logps/rejected": -409.36296106557376,
      "loss": 0.267,
      "loss/base_kto": 2.1362154483795166,
      "metrics/advantage_var": 15636.0283203125,
      "rewards/chosen": 2.073510013409515,
      "rewards/margins": 4.814888647502753,
      "rewards/rejected": -2.7413786340932376,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 7.976250648498535,
      "kl": 29.88347816467285,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -50107616.859399684,
      "logits/rejected": -49035365.29211746,
      "logps/chosen": -424.58046998420224,
      "logps/rejected": -370.0915282071097,
      "loss": 0.2641,
      "loss/base_kto": 2.1128010749816895,
      "metrics/advantage_var": 13511.4091796875,
      "rewards/chosen": 2.773539900214751,
      "rewards/margins": 4.7543954263208175,
      "rewards/rejected": -1.9808555261060665,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 7.568881988525391,
      "kl": 19.313095092773438,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -49129610.9022082,
      "logits/rejected": -48939949.57275542,
      "logps/chosen": -440.9421332807571,
      "logps/rejected": -370.9177147832817,
      "loss": 0.2474,
      "loss/base_kto": 1.9794552326202393,
      "metrics/advantage_var": 14084.15625,
      "rewards/chosen": 2.604433655362776,
      "rewards/margins": 4.982342934822915,
      "rewards/rejected": -2.3779092794601393,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 6.2476887702941895,
      "kl": 13.618782043457031,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -50499784.8488746,
      "logits/rejected": -51509512.559270516,
      "logps/chosen": -465.0994774919614,
      "logps/rejected": -411.11241451367783,
      "loss": 0.2641,
      "loss/base_kto": 2.1127593517303467,
      "metrics/advantage_var": 16641.234375,
      "rewards/chosen": 2.1833678610455185,
      "rewards/margins": 4.954416641678117,
      "rewards/rejected": -2.771048780632599,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 8.074265480041504,
      "kl": 20.739089965820312,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -51888706.983458646,
      "logits/rejected": -51872393.36585366,
      "logps/chosen": -473.73853383458646,
      "logps/rejected": -398.342987804878,
      "loss": 0.257,
      "loss/base_kto": 2.05631422996521,
      "metrics/advantage_var": 14822.2880859375,
      "rewards/chosen": 2.704866291705827,
      "rewards/margins": 4.915345680679405,
      "rewards/rejected": -2.210479388973577,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 9.068573951721191,
      "kl": 21.232534408569336,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -52277266.70319635,
      "logits/rejected": -51626911.02407704,
      "logps/chosen": -485.71004566210047,
      "logps/rejected": -399.7011687399679,
      "loss": 0.2501,
      "loss/base_kto": 2.0007503032684326,
      "metrics/advantage_var": 15226.5771484375,
      "rewards/chosen": 2.7049560546875,
      "rewards/margins": 5.207591441048857,
      "rewards/rejected": -2.5026353863613564,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 9.576692581176758,
      "kl": 28.596731185913086,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -50640729.6,
      "logits/rejected": -50948531.2,
      "logps/chosen": -447.75751953125,
      "logps/rejected": -390.83759765625,
      "loss": 0.2562,
      "loss/base_kto": 2.0498600006103516,
      "metrics/advantage_var": 14499.0576171875,
      "rewards/chosen": 2.763779067993164,
      "rewards/margins": 4.837852859497071,
      "rewards/rejected": -2.0740737915039062,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.336225509643555,
      "kl": 21.048917770385742,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -51244268.057233706,
      "logits/rejected": -53232616.40552995,
      "logps/chosen": -446.7321641494436,
      "logps/rejected": -404.64564132104454,
      "loss": 0.2442,
      "loss/base_kto": 1.9533166885375977,
      "metrics/advantage_var": 15715.361328125,
      "rewards/chosen": 2.4377893590396464,
      "rewards/margins": 5.201704249592642,
      "rewards/rejected": -2.7639148905529956,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 7.756782054901123,
      "kl": 23.05873680114746,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -53175127.69278034,
      "logits/rejected": -52764330.12400636,
      "logps/chosen": -454.0267857142857,
      "logps/rejected": -410.45588235294116,
      "loss": 0.2512,
      "loss/base_kto": 2.009918451309204,
      "metrics/advantage_var": 14705.5224609375,
      "rewards/chosen": 2.501078194004416,
      "rewards/margins": 5.026136570395513,
      "rewards/rejected": -2.525058376391097,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 10.919984817504883,
      "kl": 31.348316192626953,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -52322684.1088,
      "logits/rejected": -51838244.3480916,
      "logps/chosen": -434.9367,
      "logps/rejected": -396.92793416030537,
      "loss": 0.2649,
      "loss/base_kto": 2.1192855834960938,
      "metrics/advantage_var": 15045.2470703125,
      "rewards/chosen": 2.8001419921875,
      "rewards/margins": 4.800822044966603,
      "rewards/rejected": -2.000680052779103,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 9.527816772460938,
      "kl": 12.20826530456543,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -51634665.3164557,
      "logits/rejected": -52495334.71604938,
      "logps/chosen": -464.9095628955696,
      "logps/rejected": -403.5522762345679,
      "loss": 0.2516,
      "loss/base_kto": 2.012443780899048,
      "metrics/advantage_var": 15119.6943359375,
      "rewards/chosen": 2.351209616359276,
      "rewards/margins": 5.110908178039446,
      "rewards/rejected": -2.7596985616801697,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 9.87900447845459,
      "kl": 17.386611938476562,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -53422185.43525741,
      "logits/rejected": -53120264.41314554,
      "logps/chosen": -482.53310257410294,
      "logps/rejected": -407.90179968701096,
      "loss": 0.2483,
      "loss/base_kto": 1.986655592918396,
      "metrics/advantage_var": 16186.7998046875,
      "rewards/chosen": 2.3212818258824104,
      "rewards/margins": 5.302370479112556,
      "rewards/rejected": -2.981088653230145,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 6.876319408416748,
      "kl": 27.84088706970215,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -52926714.064914994,
      "logits/rejected": -54163553.06161138,
      "logps/chosen": -463.3681897217929,
      "logps/rejected": -392.51132997630333,
      "loss": 0.2423,
      "loss/base_kto": 1.9387661218643188,
      "metrics/advantage_var": 15857.3505859375,
      "rewards/chosen": 2.6382854762364762,
      "rewards/margins": 5.196615029825536,
      "rewards/rejected": -2.55832955358906,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 9.612847328186035,
      "kl": 19.885042190551758,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -50576158.91021672,
      "logits/rejected": -52167256.83280757,
      "logps/chosen": -458.50880417956654,
      "logps/rejected": -413.5228213722398,
      "loss": 0.2445,
      "loss/base_kto": 1.955782175064087,
      "metrics/advantage_var": 16214.2626953125,
      "rewards/chosen": 2.4565017747436144,
      "rewards/margins": 5.124602222052369,
      "rewards/rejected": -2.668100447308754,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 7.085169315338135,
      "kl": 19.323680877685547,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -51409870.65060241,
      "logits/rejected": -51447382.44155844,
      "logps/chosen": -442.01571912650604,
      "logps/rejected": -404.3957234172078,
      "loss": 0.2585,
      "loss/base_kto": 2.0679147243499756,
      "metrics/advantage_var": 14929.962890625,
      "rewards/chosen": 2.428604861339891,
      "rewards/margins": 4.838879812828832,
      "rewards/rejected": -2.4102749514889408,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 5.093369007110596,
      "kl": 21.76449203491211,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -51313167.176287055,
      "logits/rejected": -51953249.25588697,
      "logps/chosen": -461.6126657566303,
      "logps/rejected": -384.9188088697017,
      "loss": 0.2277,
      "loss/base_kto": 1.8218530416488647,
      "metrics/advantage_var": 15466.078125,
      "rewards/chosen": 3.0937618071129096,
      "rewards/margins": 5.48768481351499,
      "rewards/rejected": -2.3939230064020802,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 6.250664234161377,
      "kl": 11.092265129089355,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -52543289.6,
      "logits/rejected": -52555180.8,
      "logps/chosen": -448.8865234375,
      "logps/rejected": -426.451171875,
      "loss": 0.2125,
      "loss/base_kto": 1.699960708618164,
      "metrics/advantage_var": 17586.404296875,
      "rewards/chosen": 2.5158063888549806,
      "rewards/margins": 5.949070167541504,
      "rewards/rejected": -3.4332637786865234,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 7.516881942749023,
      "kl": 27.575057983398438,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -52040782.51757188,
      "logits/rejected": -53780530.103975534,
      "logps/chosen": -457.1747703674121,
      "logps/rejected": -412.95508409785936,
      "loss": 0.2147,
      "loss/base_kto": 1.7175458669662476,
      "metrics/advantage_var": 15806.40625,
      "rewards/chosen": 2.945287149935104,
      "rewards/margins": 5.667440186536786,
      "rewards/rejected": -2.722153036601682,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 7.4503560066223145,
      "kl": 22.494722366333008,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -53130994.61172742,
      "logits/rejected": -53505965.95377504,
      "logps/chosen": -474.13960974643425,
      "logps/rejected": -411.5372688751926,
      "loss": 0.2132,
      "loss/base_kto": 1.7054146528244019,
      "metrics/advantage_var": 17508.853515625,
      "rewards/chosen": 2.9541907454065965,
      "rewards/margins": 5.814634848112682,
      "rewards/rejected": -2.860444102706086,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 10.237329483032227,
      "kl": 13.12766170501709,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -53387424.87009063,
      "logits/rejected": -52597710.291262135,
      "logps/chosen": -483.8654645015106,
      "logps/rejected": -389.9562601132686,
      "loss": 0.2036,
      "loss/base_kto": 1.6287109851837158,
      "metrics/advantage_var": 14979.9921875,
      "rewards/chosen": 2.758353149782855,
      "rewards/margins": 5.7842583268282635,
      "rewards/rejected": -3.0259051770454084,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 7.691054344177246,
      "kl": 18.150373458862305,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -51640392.803709425,
      "logits/rejected": -53083939.99368089,
      "logps/chosen": -438.0470923493045,
      "logps/rejected": -424.92683649289097,
      "loss": 0.2083,
      "loss/base_kto": 1.6663763523101807,
      "metrics/advantage_var": 17248.298828125,
      "rewards/chosen": 2.568194098966383,
      "rewards/margins": 5.949468088771873,
      "rewards/rejected": -3.3812739898054898,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 6.853572845458984,
      "kl": 15.213935852050781,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -53337403.43550835,
      "logits/rejected": -54221581.60386474,
      "logps/chosen": -459.4839719271624,
      "logps/rejected": -401.6209490740741,
      "loss": 0.2244,
      "loss/base_kto": 1.794970154762268,
      "metrics/advantage_var": 14274.5849609375,
      "rewards/chosen": 2.6967603020675264,
      "rewards/margins": 5.361152647452993,
      "rewards/rejected": -2.664392345385467,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 6.441402912139893,
      "kl": 16.609342575073242,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51576548.280254774,
      "logits/rejected": -52661995.582822084,
      "logps/chosen": -469.22103901273886,
      "logps/rejected": -389.69545628834356,
      "loss": 0.205,
      "loss/base_kto": 1.6403454542160034,
      "metrics/advantage_var": 16205.587890625,
      "rewards/chosen": 2.563711761668989,
      "rewards/margins": 5.923686628664771,
      "rewards/rejected": -3.359974866995782,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 8.55733871459961,
      "kl": 25.37789535522461,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -52256164.345679015,
      "logits/rejected": -52587818.12658228,
      "logps/chosen": -493.7014371141975,
      "logps/rejected": -412.27558840981015,
      "loss": 0.2122,
      "loss/base_kto": 1.6972593069076538,
      "metrics/advantage_var": 16703.544921875,
      "rewards/chosen": 2.814942913290895,
      "rewards/margins": 5.731951325326101,
      "rewards/rejected": -2.9170084120352056,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 8.511039733886719,
      "kl": 16.398456573486328,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -51757954.96754251,
      "logits/rejected": -52324258.17377567,
      "logps/chosen": -449.0143933539413,
      "logps/rejected": -397.59809439178514,
      "loss": 0.2144,
      "loss/base_kto": 1.7153100967407227,
      "metrics/advantage_var": 15016.7314453125,
      "rewards/chosen": 2.92261194998551,
      "rewards/margins": 5.685572423116928,
      "rewards/rejected": -2.7629604731314177,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 7.376357078552246,
      "kl": 16.149038314819336,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -52741254.69413629,
      "logits/rejected": -54535728.12326656,
      "logps/chosen": -455.31596671949285,
      "logps/rejected": -397.3636122881356,
      "loss": 0.2114,
      "loss/base_kto": 1.6911267042160034,
      "metrics/advantage_var": 17047.3984375,
      "rewards/chosen": 2.559685998786648,
      "rewards/margins": 5.678751011667041,
      "rewards/rejected": -3.1190650128803927,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.0808024406433105,
      "kl": 19.327743530273438,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -51179646.95595432,
      "logits/rejected": -52681646.63268366,
      "logps/chosen": -452.35924755301795,
      "logps/rejected": -393.6358695652174,
      "loss": 0.2079,
      "loss/base_kto": 1.6628249883651733,
      "metrics/advantage_var": 17884.87890625,
      "rewards/chosen": 2.5431923796263254,
      "rewards/margins": 5.888858169778125,
      "rewards/rejected": -3.345665790151799,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 7.206068515777588,
      "kl": 13.962677001953125,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -52635359.09717868,
      "logits/rejected": -52769335.82554517,
      "logps/chosen": -459.94303487460814,
      "logps/rejected": -409.59657320872276,
      "loss": 0.2112,
      "loss/base_kto": 1.6896650791168213,
      "metrics/advantage_var": 16756.390625,
      "rewards/chosen": 2.7216389336182405,
      "rewards/margins": 5.849456438120772,
      "rewards/rejected": -3.127817504502531,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 7.377704620361328,
      "kl": 22.29719352722168,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -54559073.429892145,
      "logits/rejected": -54657004.52614897,
      "logps/chosen": -472.5718412942989,
      "logps/rejected": -420.8526148969889,
      "loss": 0.2174,
      "loss/base_kto": 1.7388498783111572,
      "metrics/advantage_var": 15162.697265625,
      "rewards/chosen": 2.6641059487552967,
      "rewards/margins": 5.5746843182828325,
      "rewards/rejected": -2.910578369527536,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 6.108307361602783,
      "kl": 18.389760971069336,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -51068998.73155416,
      "logits/rejected": -52572131.334370136,
      "logps/chosen": -464.33035714285717,
      "logps/rejected": -417.0138024883359,
      "loss": 0.2056,
      "loss/base_kto": 1.645011305809021,
      "metrics/advantage_var": 15969.048828125,
      "rewards/chosen": 2.8435683317798275,
      "rewards/margins": 5.9239052154671334,
      "rewards/rejected": -3.0803368836873055,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 7.382128715515137,
      "kl": 11.044879913330078,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -51034927.22330097,
      "logits/rejected": -51440080.04833837,
      "logps/chosen": -439.081715210356,
      "logps/rejected": -402.0004248489426,
      "loss": 0.2036,
      "loss/base_kto": 1.6291121244430542,
      "metrics/advantage_var": 15075.765625,
      "rewards/chosen": 2.762153502035295,
      "rewards/margins": 5.691052892495076,
      "rewards/rejected": -2.928899390459781,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 7.5083699226379395,
      "kl": 9.02786922454834,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -51969928.23391813,
      "logits/rejected": -53152341.906040266,
      "logps/chosen": -435.16282894736844,
      "logps/rejected": -393.66335465604027,
      "loss": 0.2005,
      "loss/base_kto": 1.6041138172149658,
      "metrics/advantage_var": 16324.6689453125,
      "rewards/chosen": 2.6327168425621346,
      "rewards/margins": 5.971829809214819,
      "rewards/rejected": -3.3391129666526846,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 6.553695201873779,
      "kl": 22.991432189941406,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -52758514.76575121,
      "logits/rejected": -52305994.36006051,
      "logps/chosen": -456.5091377221325,
      "logps/rejected": -399.68253593040845,
      "loss": 0.2148,
      "loss/base_kto": 1.7184559106826782,
      "metrics/advantage_var": 15160.0341796875,
      "rewards/chosen": 2.5292734075247374,
      "rewards/margins": 5.505444026733323,
      "rewards/rejected": -2.9761706192085855,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.786435127258301,
      "kl": 11.857638359069824,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -52542490.496194825,
      "logits/rejected": -53520084.85393258,
      "logps/chosen": -461.29119101978694,
      "logps/rejected": -406.9080557784912,
      "loss": 0.2095,
      "loss/base_kto": 1.675807237625122,
      "metrics/advantage_var": 16317.4560546875,
      "rewards/chosen": 2.6273080021641935,
      "rewards/margins": 5.775256437156168,
      "rewards/rejected": -3.1479484349919744,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 8.449359893798828,
      "kl": 22.929365158081055,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -53763288.906403944,
      "logits/rejected": -54059221.65126677,
      "logps/chosen": -457.6953509852217,
      "logps/rejected": -407.5998975409836,
      "loss": 0.2096,
      "loss/base_kto": 1.6766414642333984,
      "metrics/advantage_var": 15792.712890625,
      "rewards/chosen": 2.634127812628284,
      "rewards/margins": 5.735816203323143,
      "rewards/rejected": -3.1016883906948585,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 8.70055866241455,
      "kl": 14.306711196899414,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -52797474.7654321,
      "logits/rejected": -53016200.101265825,
      "logps/chosen": -480.6525848765432,
      "logps/rejected": -392.47201344936707,
      "loss": 0.1963,
      "loss/base_kto": 1.5704221725463867,
      "metrics/advantage_var": 15938.1923828125,
      "rewards/chosen": 2.8278782220534335,
      "rewards/margins": 5.77576037939125,
      "rewards/rejected": -2.9478821573378164,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 8.104622840881348,
      "kl": 15.392233848571777,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -51529518.4372093,
      "logits/rejected": -53984957.48031496,
      "logps/chosen": -461.63788759689925,
      "logps/rejected": -396.53041338582676,
      "loss": 0.1926,
      "loss/base_kto": 1.5410044193267822,
      "metrics/advantage_var": 16571.763671875,
      "rewards/chosen": 2.636381116763566,
      "rewards/margins": 6.110318432216322,
      "rewards/rejected": -3.4739373154527557,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 6.42228364944458,
      "kl": 10.110087394714355,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -51332418.37037037,
      "logits/rejected": -52560494.17721519,
      "logps/chosen": -452.5120563271605,
      "logps/rejected": -386.1056170886076,
      "loss": 0.211,
      "loss/base_kto": 1.6881433725357056,
      "metrics/advantage_var": 16214.1220703125,
      "rewards/chosen": 2.722373679832176,
      "rewards/margins": 5.686483462763347,
      "rewards/rejected": -2.9641097829311707,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 6.703454971313477,
      "kl": 16.846250534057617,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52858190.76923077,
      "logits/rejected": -53259863.414634146,
      "logps/chosen": -457.1239983974359,
      "logps/rejected": -412.1909298780488,
      "loss": 0.2052,
      "loss/base_kto": 1.641872763633728,
      "metrics/advantage_var": 17799.373046875,
      "rewards/chosen": 2.7151457957732372,
      "rewards/margins": 6.064503959002087,
      "rewards/rejected": -3.349358163228849,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 6.7395477294921875,
      "kl": 16.90166664123535,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -52330051.62264151,
      "logits/rejected": -53894061.31677019,
      "logps/chosen": -482.8420794025157,
      "logps/rejected": -388.3456424689441,
      "loss": 0.2216,
      "loss/base_kto": 1.7730960845947266,
      "metrics/advantage_var": 15315.8974609375,
      "rewards/chosen": 2.178255093172661,
      "rewards/margins": 5.402397416542227,
      "rewards/rejected": -3.2241423233695654,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 6.1936163902282715,
      "kl": 11.923944473266602,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -51702307.17557252,
      "logits/rejected": -51026467.2256,
      "logps/chosen": -469.8072041984733,
      "logps/rejected": -390.839975,
      "loss": 0.2053,
      "loss/base_kto": 1.6426628828048706,
      "metrics/advantage_var": 15595.1533203125,
      "rewards/chosen": 2.7488445252862594,
      "rewards/margins": 5.67882479872376,
      "rewards/rejected": -2.9299802734375,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 8.137475967407227,
      "kl": 18.132556915283203,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -52089820.04274809,
      "logits/rejected": -52295008.256,
      "logps/chosen": -460.9177480916031,
      "logps/rejected": -393.893775,
      "loss": 0.2014,
      "loss/base_kto": 1.6112679243087769,
      "metrics/advantage_var": 17538.55078125,
      "rewards/chosen": 2.5705998777433208,
      "rewards/margins": 6.143112377743321,
      "rewards/rejected": -3.5725125,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 8.598886489868164,
      "kl": 16.467830657958984,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -52820833.26443769,
      "logits/rejected": -53313503.073954985,
      "logps/chosen": -429.13169642857144,
      "logps/rejected": -384.4918609324759,
      "loss": 0.2201,
      "loss/base_kto": 1.7607765197753906,
      "metrics/advantage_var": 14958.587890625,
      "rewards/chosen": 2.4377450682109614,
      "rewards/margins": 5.479708406296271,
      "rewards/rejected": -3.0419633380853095,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 7.395063400268555,
      "kl": 12.96544361114502,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -53403948.905547224,
      "logits/rejected": -55372835.079934746,
      "logps/chosen": -457.15821776611693,
      "logps/rejected": -412.7927966965742,
      "loss": 0.2175,
      "loss/base_kto": 1.7399282455444336,
      "metrics/advantage_var": 15877.1318359375,
      "rewards/chosen": 2.6581939743019114,
      "rewards/margins": 5.8172340675293785,
      "rewards/rejected": -3.1590400932274676,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 5.516819477081299,
      "kl": 13.18786334991455,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -53165266.95939849,
      "logits/rejected": -52912474.328455284,
      "logps/chosen": -427.49783834586464,
      "logps/rejected": -416.99463922764227,
      "loss": 0.2309,
      "loss/base_kto": 1.847333550453186,
      "metrics/advantage_var": 15203.5458984375,
      "rewards/chosen": 2.697485902255639,
      "rewards/margins": 5.31505747714131,
      "rewards/rejected": -2.617571574885671,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 8.08092975616455,
      "kl": 24.771360397338867,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -52021904.905660376,
      "logits/rejected": -52015479.254658386,
      "logps/chosen": -466.30984669811323,
      "logps/rejected": -414.8922748447205,
      "loss": 0.2214,
      "loss/base_kto": 1.7714513540267944,
      "metrics/advantage_var": 15380.9560546875,
      "rewards/chosen": 2.7778289602987423,
      "rewards/margins": 5.55659945446363,
      "rewards/rejected": -2.7787704941648874,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 6.716619968414307,
      "kl": 17.462905883789062,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -50473313.774960384,
      "logits/rejected": -50735018.798151,
      "logps/chosen": -463.6503565768621,
      "logps/rejected": -398.61529757318954,
      "loss": 0.1975,
      "loss/base_kto": 1.579751968383789,
      "metrics/advantage_var": 15571.666015625,
      "rewards/chosen": 2.759819985947405,
      "rewards/margins": 5.925720123779646,
      "rewards/rejected": -3.165900137832242,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 7.008581161499023,
      "kl": 23.658918380737305,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -51330135.21766561,
      "logits/rejected": -53182067.71517028,
      "logps/chosen": -464.6318020504732,
      "logps/rejected": -398.1510739164087,
      "loss": 0.213,
      "loss/base_kto": 1.7040843963623047,
      "metrics/advantage_var": 13738.359375,
      "rewards/chosen": 2.749936076744874,
      "rewards/margins": 5.473095347134773,
      "rewards/rejected": -2.7231592703898992,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 6.6605424880981445,
      "kl": 15.576194763183594,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -51471504.935384616,
      "logits/rejected": -52842918.603174604,
      "logps/chosen": -452.46923076923076,
      "logps/rejected": -405.70731646825396,
      "loss": 0.2131,
      "loss/base_kto": 1.7047852277755737,
      "metrics/advantage_var": 15001.361328125,
      "rewards/chosen": 2.5818241060697114,
      "rewards/margins": 5.591657935434791,
      "rewards/rejected": -3.0098338293650793,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 10.156596183776855,
      "kl": 21.255380630493164,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -50288277.02113821,
      "logits/rejected": -52447093.41353384,
      "logps/chosen": -464.1432926829268,
      "logps/rejected": -394.63082706766914,
      "loss": 0.1964,
      "loss/base_kto": 1.5715006589889526,
      "metrics/advantage_var": 15936.7177734375,
      "rewards/chosen": 2.7470715034298783,
      "rewards/margins": 5.898373709139465,
      "rewards/rejected": -3.1513022057095865,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 7.635325908660889,
      "kl": 15.98388957977295,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -52831451.777424484,
      "logits/rejected": -54620543.80337942,
      "logps/chosen": -443.12117448330685,
      "logps/rejected": -405.9448924731183,
      "loss": 0.2129,
      "loss/base_kto": 1.7034294605255127,
      "metrics/advantage_var": 15774.0283203125,
      "rewards/chosen": 2.692273356766693,
      "rewards/margins": 5.745154178639293,
      "rewards/rejected": -3.0528808218726,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 9.039216995239258,
      "kl": 18.50642204284668,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -54375990.36450079,
      "logits/rejected": -54110608.7642527,
      "logps/chosen": -487.3832210776545,
      "logps/rejected": -377.65841679506934,
      "loss": 0.2132,
      "loss/base_kto": 1.7055686712265015,
      "metrics/advantage_var": 14751.453125,
      "rewards/chosen": 2.72312112315521,
      "rewards/margins": 5.618464733216362,
      "rewards/rejected": -2.895343610061152,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 8.901484489440918,
      "kl": 22.53584098815918,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -54056707.63981043,
      "logits/rejected": -53963812.401854716,
      "logps/chosen": -465.6426737756714,
      "logps/rejected": -418.4019513137558,
      "loss": 0.1967,
      "loss/base_kto": 1.573914885520935,
      "metrics/advantage_var": 15834.7001953125,
      "rewards/chosen": 2.8055148011700237,
      "rewards/margins": 5.915360535991797,
      "rewards/rejected": -3.1098457348217736,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.978126922009805e+17,
      "train_loss": 0.2728243423867102,
      "train_runtime": 11019.9549,
      "train_samples_per_second": 13.45,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.978126922009805e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}